Large refactor. Cleans up, adds tests, intermediate namespaces

Indexes are still broken due to past changes, but intermediate namespaces are
now linkable.
go-templates
Anthony Johnson 9 years ago
parent 6eae5e4eab
commit 4b13bebc8e

3
.gitignore vendored

@ -1,5 +1,8 @@
*.pyc
*.egg-info
.eggs/
*.egg
build
dist
wheelhouse
*.sw[opmn]

@ -1,3 +1,5 @@
import yaml
from .settings import env
@ -33,3 +35,40 @@ class UnknownType(AutoAPIBase):
def render(self, ctx=None):
print "Unknown Type: %s" % (self.obj['type'])
super(UnknownType, self).render(ctx=ctx)
class AutoAPIDomain(object):
'''Base class for domain handling
:param app: Sphinx application instance
'''
namespaces = {}
objects = []
def __init__(self, app):
self.app = app
def read_file(self, path):
'''Read file input into memory, returning deserialized objects
:param path: Path of file to read
'''
# TODO support JSON here
# TODO sphinx way of reporting errors in logs?
try:
with open(path, 'r') as handle:
obj = yaml.safe_load(handle)
except IOError:
raise Warning('Error reading file: {0}'.format(path))
except yaml.YAMLError:
raise Warning('Error parsing file: {0}'.format(path))
return obj
def create_class(self, obj):
'''Create class object from obj'''
raise NotImplementedError
def get_config(self, key):
if self.app.config is not None:
return getattr(self.app.config, key, None)

@ -0,0 +1,2 @@
from .dotnet import DotNetDomain
# from .python import PythonDomain

@ -0,0 +1,296 @@
import os
from collections import defaultdict
from sphinx.util.console import bold, darkgreen
from sphinx.util.osutil import ensuredir
from ..base import AutoAPIBase, AutoAPIDomain
from ..settings import env
class DotNetDomain(AutoAPIDomain):
'''Auto API domain handler for .NET
Searches for YAML files, and soon to be JSON files as well, for auto API
sources
:param app: Sphinx application passed in as part of the extension
'''
def find_files(self):
'''Find YAML/JSON files to parse for namespace information'''
# TODO do an intelligent glob here, we're picking up too much
files_to_read = os.listdir(self.get_config('autoapi_dir'))
for _path in self.app.status_iterator(
files_to_read,
'[AutoAPI] Reading files... ',
darkgreen,
len(files_to_read)):
yield _path
def create_class(self, data):
'''Return instance of class based on Roslyn type property
Data keys handled here:
type
Set the object class
items
Recurse into :py:meth:`create_class` to create child object
instances
:param data: dictionary data from Roslyn output artifact
'''
# TODO replace this with a global mapping
classes = [DotNetNamespace, DotNetClass, DotNetProperty, DotNetMethod,
DotNetEnum, DotNetConstructor, DotNetStruct, DotNetInterface,
DotNetDelegate, DotNetField, DotNetEvent]
obj = None
for cls in classes:
if data['type'].lower() == cls.type.lower():
obj = cls(data)
# Append child objects
# TODO this should recurse in the case we're getting back more complex
# argument listings
if 'items' in data:
obj.children = []
obj.item_map = defaultdict(list)
for item in data['items']:
child_obj = self.create_class(item)
obj.children.append(child_obj)
obj.item_map[item['type']].append(child_obj)
return obj
def get_objects(self):
'''Trigger find of serialized sources and build objects'''
for path in self.find_files():
data = self.read_file(os.path.join(self.get_config('autoapi_dir'), path))
obj = self.create_class(data)
self.add_object(obj)
def add_object(self, obj):
'''Add object to local and app environment storage
:param obj: Instance of a .NET object
'''
self.app.env.autoapi_data.append(obj)
self.objects.append(obj)
def sort_objects(self):
'''Not implemented yet'''
pass
# print "Sorting objects"
# Sort objects
# for obj in app.env.autoapi_data:
# rst = parse(obj, 'dotnet')
# if rst:
# path = os.path.join(app.config.autoapi_root, '%s%s' % (obj['name']['CSharp'], app.config.source_suffix[0]))
# ensuredir(app.config.autoapi_root)
# with open(path, 'w+') as fp:
# fp.write(rst)
def organize_objects(self):
'''Organize objects and namespaces'''
def _recurse_ns(obj):
namespace = obj.namespace()
if namespace is not None:
ns_obj = None
for (n, search_obj) in enumerate(self.app.env.autoapi_data):
if (search_obj.id == namespace and
isinstance(search_obj, DotNetNamespace)):
ns_obj = self.app.env.autoapi_data[n]
if ns_obj is None:
ns_obj = self.create_class({'id': namespace,
'type': 'namespace'})
self.app.env.autoapi_data.append(ns_obj)
self.namespaces[ns_obj.id] = ns_obj
if obj not in ns_obj.children:
ns_obj.children.append(obj)
_recurse_ns(ns_obj)
for obj in self.app.env.autoapi_data:
_recurse_ns(obj)
def full(self):
print "Reading"
self.get_objects()
self.organize_objects()
print "Writing"
self.generate_output()
self.write_indexes()
def generate_output(self):
# for namespace, objs in namespaces.items():
for obj in self.app.env.autoapi_data:
# path = os.path.join(app.config.autoapi_root, '%s%s' % (namespace, app.config.source_suffix[0]))
# namespace_obj = DotNetNamespace(namespace, objs)
# ensuredir(app.config.autoapi_root)
# with open(path, 'w+') as index_file:
# namespace_rst = namespace_obj.render()
# if namespace_rst:
# index_file.write(namespace_rst)
# for obj in objs:
# TODO not here!
obj.item_map = defaultdict(list)
for child in obj.children:
obj.item_map[child.type].append(child)
rst = obj.render()
# Detail
detail_dir = os.path.join(self.get_config('autoapi_root'),
*obj.name.split('.'))
ensuredir(detail_dir)
path = os.path.join(detail_dir, '%s%s' % ('index', self.get_config('source_suffix')[0]))
if rst:
with open(path, 'w+') as detail_file:
detail_file.write(rst)
for namespace, obj in self.namespaces.items():
path = os.path.join(self.get_config('autoapi_root'), '%s%s' % (namespace, self.get_config('source_suffix')[0]))
ensuredir(self.get_config('autoapi_root'))
with open(path, 'w+') as index_file:
namespace_rst = obj.render()
if namespace_rst:
index_file.write(namespace_rst)
def write_indexes(self):
# Write Index
top_level_index = os.path.join(self.get_config('autoapi_root'),
'index.rst')
with open(top_level_index, 'w+') as top_level_file:
content = env.get_template('index.rst')
top_level_file.write(content.render())
class DotNetBase(AutoAPIBase):
'''Base .NET object representation'''
language = 'dotnet'
def __init__(self, obj):
super(DotNetBase, self).__init__(obj)
# Always exist
self.id = obj['id']
self.type = obj['type']
# Use name or id
try:
self.name = obj['qualifiedName']['CSharp']
except:
self.name = self.id
self.short_name = self.name.split('.')[-1]
# Optional
self.summary = obj.get('summary', '')
self.parameters = []
self.items = []
self.children = []
# Syntax example and parameter list
syntax = obj.get('syntax', None)
self.example = ''
if syntax is not None:
# Code example
try:
self.example = syntax['content']['CSharp']
except KeyError:
pass
self.parameters = []
for param in syntax.get('parameters', []):
if 'id' in param:
self.parameters.append({
'name': param.get('id'),
'type': param.get('type', {}).get('id', None),
'desc': param.get('description', '')
})
self.items = obj.get('items', [])
def __str__(self):
return '<{cls} {id}>'.format(cls=self.__class__.__name__,
id=self.id)
def namespace(self):
pieces = self.id.split('.')[:-1]
if pieces:
return '.'.join(pieces)
@property
def ref_type(self):
return self.type.lower().replace('class', 'cls').replace('interface', 'iface').replace('delegate', 'del')
def to_ref_type(self, _type):
return _type.lower().replace('class', 'cls').replace('interface', 'iface').replace('delegate', 'del')
class DotNetNamespace(DotNetBase):
type = 'namespace'
reftype = 'ns'
class DotNetMethod(DotNetBase):
type = 'method'
reftype = 'meth'
class DotNetProperty(DotNetBase):
type = 'property'
reftype = 'prop'
class DotNetEnum(DotNetBase):
type = 'enum'
reftype = 'enum'
class DotNetStruct(DotNetBase):
type = 'struct'
reftype = 'struct'
class DotNetConstructor(DotNetBase):
type = 'constructor'
reftype = 'ctor'
class DotNetInterface(DotNetBase):
type = 'interface'
reftype = 'iface'
class DotNetDelegate(DotNetBase):
type = 'delegate'
reftype = 'del'
class DotNetClass(DotNetBase):
type = 'class'
reftype = 'cls'
class DotNetField(DotNetBase):
type = 'field'
reftype = 'field'
class DotNetEvent(DotNetBase):
type = 'event'
reftype = 'event'
class DotNetVirtualNamespace(AutoAPIBase):
language = 'dotnet'
type = 'namespace'
reftype = 'ns'
def __init__(self, name, objs):
self.name = self.short_name = name
self.children = []
self.type = 'namespace'
for obj in objs:
self.children.append(obj.obj)

@ -1,9 +1,35 @@
import sys
from collections import defaultdict
from .base import AutoAPIBase
from .settings import env
from epyparse import parsed
from ..base import AutoAPIBase
from ..settings import env
# for root, dirnames, filenames in os.walk(app.config.autoapi_dir):
# for filename in fnmatch.filter(filenames, u'*.py'):
# to_open = os.path.join(root, filename)
# if ignore_file(app, to_open):
# continue
# # print "Parsing Python File from %s" % to_open
# try:
# parsed_data = parsed(to_open)
# app.env.autoapi_data.append(parsed_data)
# except Exception:
# print "Exception, Keeping going: %s" % to_open
# import traceback
# traceback.print_exc()
# app.env.autoapi_enabled = True
#
# # Generate RST
# for obj in app.env.autoapi_data:
# # print "Parsing %s" % obj['fullname']
# rst = classify(obj, 'python').render()
# if rst:
# path = os.path.join(app.config.autoapi_root, '%s%s' % (obj['fullname'], app.config.source_suffix[0]))
# ensuredir(app.config.autoapi_root)
# with open(path, 'w+') as fp:
# fp.write(rst.encode('utf8'))
class PythonBase(AutoAPIBase):

@ -1,121 +0,0 @@
from collections import defaultdict
from .base import AutoAPIBase
from .settings import env
class DotNetBase(AutoAPIBase):
language = 'dotnet'
def __init__(self, obj):
super(DotNetBase, self).__init__(obj)
# Always exist
self.id = obj['id']
self.type = obj['type']
# Use name or id
try:
self.name = obj['qualifiedName']['CSharp']
except:
self.name = self.id
self.short_name = self.name.split('.')[-1]
self.namespace = self.name.split('.')[0]
# Optional
self.summary = obj.get('summary', '')
# Syntax example and parameter list
syntax = obj.get('syntax', None)
self.example = ''
if syntax is not None:
# Code example
try:
self.example = syntax['content']['CSharp']
except KeyError:
pass
self.parameters = []
for param in syntax.get('parameters', []):
if 'id' in param:
self.parameters.append({
'name': param.get('id'),
'type': param.get('type', {}).get('id', None),
'desc': param.get('description', '')
})
self.children = obj.get('items', [])
if self.children:
self.item_map = defaultdict(list)
self.sort()
@property
def ref_type(self):
return self.type.lower().replace('class', 'cls').replace('interface', 'iface').replace('delegate', 'del')
def to_ref_type(self, _type):
return _type.lower().replace('class', 'cls').replace('interface', 'iface').replace('delegate', 'del')
def sort(self):
from .utils import classify
for item in self.children:
if 'type' not in item:
print "Missing Type: %s" % item
continue
classified = classify(item, 'dotnet')
self.item_map[item['type']].append(classified)
class DotNetNamespace(DotNetBase):
type = 'namespace'
class DotNetMethod(DotNetBase):
type = 'method'
class DotNetProperty(DotNetBase):
type = 'property'
class DotNetEnum(DotNetBase):
type = 'enum'
class DotNetStruct(DotNetBase):
type = 'struct'
class DotNetConstructor(DotNetBase):
type = 'constructor'
class DotNetInterface(DotNetBase):
type = 'interface'
class DotNetDelegate(DotNetBase):
type = 'delegate'
class DotNetClass(DotNetBase):
type = 'class'
class DotNetField(DotNetBase):
type = 'field'
class DotNetEvent(DotNetBase):
type = 'event'
class VirtualNamespace(AutoAPIBase):
language = 'dotnet'
type = 'namespace'
def __init__(self, name, objs):
self.name = self.short_name = name
self.children = []
self.type = 'namespace'
for obj in objs:
self.children.append(obj.obj)

@ -4,20 +4,13 @@ Sphinx Auto-API
"""
import os
import yaml
import fnmatch
import shutil
from collections import defaultdict
import traceback
from sphinx.util.osutil import ensuredir
from sphinx.util.console import bold, darkgreen
from .utils import classify
from .dotnet import VirtualNamespace
from .settings import env
from epyparse import parsed
from .domains import *
def ignore_file(app, filename):
@ -35,102 +28,10 @@ def load_yaml(app):
namespaces = defaultdict(list)
if app.config.autoapi_type == 'dotnet':
print "Reading Autodoc Yaml"
# Load Yaml
files_to_read = os.listdir(app.config.autoapi_dir)
for _file in app.status_iterator(
files_to_read, '[AutoAPI] Reading files... ', darkgreen, len(files_to_read)):
# print "Loading Yaml from %s" % _file
to_open = os.path.join(app.config.autoapi_dir, _file)
yaml_obj = yaml.safe_load(open(to_open, 'r'))
obj = classify(yaml_obj, 'dotnet')
app.env.autoapi_data.append(obj)
# Add to namespace dict
if yaml_obj.get('type') != 'Namespace':
try:
top, mid, last = obj.name.split('.')[0:3]
namespaces[top].append(obj)
namespaces[top+'.'+mid].append(obj)
# namespaces[top+'.'+mid+'.'+last].append(obj)
except:
traceback.print_exc()
pass
# print "Sorting objects"
# Sort objects
# for obj in app.env.autoapi_data:
# rst = parse(obj, 'dotnet')
# if rst:
# path = os.path.join(app.config.autoapi_root, '%s%s' % (obj['name']['CSharp'], app.config.source_suffix[0]))
# ensuredir(app.config.autoapi_root)
# with open(path, 'w+') as fp:
# fp.write(rst)
print "Generating RST"
# Generate RST
# for namespace, objs in namespaces.items():
for obj in app.env.autoapi_data:
# path = os.path.join(app.config.autoapi_root, '%s%s' % (namespace, app.config.source_suffix[0]))
# namespace_obj = DotNetNamespace(namespace, objs)
# ensuredir(app.config.autoapi_root)
# with open(path, 'w+') as index_file:
# namespace_rst = namespace_obj.render()
# if namespace_rst:
# index_file.write(namespace_rst)
# for obj in objs:
rst = obj.render()
# Detail
detail_dir = os.path.join(app.config.autoapi_root, *obj.name.split('.'))
ensuredir(detail_dir)
path = os.path.join(detail_dir, '%s%s' % ('index', app.config.source_suffix[0]))
if rst:
with open(path, 'w+') as detail_file:
detail_file.write(rst)
for namespace, objs in namespaces.items():
namespace_obj = VirtualNamespace(namespace, objs)
ensuredir(app.config.autoapi_root)
virtual_dir = os.path.join(app.config.autoapi_root, 'Virtual')
ensuredir(virtual_dir)
virtual_path = os.path.join(virtual_dir, '%s%s' % (namespace, app.config.source_suffix[0]))
with open(virtual_path, 'w+') as index_file:
namespace_rst = namespace_obj.render()
if namespace_rst:
index_file.write(namespace_rst)
# Write Index
top_level_index = os.path.join(app.config.autoapi_root, 'index.rst')
with open(top_level_index, 'w+') as top_level_file:
content = env.get_template('index.rst')
top_level_file.write(content.render())
elif app.config.autoapi_type == 'python':
for root, dirnames, filenames in os.walk(app.config.autoapi_dir):
for filename in fnmatch.filter(filenames, u'*.py'):
to_open = os.path.join(root, filename)
if ignore_file(app, to_open):
continue
# print "Parsing Python File from %s" % to_open
try:
parsed_data = parsed(to_open)
app.env.autoapi_data.append(parsed_data)
except Exception:
print "Exception, Keeping going: %s" % to_open
import traceback
traceback.print_exc()
app.env.autoapi_enabled = True
# Generate RST
for obj in app.env.autoapi_data:
# print "Parsing %s" % obj['fullname']
rst = classify(obj, 'python').render()
if rst:
path = os.path.join(app.config.autoapi_root, '%s%s' % (obj['fullname'], app.config.source_suffix[0]))
ensuredir(app.config.autoapi_root)
with open(path, 'w+') as fp:
fp.write(rst.encode('utf8'))
domain = DotNetDomain(app)
#elif app.config.autoapi_type == 'python':
# domain = PythonDomain
domain.full()
def build_finished(app, exception):

@ -3,6 +3,8 @@
{{ short_name }} {{ type.title()}}
{{ "=" * (short_name|length + type|length + 1) }}
.. dn:{{ type.lower() }}:: {{ name }}
{% endblock %}
{% block toc %}
@ -32,7 +34,7 @@
- Description
{%- for item in children %}
{% macro render() %}{{ item.summary }}{% endmacro %}
* - :dn:{{ item.type.lower().replace('class', 'cls').replace('interface', 'iface').replace('delegate', 'del') }}:`{{ item.id }}`
* - :dn:{{ item.reftype }}:`{{ item.id }}`
- {{ render()|indent(7) }}
{% endfor %}

@ -1,44 +1,16 @@
from .base import UnknownType
from .dotnet import (
DotNetNamespace, DotNetClass, DotNetMethod, DotNetProperty,
DotNetEnum, DotNetConstructor, DotNetStruct, DotNetInterface,
DotNetDelegate, DotNetField, DotNetEvent
)
from .python import PythonModule, PythonClass, PythonFunction
#from .python import PythonModule, PythonClass, PythonFunction
def classify(obj, obj_type):
if 'type' not in obj:
return ''
if obj_type == 'python':
if obj['type'] == 'module':
return PythonModule(obj)
if obj['type'] == 'class':
return PythonClass(obj)
if obj['type'] == 'function':
return PythonFunction(obj)
if obj_type == 'dotnet':
if obj['type'] == 'Class':
return DotNetClass(obj)
if obj['type'] == 'Namespace':
return DotNetNamespace(obj)
if obj['type'] == 'Property':
return DotNetProperty(obj)
if obj['type'] == 'Method':
return DotNetMethod(obj)
if obj['type'] == 'Enum':
return DotNetEnum(obj)
if obj['type'] == 'Constructor':
return DotNetConstructor(obj)
if obj['type'] == 'Struct':
return DotNetStruct(obj)
if obj['type'] == 'Interface':
return DotNetInterface(obj)
if obj['type'] == 'Delegate':
return DotNetDelegate(obj)
if obj['type'] == 'Field':
return DotNetField(obj)
if obj['type'] == 'Event':
return DotNetEvent(obj)
return UnknownType(obj)
#def classify(obj, obj_type):
# if 'type' not in obj:
# return ''
#
# if obj_type == 'python':
# if obj['type'] == 'module':
# return PythonModule(obj)
# if obj['type'] == 'class':
# return PythonClass(obj)
# if obj['type'] == 'function':
# return PythonFunction(obj)
# return UnknownType(obj)

@ -7,6 +7,8 @@ try:
'epyparse',
'epydoc',
],
test_suite='nose.collector',
tests_require=['nose', 'mock'],
)
except ImportError:
from distutils.core import setup

@ -0,0 +1,93 @@
'''Test .NET autoapi domain'''
import unittest
import time
from contextlib import nested
from mock import patch
from autoapi.domains import dotnet
class DomainTests(unittest.TestCase):
def test_config(self):
'''Sphinx app config'''
class _config(object):
autoapi_dir = '/tmp/autoapi/tmp'
autoapi_root = '/tmp/autoapi/root'
class _application(object):
config = _config()
dom = dotnet.DotNetDomain(_application())
self.assertEqual(dom.get_config('autoapi_dir'), '/tmp/autoapi/tmp')
self.assertEqual(dom.get_config('autoapi_dir'), '/tmp/autoapi/tmp')
def test_create_class(self):
'''Test .NET class instance creation helper'''
dom = dotnet.DotNetDomain({})
cls = dom.create_class({'id': 'Foo.Bar', 'type': 'Namespace'})
self.assertIsInstance(cls, dotnet.DotNetNamespace)
cls = dom.create_class({'id': 'Foo.Bar', 'type': 'Class'})
self.assertIsInstance(cls, dotnet.DotNetClass)
cls = dom.create_class({'id': 'Foo.Bar', 'type': 'Property'})
self.assertIsInstance(cls, dotnet.DotNetProperty)
cls = dom.create_class({'id': 'Foo.Bar', 'type': 'Method'})
self.assertIsInstance(cls, dotnet.DotNetMethod)
cls = dom.create_class({'id': 'Foo.Bar', 'type': 'Enum'})
self.assertIsInstance(cls, dotnet.DotNetEnum)
cls = dom.create_class({'id': 'Foo.Bar', 'type': 'Constructor'})
self.assertIsInstance(cls, dotnet.DotNetConstructor)
cls = dom.create_class({'id': 'Foo.Bar', 'type': 'Struct'})
self.assertIsInstance(cls, dotnet.DotNetStruct)
cls = dom.create_class({'id': 'Foo.Bar', 'type': 'Interface'})
self.assertIsInstance(cls, dotnet.DotNetInterface)
cls = dom.create_class({'id': 'Foo.Bar', 'type': 'Delegate'})
self.assertIsInstance(cls, dotnet.DotNetDelegate)
cls = dom.create_class({'id': 'Foo.Bar', 'type': 'Field'})
self.assertIsInstance(cls, dotnet.DotNetField)
cls = dom.create_class({'id': 'Foo.Bar', 'type': 'Event'})
self.assertIsInstance(cls, dotnet.DotNetEvent)
def test_create_class_with_children(self):
dom = dotnet.DotNetDomain({})
cls = dom.create_class({'id': 'Foo.Bar',
'type': 'Class',
'items': [
{'id': 'Foo.Bar.Baz',
'type': 'Method'}
]})
self.assertIsInstance(cls, dotnet.DotNetClass)
self.assertIsInstance(cls.item_map['Method'][0], dotnet.DotNetMethod)
def test_get_objects(self):
'''Test basic get objects'''
objs = []
def _mock_find(self):
return ['foo', 'bar']
def _mock_read(self, path):
return {'id': 'Foo.Bar', 'type': 'Class', 'summary': path}
def _mock_add(self, obj):
objs.append(obj)
def _mock_config(self, key):
return 'foo'
with nested(
patch('autoapi.domains.dotnet.DotNetDomain.find_files', _mock_find),
patch('autoapi.domains.dotnet.DotNetDomain.read_file', _mock_read),
patch('autoapi.domains.dotnet.DotNetDomain.add_object', _mock_add),
patch('autoapi.domains.dotnet.DotNetDomain.get_config', _mock_config),
):
dom = dotnet.DotNetDomain({})
dom.get_objects()
self.assertEqual(len(objs), 2)
self.assertEqual(objs[0].id, 'Foo.Bar')
self.assertEqual(objs[0].summary, 'foo/foo')
self.assertEqual(objs[1].id, 'Foo.Bar')
self.assertEqual(objs[1].summary, 'foo/bar')

@ -0,0 +1,33 @@
'''Test .NET autoapi objects'''
import unittest
import time
from autoapi.domains import dotnet
class NamespaceTests(unittest.TestCase):
def test_namespace_namespace(self):
'''Namespace parent resolution'''
ns = dotnet.DotNetNamespace(dict(id='Foo.Bar.Widgets',
type='namespace'))
self.assertEqual(ns.namespace(), 'Foo.Bar')
ns = dotnet.DotNetNamespace(dict(id='Foo.Bar',
type='namespace'))
self.assertEqual(ns.namespace(), 'Foo')
ns = dotnet.DotNetNamespace(dict(id='Foo',
type='namespace'))
self.assertIsNone(ns.namespace())
def test_class_namespace(self):
'''Class parent resolution'''
cls = dotnet.DotNetClass(dict(id='Foo.Bar.Widget',
type='class'))
self.assertEqual(cls.namespace(), 'Foo.Bar')
cls = dotnet.DotNetClass(dict(id='Foo.Bar',
type='class'))
self.assertEqual(cls.namespace(), 'Foo')
cls = dotnet.DotNetClass(dict(id='Foo',
type='class'))
self.assertIsNone(cls.namespace())
Loading…
Cancel
Save