micropython/tools/gendoc.py

529 lines
17 KiB
Python
Raw Normal View History

"""
Generate documentation for pyboard API from C files.
"""
import os
import argparse
import re
import markdown
# given a list of (name,regex) pairs, find the first one that matches the given line
def re_match_first(regexs, line):
for name, regex in regexs:
match = re.match(regex, line)
if match:
return name, match
return None, None
def makedirs(d):
if not os.path.isdir(d):
os.makedirs(d)
class Lexer:
class LexerError(Exception):
pass
class EOF(Exception):
pass
class Break(Exception):
pass
def __init__(self, file):
self.filename = file
with open(file, 'rt') as f:
line_num = 0
lines = []
for line in f:
line_num += 1
line = line.strip()
if line == '///':
lines.append((line_num, ''))
elif line.startswith('/// '):
lines.append((line_num, line[4:]))
elif len(lines) > 0 and lines[-1][1] is not None:
lines.append((line_num, None))
if len(lines) > 0 and lines[-1][1] is not None:
lines.append((line_num, None))
self.cur_line = 0
self.lines = lines
def opt_break(self):
if len(self.lines) > 0 and self.lines[0][1] is None:
self.lines.pop(0)
def next(self):
if len(self.lines) == 0:
raise Lexer.EOF
else:
l = self.lines.pop(0)
self.cur_line = l[0]
if l[1] is None:
raise Lexer.Break
else:
return l[1]
def error(self, msg):
print('({}:{}) {}'.format(self.filename, self.cur_line, msg))
raise Lexer.LexerError
class MarkdownWriter:
def __init__(self):
pass
def start(self):
self.lines = []
def end(self):
return '\n'.join(self.lines)
def heading(self, level, text):
if len(self.lines) > 0:
self.lines.append('')
self.lines.append(level * '#' + ' ' + text)
self.lines.append('')
def para(self, text):
if len(self.lines) > 0 and self.lines[-1] != '':
self.lines.append('')
if isinstance(text, list):
self.lines.extend(text)
elif isinstance(text, str):
self.lines.append(text)
else:
assert False
self.lines.append('')
def single_line(self, text):
self.lines.append(text)
def module(self, name, short_descr, descr):
self.heading(1, 'module {}'.format(name))
self.para(descr)
def function(self, ctx, name, args, descr):
proto = '{}.{}{}'.format(ctx, self.name, self.args)
self.heading(3, '`' + proto + '`')
self.para(descr)
def method(self, ctx, name, args, descr):
if name == '\\constructor':
proto = '{}{}'.format(ctx, args)
elif name == '\\call':
proto = '{}{}'.format(ctx, args)
else:
proto = '{}.{}{}'.format(ctx, name, args)
self.heading(3, '`' + proto + '`')
self.para(descr)
def constant(self, ctx, name, descr):
self.single_line('`{}.{}` - {}'.format(ctx, name, descr))
class ReStructuredTextWriter:
head_chars = {1:'=', 2:'-', 3:'.'}
def __init__(self):
pass
def start(self):
self.lines = []
def end(self):
return '\n'.join(self.lines)
def _convert(self, text):
return text.replace('`', '``').replace('*', '\\*')
def heading(self, level, text, convert=True):
if len(self.lines) > 0:
self.lines.append('')
if convert:
text = self._convert(text)
self.lines.append(text)
self.lines.append(len(text) * self.head_chars[level])
self.lines.append('')
def para(self, text, indent=''):
if len(self.lines) > 0 and self.lines[-1] != '':
self.lines.append('')
if isinstance(text, list):
for t in text:
self.lines.append(indent + self._convert(t))
elif isinstance(text, str):
self.lines.append(indent + self._convert(text))
else:
assert False
self.lines.append('')
def single_line(self, text):
self.lines.append(self._convert(text))
def module(self, name, short_descr, descr):
self.heading(1, ':mod:`{}` --- {}'.format(name, self._convert(short_descr)), convert=False)
self.lines.append('.. module:: {}'.format(name))
self.lines.append(' :synopsis: {}'.format(short_descr))
self.para(descr)
def function(self, ctx, name, args, descr):
args = self._convert(args)
self.lines.append('.. function:: ' + name + args)
self.para(descr, indent=' ')
def method(self, ctx, name, args, descr):
args = self._convert(args)
if name == '\\constructor':
self.lines.append('.. class:: ' + ctx + args)
elif name == '\\call':
self.lines.append('.. method:: ' + ctx + args)
else:
self.lines.append('.. method:: ' + ctx + '.' + name + args)
self.para(descr, indent=' ')
def constant(self, ctx, name, descr):
self.lines.append('.. data:: ' + name)
self.para(descr, indent=' ')
class DocValidateError(Exception):
pass
class DocItem:
def __init__(self):
self.doc = []
def add_doc(self, lex):
try:
while True:
line = lex.next()
if len(line) > 0 or len(self.doc) > 0:
self.doc.append(line)
except Lexer.Break:
pass
def dump(self, writer):
writer.para(self.doc)
class DocConstant(DocItem):
def __init__(self, name, descr):
super().__init__()
self.name = name
self.descr = descr
def dump(self, ctx, writer):
writer.constant(ctx, self.name, self.descr)
class DocFunction(DocItem):
def __init__(self, name, args):
super().__init__()
self.name = name
self.args = args
def dump(self, ctx, writer):
writer.function(ctx, self.name, self.args, self.doc)
class DocMethod(DocItem):
def __init__(self, name, args):
super().__init__()
self.name = name
self.args = args
def dump(self, ctx, writer):
writer.method(ctx, self.name, self.args, self.doc)
class DocClass(DocItem):
def __init__(self, name, descr):
super().__init__()
self.name = name
self.descr = descr
self.constructors = {}
self.classmethods = {}
self.methods = {}
self.constants = {}
def process_classmethod(self, lex, d):
name = d['id']
if name == '\\constructor':
dict_ = self.constructors
else:
dict_ = self.classmethods
if name in dict_:
lex.error("multiple definition of method '{}'".format(name))
method = dict_[name] = DocMethod(name, d['args'])
method.add_doc(lex)
def process_method(self, lex, d):
name = d['id']
dict_ = self.methods
if name in dict_:
lex.error("multiple definition of method '{}'".format(name))
method = dict_[name] = DocMethod(name, d['args'])
method.add_doc(lex)
def process_constant(self, lex, d):
name = d['id']
if name in self.constants:
lex.error("multiple definition of constant '{}'".format(name))
self.constants[name] = DocConstant(name, d['descr'])
lex.opt_break()
def dump(self, writer):
writer.heading(1, 'class {}'.format(self.name))
super().dump(writer)
if len(self.constructors) > 0:
writer.heading(2, 'Constructors')
for f in sorted(self.constructors.values(), key=lambda x:x.name):
f.dump(self.name, writer)
if len(self.classmethods) > 0:
writer.heading(2, 'Class methods')
for f in sorted(self.classmethods.values(), key=lambda x:x.name):
f.dump(self.name, writer)
if len(self.methods) > 0:
writer.heading(2, 'Methods')
for f in sorted(self.methods.values(), key=lambda x:x.name):
f.dump(self.name.lower(), writer)
if len(self.constants) > 0:
writer.heading(2, 'Constants')
for c in sorted(self.constants.values(), key=lambda x:x.name):
c.dump(self.name, writer)
class DocModule(DocItem):
def __init__(self, name, descr):
super().__init__()
self.name = name
self.descr = descr
self.functions = {}
self.constants = {}
self.classes = {}
self.cur_class = None
def new_file(self):
self.cur_class = None
def process_function(self, lex, d):
name = d['id']
if name in self.functions:
lex.error("multiple definition of function '{}'".format(name))
function = self.functions[name] = DocFunction(name, d['args'])
function.add_doc(lex)
#def process_classref(self, lex, d):
# name = d['id']
# self.classes[name] = name
# lex.opt_break()
def process_class(self, lex, d):
name = d['id']
if name in self.classes:
lex.error("multiple definition of class '{}'".format(name))
self.cur_class = self.classes[name] = DocClass(name, d['descr'])
self.cur_class.add_doc(lex)
def process_classmethod(self, lex, d):
self.cur_class.process_classmethod(lex, d)
def process_method(self, lex, d):
self.cur_class.process_method(lex, d)
def process_constant(self, lex, d):
if self.cur_class is None:
# a module-level constant
name = d['id']
if name in self.constants:
lex.error("multiple definition of constant '{}'".format(name))
self.constants[name] = DocConstant(name, d['descr'])
lex.opt_break()
else:
# a class-level constant
self.cur_class.process_constant(lex, d)
def validate(self):
if self.descr is None:
raise DocValidateError('module {} referenced but never defined'.format(self.name))
def dump(self, writer):
writer.module(self.name, self.descr, self.doc)
if self.functions:
writer.heading(2, 'Functions')
for f in sorted(self.functions.values(), key=lambda x:x.name):
f.dump(self.name, writer)
if self.constants:
writer.heading(2, 'Constants')
for c in sorted(self.constants.values(), key=lambda x:x.name):
c.dump(self.name, writer)
if self.classes:
writer.heading(2, 'Classes')
for c in sorted(self.classes.values(), key=lambda x:x.name):
writer.para('[`{}.{}`]({}) - {}'.format(self.name, c.name, c.name, c.descr))
def write_html(self, dir):
md_writer = MarkdownWriter()
md_writer.start()
self.dump(md_writer)
with open(os.path.join(dir, 'index.html'), 'wt') as f:
f.write(markdown.markdown(md_writer.end()))
for c in self.classes.values():
class_dir = os.path.join(dir, c.name)
makedirs(class_dir)
md_writer.start()
md_writer.para('part of the [{} module](./)'.format(self.name))
c.dump(md_writer)
with open(os.path.join(class_dir, 'index.html'), 'wt') as f:
f.write(markdown.markdown(md_writer.end()))
def write_rst(self, dir):
rst_writer = ReStructuredTextWriter()
rst_writer.start()
self.dump(rst_writer)
with open(dir + '/' + self.name + '.rst', 'wt') as f:
f.write(rst_writer.end())
for c in self.classes.values():
rst_writer.start()
c.dump(rst_writer)
with open(dir + '/' + self.name + '.' + c.name + '.rst', 'wt') as f:
f.write(rst_writer.end())
class Doc:
def __init__(self):
self.modules = {}
self.cur_module = None
def new_file(self):
self.cur_module = None
for m in self.modules.values():
m.new_file()
def check_module(self, lex):
if self.cur_module is None:
lex.error('module not defined')
def process_module(self, lex, d):
name = d['id']
if name not in self.modules:
self.modules[name] = DocModule(name, None)
self.cur_module = self.modules[name]
if self.cur_module.descr is not None:
lex.error("multiple definition of module '{}'".format(name))
self.cur_module.descr = d['descr']
self.cur_module.add_doc(lex)
def process_moduleref(self, lex, d):
name = d['id']
if name not in self.modules:
self.modules[name] = DocModule(name, None)
self.cur_module = self.modules[name]
lex.opt_break()
def process_class(self, lex, d):
self.check_module(lex)
self.cur_module.process_class(lex, d)
def process_function(self, lex, d):
self.check_module(lex)
self.cur_module.process_function(lex, d)
def process_classmethod(self, lex, d):
self.check_module(lex)
self.cur_module.process_classmethod(lex, d)
def process_method(self, lex, d):
self.check_module(lex)
self.cur_module.process_method(lex, d)
def process_constant(self, lex, d):
self.check_module(lex)
self.cur_module.process_constant(lex, d)
def validate(self):
for m in self.modules.values():
m.validate()
def dump(self, writer):
writer.heading(1, 'Modules')
writer.para('These are the Python modules that are implemented.')
for m in sorted(self.modules.values(), key=lambda x:x.name):
writer.para('[`{}`]({}/) - {}'.format(m.name, m.name, m.descr))
def write_html(self, dir):
md_writer = MarkdownWriter()
with open(os.path.join(dir, 'module', 'index.html'), 'wt') as f:
md_writer.start()
self.dump(md_writer)
f.write(markdown.markdown(md_writer.end()))
for m in self.modules.values():
mod_dir = os.path.join(dir, 'module', m.name)
makedirs(mod_dir)
m.write_html(mod_dir)
def write_rst(self, dir):
#with open(os.path.join(dir, 'module', 'index.html'), 'wt') as f:
# f.write(markdown.markdown(self.dump()))
for m in self.modules.values():
m.write_rst(dir)
regex_descr = r'(?P<descr>.*)'
doc_regexs = (
(Doc.process_module, re.compile(r'\\module (?P<id>[a-z][a-z0-9]*) - ' + regex_descr + r'$')),
(Doc.process_moduleref, re.compile(r'\\moduleref (?P<id>[a-z]+)$')),
(Doc.process_function, re.compile(r'\\function (?P<id>[a-z0-9_]+)(?P<args>\(.*\))$')),
(Doc.process_classmethod, re.compile(r'\\classmethod (?P<id>\\?[a-z0-9_]+)(?P<args>\(.*\))$')),
(Doc.process_method, re.compile(r'\\method (?P<id>\\?[a-z0-9_]+)(?P<args>\(.*\))$')),
(Doc.process_constant, re.compile(r'\\constant (?P<id>[A-Za-z0-9_]+) - ' + regex_descr + r'$')),
#(Doc.process_classref, re.compile(r'\\classref (?P<id>[A-Za-z0-9_]+)$')),
(Doc.process_class, re.compile(r'\\class (?P<id>[A-Za-z0-9_]+) - ' + regex_descr + r'$')),
)
def process_file(file, doc):
lex = Lexer(file)
doc.new_file()
try:
try:
while True:
line = lex.next()
fun, match = re_match_first(doc_regexs, line)
if fun == None:
lex.error('unknown line format: {}'.format(line))
fun(doc, lex, match.groupdict())
except Lexer.Break:
lex.error('unexpected break')
except Lexer.EOF:
pass
except Lexer.LexerError:
return False
return True
def main():
cmd_parser = argparse.ArgumentParser(description='Generate documentation for pyboard API from C files.')
cmd_parser.add_argument('--outdir', metavar='<output dir>', default='gendoc-out', help='ouput directory')
cmd_parser.add_argument('--format', default='html', help='output format: html or rst')
cmd_parser.add_argument('files', nargs='+', help='input files')
args = cmd_parser.parse_args()
doc = Doc()
for file in args.files:
print('processing', file)
if not process_file(file, doc):
return
try:
doc.validate()
except DocValidateError as e:
print(e)
makedirs(args.outdir)
if args.format == 'html':
doc.write_html(args.outdir)
elif args.format == 'rst':
doc.write_rst(args.outdir)
else:
print('unknown format:', args.format)
return
print('written to', args.outdir)
if __name__ == "__main__":
main()