diff options
Diffstat (limited to 'bin/SConsDoc.py')
| -rw-r--r-- | bin/SConsDoc.py | 823 |
1 files changed, 823 insertions, 0 deletions
diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py new file mode 100644 index 0000000..b42e994 --- /dev/null +++ b/bin/SConsDoc.py @@ -0,0 +1,823 @@ +#!/usr/bin/env python +# +# Copyright (c) 2010 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# +# Module for handling SCons documentation processing. +# + +__doc__ = """ +This module parses home-brew XML files that document various things +in SCons. Right now, it handles Builders, functions, construction +variables, and Tools, but we expect it to get extended in the future. + +In general, you can use any DocBook tag in the input, and this module +just adds processing various home-brew tags to try to make life a +little easier. + +Builder example: + + <builder name="BUILDER"> + <summary> + <para>This is the summary description of an SCons Builder. + It will get placed in the man page, + and in the appropriate User's Guide appendix. + The name of any builder may be interpolated + anywhere in the document by specifying the + &b-BUILDER; element. It need not be on a line by itself.</para> + + Unlike normal XML, blank lines are significant in these + descriptions and serve to separate paragraphs. + They'll get replaced in DocBook output with appropriate tags + to indicate a new paragraph. + + <example> + print "this is example code, it will be offset and indented" + </example> + </summary> + </builder> + +Function example: + + <scons_function name="FUNCTION"> + <arguments> + (arg1, arg2, key=value) + </arguments> + <summary> + <para>This is the summary description of an SCons function. + It will get placed in the man page, + and in the appropriate User's Guide appendix. + The name of any builder may be interpolated + anywhere in the document by specifying the + &f-FUNCTION; element. It need not be on a line by itself.</para> + + <example> + print "this is example code, it will be offset and indented" + </example> + </summary> + </scons_function> + +Construction variable example: + + <cvar name="VARIABLE"> + <summary> + <para>This is the summary description of a construction variable. + It will get placed in the man page, + and in the appropriate User's Guide appendix. + The name of any construction variable may be interpolated + anywhere in the document by specifying the + &t-VARIABLE; element. It need not be on a line by itself.</para> + + <example> + print "this is example code, it will be offset and indented" + </example> + </summary> + </cvar> + +Tool example: + + <tool name="TOOL"> + <summary> + <para>This is the summary description of an SCons Tool. + It will get placed in the man page, + and in the appropriate User's Guide appendix. + The name of any tool may be interpolated + anywhere in the document by specifying the + &t-TOOL; element. It need not be on a line by itself.</para> + + <example> + print "this is example code, it will be offset and indented" + </example> + </summary> + </tool> +""" + +import imp +import os.path +import re +import sys +import copy + +# Do we have libxml2/libxslt/lxml? +has_libxml2 = True +try: + import libxml2 + import libxslt +except: + has_libxml2 = False + try: + import lxml + except: + print("Failed to import either libxml2/libxslt or lxml") + sys.exit(1) + +has_etree = False +if not has_libxml2: + try: + from lxml import etree + has_etree = True + except ImportError: + pass +if not has_etree: + try: + # Python 2.5 + import xml.etree.cElementTree as etree + except ImportError: + try: + # Python 2.5 + import xml.etree.ElementTree as etree + except ImportError: + try: + # normal cElementTree install + import cElementTree as etree + except ImportError: + try: + # normal ElementTree install + import elementtree.ElementTree as etree + except ImportError: + print("Failed to import ElementTree from any known place") + sys.exit(1) + +re_entity = re.compile("\&([^;]+);") +re_entity_header = re.compile("<!DOCTYPE\s+sconsdoc\s+[^\]]+\]>") + +# Namespace for the SCons Docbook XSD +dbxsd="http://www.scons.org/dbxsd/v1.0" +# Namespace map identifier for the SCons Docbook XSD +dbxid="dbx" +# Namespace for schema instances +xsi = "http://www.w3.org/2001/XMLSchema-instance" + +# Header comment with copyright +copyright_comment = """ +Copyright (c) 2001 - 2014 The SCons Foundation + +This file is processed by the bin/SConsDoc.py module. +See its __doc__ string for a discussion of the format. +""" + +def isSConsXml(fpath): + """ Check whether the given file is a SCons XML file, i.e. it + contains the default target namespace definition. + """ + try: + f = open(fpath,'r') + content = f.read() + f.close() + if content.find('xmlns="%s"' % dbxsd) >= 0: + return True + except: + pass + + return False + +def remove_entities(content): + # Cut out entity inclusions + content = re_entity_header.sub("", content, re.M) + # Cut out entities themselves + content = re_entity.sub(lambda match: match.group(1), content) + + return content + +default_xsd = os.path.join('doc','xsd','scons.xsd') + +ARG = "dbscons" + +class Libxml2ValidityHandler: + + def __init__(self): + self.errors = [] + self.warnings = [] + + def error(self, msg, data): + if data != ARG: + raise Exception, "Error handler did not receive correct argument" + self.errors.append(msg) + + def warning(self, msg, data): + if data != ARG: + raise Exception, "Warning handler did not receive correct argument" + self.warnings.append(msg) + + +class DoctypeEntity: + def __init__(self, name_, uri_): + self.name = name_ + self.uri = uri_ + + def getEntityString(self): + txt = """ <!ENTITY %(perc)s %(name)s SYSTEM "%(uri)s"> + %(perc)s%(name)s; +""" % {'perc' : perc, 'name' : self.name, 'uri' : self.uri} + + return txt + +class DoctypeDeclaration: + def __init__(self, name_=None): + self.name = name_ + self.entries = [] + if self.name is None: + # Add default entries + self.name = "sconsdoc" + self.addEntity("scons", "../scons.mod") + self.addEntity("builders-mod", "builders.mod") + self.addEntity("functions-mod", "functions.mod") + self.addEntity("tools-mod", "tools.mod") + self.addEntity("variables-mod", "variables.mod") + + def addEntity(self, name, uri): + self.entries.append(DoctypeEntity(name, uri)) + + def createDoctype(self): + content = '<!DOCTYPE %s [\n' % self.name + for e in self.entries: + content += e.getEntityString() + content += ']>\n' + + return content + +if not has_libxml2: + class TreeFactory: + def __init__(self): + pass + + def newNode(self, tag): + return etree.Element(tag) + + def newEtreeNode(self, tag, init_ns=False): + if init_ns: + NSMAP = {None: dbxsd, + 'xsi' : xsi} + return etree.Element(tag, nsmap=NSMAP) + + return etree.Element(tag) + + def copyNode(self, node): + return copy.deepcopy(node) + + def appendNode(self, parent, child): + parent.append(child) + + def hasAttribute(self, node, att): + return att in node.attrib + + def getAttribute(self, node, att): + return node.attrib[att] + + def setAttribute(self, node, att, value): + node.attrib[att] = value + + def getText(self, root): + return root.text + + def setText(self, root, txt): + root.text = txt + + def writeGenTree(self, root, fp): + dt = DoctypeDeclaration() + fp.write(etree.tostring(root, xml_declaration=True, + encoding="UTF-8", pretty_print=True, + doctype=dt.createDoctype())) + + def writeTree(self, root, fpath): + fp = open(fpath, 'w') + fp.write(etree.tostring(root, xml_declaration=True, + encoding="UTF-8", pretty_print=True)) + fp.close() + + def prettyPrintFile(self, fpath): + fin = open(fpath,'r') + tree = etree.parse(fin) + pretty_content = etree.tostring(tree, pretty_print=True) + fin.close() + + fout = open(fpath,'w') + fout.write(pretty_content) + fout.close() + + def decorateWithHeader(self, root): + root.attrib["{"+xsi+"}schemaLocation"] = "%s %s/scons.xsd" % (dbxsd, dbxsd) + return root + + def newXmlTree(self, root): + """ Return a XML file tree with the correct namespaces set, + the element root as top entry and the given header comment. + """ + NSMAP = {None: dbxsd, + 'xsi' : xsi} + t = etree.Element(root, nsmap=NSMAP) + return self.decorateWithHeader(t) + + def validateXml(self, fpath, xmlschema_context): + # Use lxml + xmlschema = etree.XMLSchema(xmlschema_context) + try: + doc = etree.parse(fpath) + except Exception, e: + print "ERROR: %s fails to parse:"%fpath + print e + return False + doc.xinclude() + try: + xmlschema.assertValid(doc) + except Exception, e: + print "ERROR: %s fails to validate:" % fpath + print e + return False + return True + + def findAll(self, root, tag, ns=None, xp_ctxt=None, nsmap=None): + expression = ".//{%s}%s" % (nsmap[ns], tag) + if not ns or not nsmap: + expression = ".//%s" % tag + return root.findall(expression) + + def findAllChildrenOf(self, root, tag, ns=None, xp_ctxt=None, nsmap=None): + expression = "./{%s}%s/*" % (nsmap[ns], tag) + if not ns or not nsmap: + expression = "./%s/*" % tag + return root.findall(expression) + + def convertElementTree(self, root): + """ Convert the given tree of etree.Element + entries to a list of tree nodes for the + current XML toolkit. + """ + return [root] + +else: + class TreeFactory: + def __init__(self): + pass + + def newNode(self, tag): + return libxml2.newNode(tag) + + def newEtreeNode(self, tag, init_ns=False): + return etree.Element(tag) + + def copyNode(self, node): + return node.copyNode(1) + + def appendNode(self, parent, child): + if hasattr(parent, 'addChild'): + parent.addChild(child) + else: + parent.append(child) + + def hasAttribute(self, node, att): + if hasattr(node, 'hasProp'): + return node.hasProp(att) + return att in node.attrib + + def getAttribute(self, node, att): + if hasattr(node, 'prop'): + return node.prop(att) + return node.attrib[att] + + def setAttribute(self, node, att, value): + if hasattr(node, 'setProp'): + node.setProp(att, value) + else: + node.attrib[att] = value + + def getText(self, root): + if hasattr(root, 'getContent'): + return root.getContent() + return root.text + + def setText(self, root, txt): + if hasattr(root, 'setContent'): + root.setContent(txt) + else: + root.text = txt + + def writeGenTree(self, root, fp): + doc = libxml2.newDoc('1.0') + dtd = doc.newDtd("sconsdoc", None, None) + doc.addChild(dtd) + doc.setRootElement(root) + content = doc.serialize("UTF-8", 1) + dt = DoctypeDeclaration() + # This is clearly a hack, but unfortunately libxml2 + # doesn't support writing PERs (Parsed Entity References). + # So, we simply replace the empty doctype with the + # text we need... + content = content.replace("<!DOCTYPE sconsdoc>", dt.createDoctype()) + fp.write(content) + doc.freeDoc() + + def writeTree(self, root, fpath): + fp = open(fpath, 'w') + doc = libxml2.newDoc('1.0') + doc.setRootElement(root) + fp.write(doc.serialize("UTF-8", 1)) + doc.freeDoc() + fp.close() + + def prettyPrintFile(self, fpath): + # Read file and resolve entities + doc = libxml2.readFile(fpath, None, libxml2d.XML_PARSE_NOENT) + fp = open(fpath, 'w') + # Prettyprint + fp.write(doc.serialize("UTF-8", 1)) + fp.close() + # Cleanup + doc.freeDoc() + + def decorateWithHeader(self, root): + # Register the namespaces + ns = root.newNs(dbxsd, None) + xi = root.newNs(xsi, 'xsi') + root.setNs(ns) #put this node in the target namespace + + root.setNsProp(xi, 'schemaLocation', "%s %s/scons.xsd" % (dbxsd, dbxsd)) + + return root + + def newXmlTree(self, root): + """ Return a XML file tree with the correct namespaces set, + the element root as top entry and the given header comment. + """ + t = libxml2.newNode(root) + return self.decorateWithHeader(t) + + def validateXml(self, fpath, xmlschema_context): + # Create validation context + validation_context = xmlschema_context.schemaNewValidCtxt() + # Set error/warning handlers + eh = Libxml2ValidityHandler() + validation_context.setValidityErrorHandler(eh.error, eh.warning, ARG) + # Read file and resolve entities + doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT) + doc.xincludeProcessFlags(libxml2.XML_PARSE_NOENT) + err = validation_context.schemaValidateDoc(doc) + # Cleanup + doc.freeDoc() + del validation_context + + if err or eh.errors: + for e in eh.errors: + print e.rstrip("\n") + print "%s fails to validate" % fpath + return False + + return True + + def findAll(self, root, tag, ns=None, xpath_context=None, nsmap=None): + if hasattr(root, 'xpathEval') and xpath_context: + # Use the xpath context + xpath_context.setContextNode(root) + expression = ".//%s" % tag + if ns: + expression = ".//%s:%s" % (ns, tag) + return xpath_context.xpathEval(expression) + else: + expression = ".//{%s}%s" % (nsmap[ns], tag) + if not ns or not nsmap: + expression = ".//%s" % tag + return root.findall(expression) + + def findAllChildrenOf(self, root, tag, ns=None, xpath_context=None, nsmap=None): + if hasattr(root, 'xpathEval') and xpath_context: + # Use the xpath context + xpath_context.setContextNode(root) + expression = "./%s/node()" % tag + if ns: + expression = "./%s:%s/node()" % (ns, tag) + + return xpath_context.xpathEval(expression) + else: + expression = "./{%s}%s/node()" % (nsmap[ns], tag) + if not ns or not nsmap: + expression = "./%s/node()" % tag + return root.findall(expression) + + def expandChildElements(self, child): + """ Helper function for convertElementTree, + converts a single child recursively. + """ + nchild = self.newNode(child.tag) + # Copy attributes + for key, val in child.attrib: + self.setAttribute(nchild, key, val) + elements = [] + # Add text + if child.text: + t = libxml2.newText(child.text) + self.appendNode(nchild, t) + # Add children + for c in child: + for n in self.expandChildElements(c): + self.appendNode(nchild, n) + elements.append(nchild) + # Add tail + if child.tail: + tail = libxml2.newText(child.tail) + elements.append(tail) + + return elements + + def convertElementTree(self, root): + """ Convert the given tree of etree.Element + entries to a list of tree nodes for the + current XML toolkit. + """ + nroot = self.newNode(root.tag) + # Copy attributes + for key, val in root.attrib: + self.setAttribute(nroot, key, val) + elements = [] + # Add text + if root.text: + t = libxml2.newText(root.text) + self.appendNode(nroot, t) + # Add children + for c in root: + for n in self.expandChildElements(c): + self.appendNode(nroot, n) + elements.append(nroot) + # Add tail + if root.tail: + tail = libxml2.newText(root.tail) + elements.append(tail) + + return elements + +tf = TreeFactory() + + +class SConsDocTree: + def __init__(self): + self.nsmap = {'dbx' : dbxsd} + self.doc = None + self.root = None + self.xpath_context = None + + def parseContent(self, content, include_entities=True): + """ Parses the given content as XML file. This method + is used when we generate the basic lists of entities + for the builders, tools and functions. + So we usually don't bother about namespaces and resolving + entities here...this is handled in parseXmlFile below + (step 2 of the overall process). + """ + if not include_entities: + content = remove_entities(content) + # Create domtree from given content string + self.root = etree.fromstring(content) + + def parseXmlFile(self, fpath): + nsmap = {'dbx' : dbxsd} + if not has_libxml2: + # Create domtree from file + domtree = etree.parse(fpath) + self.root = domtree.getroot() + else: + # Read file and resolve entities + self.doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT) + self.root = self.doc.getRootElement() + # Create xpath context + self.xpath_context = self.doc.xpathNewContext() + # Register namespaces + for key, val in self.nsmap.iteritems(): + self.xpath_context.xpathRegisterNs(key, val) + + def __del__(self): + if self.doc is not None: + self.doc.freeDoc() + if self.xpath_context is not None: + self.xpath_context.xpathFreeContext() + +perc="%" + +def validate_all_xml(dpaths, xsdfile=default_xsd): + xmlschema_context = None + if not has_libxml2: + # Use lxml + xmlschema_context = etree.parse(xsdfile) + else: + # Use libxml2 and prepare the schema validation context + ctxt = libxml2.schemaNewParserCtxt(xsdfile) + xmlschema_context = ctxt.schemaParse() + del ctxt + + fpaths = [] + for dp in dpaths: + if dp.endswith('.xml') and isSConsXml(dp): + path='.' + fpaths.append(dp) + else: + for path, dirs, files in os.walk(dp): + for f in files: + if f.endswith('.xml'): + fp = os.path.join(path, f) + if isSConsXml(fp): + fpaths.append(fp) + + fails = [] + for idx, fp in enumerate(fpaths): + fpath = os.path.join(path, fp) + print "%.2f%s (%d/%d) %s" % (float(idx+1)*100.0/float(len(fpaths)), + perc, idx+1, len(fpaths),fp) + + if not tf.validateXml(fp, xmlschema_context): + fails.append(fp) + continue + + if has_libxml2: + # Cleanup + del xmlschema_context + + if fails: + return False + + return True + +class Item(object): + def __init__(self, name): + self.name = name + self.sort_name = name.lower() + if self.sort_name[0] == '_': + self.sort_name = self.sort_name[1:] + self.sets = [] + self.uses = [] + self.summary = None + self.arguments = None + def cmp_name(self, name): + if name[0] == '_': + name = name[1:] + return name.lower() + def __cmp__(self, other): + return cmp(self.sort_name, other.sort_name) + +class Builder(Item): + pass + +class Function(Item): + def __init__(self, name): + super(Function, self).__init__(name) + +class Tool(Item): + def __init__(self, name): + Item.__init__(self, name) + self.entity = self.name.replace('+', 'X') + +class ConstructionVariable(Item): + pass + +class Arguments(object): + def __init__(self, signature, body=None): + if not body: + body = [] + self.body = body + self.signature = signature + def __str__(self): + s = ''.join(self.body).strip() + result = [] + for m in re.findall('([a-zA-Z/_]+|[^a-zA-Z/_]+)', s): + if ' ' in m: + m = '"%s"' % m + result.append(m) + return ' '.join(result) + def append(self, data): + self.body.append(data) + +class SConsDocHandler(object): + def __init__(self): + self.builders = {} + self.functions = {} + self.tools = {} + self.cvars = {} + + def parseItems(self, domelem, xpath_context, nsmap): + items = [] + + for i in tf.findAll(domelem, "item", dbxid, xpath_context, nsmap): + txt = tf.getText(i) + if txt is not None: + txt = txt.strip() + if len(txt): + items.append(txt.strip()) + + return items + + def parseUsesSets(self, domelem, xpath_context, nsmap): + uses = [] + sets = [] + + for u in tf.findAll(domelem, "uses", dbxid, xpath_context, nsmap): + uses.extend(self.parseItems(u, xpath_context, nsmap)) + for s in tf.findAll(domelem, "sets", dbxid, xpath_context, nsmap): + sets.extend(self.parseItems(s, xpath_context, nsmap)) + + return sorted(uses), sorted(sets) + + def parseInstance(self, domelem, map, Class, + xpath_context, nsmap, include_entities=True): + name = 'unknown' + if tf.hasAttribute(domelem, 'name'): + name = tf.getAttribute(domelem, 'name') + try: + instance = map[name] + except KeyError: + instance = Class(name) + map[name] = instance + uses, sets = self.parseUsesSets(domelem, xpath_context, nsmap) + instance.uses.extend(uses) + instance.sets.extend(sets) + if include_entities: + # Parse summary and function arguments + for s in tf.findAllChildrenOf(domelem, "summary", dbxid, xpath_context, nsmap): + if instance.summary is None: + instance.summary = [] + instance.summary.append(tf.copyNode(s)) + for a in tf.findAll(domelem, "arguments", dbxid, xpath_context, nsmap): + if instance.arguments is None: + instance.arguments = [] + instance.arguments.append(tf.copyNode(a)) + + def parseDomtree(self, root, xpath_context=None, nsmap=None, include_entities=True): + # Process Builders + for b in tf.findAll(root, "builder", dbxid, xpath_context, nsmap): + self.parseInstance(b, self.builders, Builder, + xpath_context, nsmap, include_entities) + # Process Functions + for f in tf.findAll(root, "scons_function", dbxid, xpath_context, nsmap): + self.parseInstance(f, self.functions, Function, + xpath_context, nsmap, include_entities) + # Process Tools + for t in tf.findAll(root, "tool", dbxid, xpath_context, nsmap): + self.parseInstance(t, self.tools, Tool, + xpath_context, nsmap, include_entities) + # Process CVars + for c in tf.findAll(root, "cvar", dbxid, xpath_context, nsmap): + self.parseInstance(c, self.cvars, ConstructionVariable, + xpath_context, nsmap, include_entities) + + def parseContent(self, content, include_entities=True): + """ Parses the given content as XML file. This method + is used when we generate the basic lists of entities + for the builders, tools and functions. + So we usually don't bother about namespaces and resolving + entities here...this is handled in parseXmlFile below + (step 2 of the overall process). + """ + # Create doctree + t = SConsDocTree() + t.parseContent(content, include_entities) + # Parse it + self.parseDomtree(t.root, t.xpath_context, t.nsmap, include_entities) + + def parseXmlFile(self, fpath): + # Create doctree + t = SConsDocTree() + t.parseXmlFile(fpath) + # Parse it + self.parseDomtree(t.root, t.xpath_context, t.nsmap) + +# lifted from Ka-Ping Yee's way cool pydoc module. +def importfile(path): + """Import a Python source file or compiled file given its path.""" + magic = imp.get_magic() + file = open(path, 'r') + if file.read(len(magic)) == magic: + kind = imp.PY_COMPILED + else: + kind = imp.PY_SOURCE + file.close() + filename = os.path.basename(path) + name, ext = os.path.splitext(filename) + file = open(path, 'r') + try: + module = imp.load_module(name, file, path, (ext, 'r', kind)) + except ImportError, e: + sys.stderr.write("Could not import %s: %s\n" % (path, e)) + return None + file.close() + return module + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: |