diff options
Diffstat (limited to 'bin')
40 files changed, 7166 insertions, 0 deletions
diff --git a/bin/Command.py b/bin/Command.py new file mode 100644 index 0000000..8702f51 --- /dev/null +++ b/bin/Command.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# +# XXX Python script template +# +# XXX Describe what the script does here. +# + +import getopt +import os +import shlex +import sys + +class Usage(Exception): + def __init__(self, msg): + self.msg = msg + +class CommandRunner(object): + """ + Representation of a command to be executed. + """ + + def __init__(self, dictionary={}): + self.subst_dictionary(dictionary) + + def subst_dictionary(self, dictionary): + self._subst_dictionary = dictionary + + def subst(self, string, dictionary=None): + """ + Substitutes (via the format operator) the values in the specified + dictionary into the specified command. + + The command can be an (action, string) tuple. In all cases, we + perform substitution on strings and don't worry if something isn't + a string. (It's probably a Python function to be executed.) + """ + if dictionary is None: + dictionary = self._subst_dictionary + if dictionary: + try: + string = string % dictionary + except TypeError: + pass + return string + + def do_display(self, string): + if isinstance(string, tuple): + func = string[0] + args = string[1:] + s = '%s(%s)' % (func.__name__, ', '.join(map(repr, args))) + else: + s = self.subst(string) + if not s.endswith('\n'): + s += '\n' + sys.stdout.write(s) + sys.stdout.flush() + + def do_not_display(self, string): + pass + + def do_execute(self, command): + if isinstance(command, str): + command = self.subst(command) + cmdargs = shlex.split(command) + if cmdargs[0] == 'cd': + command = (os.chdir,) + tuple(cmdargs[1:]) + elif cmdargs[0] == 'mkdir': + command = (os.mkdir,) + tuple(cmdargs[1:]) + if isinstance(command, tuple): + func = command[0] + args = command[1:] + return func(*args) + else: + return os.system(command) + + def do_not_execute(self, command): + pass + + display = do_display + execute = do_execute + + def run(self, command, display=None): + """ + Runs this command, displaying it first. + + The actual display() and execute() methods we call may be + overridden if we're printing but not executing, or vice versa. + """ + if display is None: + display = command + self.display(display) + return self.execute(command) + +def main(argv=None): + if argv is None: + argv = sys.argv + + short_options = 'hnq' + long_options = ['help', 'no-exec', 'quiet'] + + helpstr = """\ +Usage: script-template.py [-hnq] + + -h, --help Print this help and exit + -n, --no-exec No execute, just print command lines + -q, --quiet Quiet, don't print command lines +""" + + try: + try: + opts, args = getopt.getopt(argv[1:], short_options, long_options) + except getopt.error, msg: + raise Usage(msg) + + for o, a in opts: + if o in ('-h', '--help'): + print helpstr + sys.exit(0) + elif o in ('-n', '--no-exec'): + Command.execute = Command.do_not_execute + elif o in ('-q', '--quiet'): + Command.display = Command.do_not_display + except Usage, err: + sys.stderr.write(err.msg) + sys.stderr.write('use -h to get help') + return 2 + + commands = [ + ] + + for command in [ Command(c) for c in commands ]: + status = command.run(command) + if status: + sys.exit(status) + +if __name__ == "__main__": + sys.exit(main()) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py new file mode 100644 index 0000000..b42e994 --- /dev/null +++ b/bin/SConsDoc.py @@ -0,0 +1,823 @@ +#!/usr/bin/env python +# +# Copyright (c) 2010 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# +# Module for handling SCons documentation processing. +# + +__doc__ = """ +This module parses home-brew XML files that document various things +in SCons. Right now, it handles Builders, functions, construction +variables, and Tools, but we expect it to get extended in the future. + +In general, you can use any DocBook tag in the input, and this module +just adds processing various home-brew tags to try to make life a +little easier. + +Builder example: + + <builder name="BUILDER"> + <summary> + <para>This is the summary description of an SCons Builder. + It will get placed in the man page, + and in the appropriate User's Guide appendix. + The name of any builder may be interpolated + anywhere in the document by specifying the + &b-BUILDER; element. It need not be on a line by itself.</para> + + Unlike normal XML, blank lines are significant in these + descriptions and serve to separate paragraphs. + They'll get replaced in DocBook output with appropriate tags + to indicate a new paragraph. + + <example> + print "this is example code, it will be offset and indented" + </example> + </summary> + </builder> + +Function example: + + <scons_function name="FUNCTION"> + <arguments> + (arg1, arg2, key=value) + </arguments> + <summary> + <para>This is the summary description of an SCons function. + It will get placed in the man page, + and in the appropriate User's Guide appendix. + The name of any builder may be interpolated + anywhere in the document by specifying the + &f-FUNCTION; element. It need not be on a line by itself.</para> + + <example> + print "this is example code, it will be offset and indented" + </example> + </summary> + </scons_function> + +Construction variable example: + + <cvar name="VARIABLE"> + <summary> + <para>This is the summary description of a construction variable. + It will get placed in the man page, + and in the appropriate User's Guide appendix. + The name of any construction variable may be interpolated + anywhere in the document by specifying the + &t-VARIABLE; element. It need not be on a line by itself.</para> + + <example> + print "this is example code, it will be offset and indented" + </example> + </summary> + </cvar> + +Tool example: + + <tool name="TOOL"> + <summary> + <para>This is the summary description of an SCons Tool. + It will get placed in the man page, + and in the appropriate User's Guide appendix. + The name of any tool may be interpolated + anywhere in the document by specifying the + &t-TOOL; element. It need not be on a line by itself.</para> + + <example> + print "this is example code, it will be offset and indented" + </example> + </summary> + </tool> +""" + +import imp +import os.path +import re +import sys +import copy + +# Do we have libxml2/libxslt/lxml? +has_libxml2 = True +try: + import libxml2 + import libxslt +except: + has_libxml2 = False + try: + import lxml + except: + print("Failed to import either libxml2/libxslt or lxml") + sys.exit(1) + +has_etree = False +if not has_libxml2: + try: + from lxml import etree + has_etree = True + except ImportError: + pass +if not has_etree: + try: + # Python 2.5 + import xml.etree.cElementTree as etree + except ImportError: + try: + # Python 2.5 + import xml.etree.ElementTree as etree + except ImportError: + try: + # normal cElementTree install + import cElementTree as etree + except ImportError: + try: + # normal ElementTree install + import elementtree.ElementTree as etree + except ImportError: + print("Failed to import ElementTree from any known place") + sys.exit(1) + +re_entity = re.compile("\&([^;]+);") +re_entity_header = re.compile("<!DOCTYPE\s+sconsdoc\s+[^\]]+\]>") + +# Namespace for the SCons Docbook XSD +dbxsd="http://www.scons.org/dbxsd/v1.0" +# Namespace map identifier for the SCons Docbook XSD +dbxid="dbx" +# Namespace for schema instances +xsi = "http://www.w3.org/2001/XMLSchema-instance" + +# Header comment with copyright +copyright_comment = """ +Copyright (c) 2001 - 2014 The SCons Foundation + +This file is processed by the bin/SConsDoc.py module. +See its __doc__ string for a discussion of the format. +""" + +def isSConsXml(fpath): + """ Check whether the given file is a SCons XML file, i.e. it + contains the default target namespace definition. + """ + try: + f = open(fpath,'r') + content = f.read() + f.close() + if content.find('xmlns="%s"' % dbxsd) >= 0: + return True + except: + pass + + return False + +def remove_entities(content): + # Cut out entity inclusions + content = re_entity_header.sub("", content, re.M) + # Cut out entities themselves + content = re_entity.sub(lambda match: match.group(1), content) + + return content + +default_xsd = os.path.join('doc','xsd','scons.xsd') + +ARG = "dbscons" + +class Libxml2ValidityHandler: + + def __init__(self): + self.errors = [] + self.warnings = [] + + def error(self, msg, data): + if data != ARG: + raise Exception, "Error handler did not receive correct argument" + self.errors.append(msg) + + def warning(self, msg, data): + if data != ARG: + raise Exception, "Warning handler did not receive correct argument" + self.warnings.append(msg) + + +class DoctypeEntity: + def __init__(self, name_, uri_): + self.name = name_ + self.uri = uri_ + + def getEntityString(self): + txt = """ <!ENTITY %(perc)s %(name)s SYSTEM "%(uri)s"> + %(perc)s%(name)s; +""" % {'perc' : perc, 'name' : self.name, 'uri' : self.uri} + + return txt + +class DoctypeDeclaration: + def __init__(self, name_=None): + self.name = name_ + self.entries = [] + if self.name is None: + # Add default entries + self.name = "sconsdoc" + self.addEntity("scons", "../scons.mod") + self.addEntity("builders-mod", "builders.mod") + self.addEntity("functions-mod", "functions.mod") + self.addEntity("tools-mod", "tools.mod") + self.addEntity("variables-mod", "variables.mod") + + def addEntity(self, name, uri): + self.entries.append(DoctypeEntity(name, uri)) + + def createDoctype(self): + content = '<!DOCTYPE %s [\n' % self.name + for e in self.entries: + content += e.getEntityString() + content += ']>\n' + + return content + +if not has_libxml2: + class TreeFactory: + def __init__(self): + pass + + def newNode(self, tag): + return etree.Element(tag) + + def newEtreeNode(self, tag, init_ns=False): + if init_ns: + NSMAP = {None: dbxsd, + 'xsi' : xsi} + return etree.Element(tag, nsmap=NSMAP) + + return etree.Element(tag) + + def copyNode(self, node): + return copy.deepcopy(node) + + def appendNode(self, parent, child): + parent.append(child) + + def hasAttribute(self, node, att): + return att in node.attrib + + def getAttribute(self, node, att): + return node.attrib[att] + + def setAttribute(self, node, att, value): + node.attrib[att] = value + + def getText(self, root): + return root.text + + def setText(self, root, txt): + root.text = txt + + def writeGenTree(self, root, fp): + dt = DoctypeDeclaration() + fp.write(etree.tostring(root, xml_declaration=True, + encoding="UTF-8", pretty_print=True, + doctype=dt.createDoctype())) + + def writeTree(self, root, fpath): + fp = open(fpath, 'w') + fp.write(etree.tostring(root, xml_declaration=True, + encoding="UTF-8", pretty_print=True)) + fp.close() + + def prettyPrintFile(self, fpath): + fin = open(fpath,'r') + tree = etree.parse(fin) + pretty_content = etree.tostring(tree, pretty_print=True) + fin.close() + + fout = open(fpath,'w') + fout.write(pretty_content) + fout.close() + + def decorateWithHeader(self, root): + root.attrib["{"+xsi+"}schemaLocation"] = "%s %s/scons.xsd" % (dbxsd, dbxsd) + return root + + def newXmlTree(self, root): + """ Return a XML file tree with the correct namespaces set, + the element root as top entry and the given header comment. + """ + NSMAP = {None: dbxsd, + 'xsi' : xsi} + t = etree.Element(root, nsmap=NSMAP) + return self.decorateWithHeader(t) + + def validateXml(self, fpath, xmlschema_context): + # Use lxml + xmlschema = etree.XMLSchema(xmlschema_context) + try: + doc = etree.parse(fpath) + except Exception, e: + print "ERROR: %s fails to parse:"%fpath + print e + return False + doc.xinclude() + try: + xmlschema.assertValid(doc) + except Exception, e: + print "ERROR: %s fails to validate:" % fpath + print e + return False + return True + + def findAll(self, root, tag, ns=None, xp_ctxt=None, nsmap=None): + expression = ".//{%s}%s" % (nsmap[ns], tag) + if not ns or not nsmap: + expression = ".//%s" % tag + return root.findall(expression) + + def findAllChildrenOf(self, root, tag, ns=None, xp_ctxt=None, nsmap=None): + expression = "./{%s}%s/*" % (nsmap[ns], tag) + if not ns or not nsmap: + expression = "./%s/*" % tag + return root.findall(expression) + + def convertElementTree(self, root): + """ Convert the given tree of etree.Element + entries to a list of tree nodes for the + current XML toolkit. + """ + return [root] + +else: + class TreeFactory: + def __init__(self): + pass + + def newNode(self, tag): + return libxml2.newNode(tag) + + def newEtreeNode(self, tag, init_ns=False): + return etree.Element(tag) + + def copyNode(self, node): + return node.copyNode(1) + + def appendNode(self, parent, child): + if hasattr(parent, 'addChild'): + parent.addChild(child) + else: + parent.append(child) + + def hasAttribute(self, node, att): + if hasattr(node, 'hasProp'): + return node.hasProp(att) + return att in node.attrib + + def getAttribute(self, node, att): + if hasattr(node, 'prop'): + return node.prop(att) + return node.attrib[att] + + def setAttribute(self, node, att, value): + if hasattr(node, 'setProp'): + node.setProp(att, value) + else: + node.attrib[att] = value + + def getText(self, root): + if hasattr(root, 'getContent'): + return root.getContent() + return root.text + + def setText(self, root, txt): + if hasattr(root, 'setContent'): + root.setContent(txt) + else: + root.text = txt + + def writeGenTree(self, root, fp): + doc = libxml2.newDoc('1.0') + dtd = doc.newDtd("sconsdoc", None, None) + doc.addChild(dtd) + doc.setRootElement(root) + content = doc.serialize("UTF-8", 1) + dt = DoctypeDeclaration() + # This is clearly a hack, but unfortunately libxml2 + # doesn't support writing PERs (Parsed Entity References). + # So, we simply replace the empty doctype with the + # text we need... + content = content.replace("<!DOCTYPE sconsdoc>", dt.createDoctype()) + fp.write(content) + doc.freeDoc() + + def writeTree(self, root, fpath): + fp = open(fpath, 'w') + doc = libxml2.newDoc('1.0') + doc.setRootElement(root) + fp.write(doc.serialize("UTF-8", 1)) + doc.freeDoc() + fp.close() + + def prettyPrintFile(self, fpath): + # Read file and resolve entities + doc = libxml2.readFile(fpath, None, libxml2d.XML_PARSE_NOENT) + fp = open(fpath, 'w') + # Prettyprint + fp.write(doc.serialize("UTF-8", 1)) + fp.close() + # Cleanup + doc.freeDoc() + + def decorateWithHeader(self, root): + # Register the namespaces + ns = root.newNs(dbxsd, None) + xi = root.newNs(xsi, 'xsi') + root.setNs(ns) #put this node in the target namespace + + root.setNsProp(xi, 'schemaLocation', "%s %s/scons.xsd" % (dbxsd, dbxsd)) + + return root + + def newXmlTree(self, root): + """ Return a XML file tree with the correct namespaces set, + the element root as top entry and the given header comment. + """ + t = libxml2.newNode(root) + return self.decorateWithHeader(t) + + def validateXml(self, fpath, xmlschema_context): + # Create validation context + validation_context = xmlschema_context.schemaNewValidCtxt() + # Set error/warning handlers + eh = Libxml2ValidityHandler() + validation_context.setValidityErrorHandler(eh.error, eh.warning, ARG) + # Read file and resolve entities + doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT) + doc.xincludeProcessFlags(libxml2.XML_PARSE_NOENT) + err = validation_context.schemaValidateDoc(doc) + # Cleanup + doc.freeDoc() + del validation_context + + if err or eh.errors: + for e in eh.errors: + print e.rstrip("\n") + print "%s fails to validate" % fpath + return False + + return True + + def findAll(self, root, tag, ns=None, xpath_context=None, nsmap=None): + if hasattr(root, 'xpathEval') and xpath_context: + # Use the xpath context + xpath_context.setContextNode(root) + expression = ".//%s" % tag + if ns: + expression = ".//%s:%s" % (ns, tag) + return xpath_context.xpathEval(expression) + else: + expression = ".//{%s}%s" % (nsmap[ns], tag) + if not ns or not nsmap: + expression = ".//%s" % tag + return root.findall(expression) + + def findAllChildrenOf(self, root, tag, ns=None, xpath_context=None, nsmap=None): + if hasattr(root, 'xpathEval') and xpath_context: + # Use the xpath context + xpath_context.setContextNode(root) + expression = "./%s/node()" % tag + if ns: + expression = "./%s:%s/node()" % (ns, tag) + + return xpath_context.xpathEval(expression) + else: + expression = "./{%s}%s/node()" % (nsmap[ns], tag) + if not ns or not nsmap: + expression = "./%s/node()" % tag + return root.findall(expression) + + def expandChildElements(self, child): + """ Helper function for convertElementTree, + converts a single child recursively. + """ + nchild = self.newNode(child.tag) + # Copy attributes + for key, val in child.attrib: + self.setAttribute(nchild, key, val) + elements = [] + # Add text + if child.text: + t = libxml2.newText(child.text) + self.appendNode(nchild, t) + # Add children + for c in child: + for n in self.expandChildElements(c): + self.appendNode(nchild, n) + elements.append(nchild) + # Add tail + if child.tail: + tail = libxml2.newText(child.tail) + elements.append(tail) + + return elements + + def convertElementTree(self, root): + """ Convert the given tree of etree.Element + entries to a list of tree nodes for the + current XML toolkit. + """ + nroot = self.newNode(root.tag) + # Copy attributes + for key, val in root.attrib: + self.setAttribute(nroot, key, val) + elements = [] + # Add text + if root.text: + t = libxml2.newText(root.text) + self.appendNode(nroot, t) + # Add children + for c in root: + for n in self.expandChildElements(c): + self.appendNode(nroot, n) + elements.append(nroot) + # Add tail + if root.tail: + tail = libxml2.newText(root.tail) + elements.append(tail) + + return elements + +tf = TreeFactory() + + +class SConsDocTree: + def __init__(self): + self.nsmap = {'dbx' : dbxsd} + self.doc = None + self.root = None + self.xpath_context = None + + def parseContent(self, content, include_entities=True): + """ Parses the given content as XML file. This method + is used when we generate the basic lists of entities + for the builders, tools and functions. + So we usually don't bother about namespaces and resolving + entities here...this is handled in parseXmlFile below + (step 2 of the overall process). + """ + if not include_entities: + content = remove_entities(content) + # Create domtree from given content string + self.root = etree.fromstring(content) + + def parseXmlFile(self, fpath): + nsmap = {'dbx' : dbxsd} + if not has_libxml2: + # Create domtree from file + domtree = etree.parse(fpath) + self.root = domtree.getroot() + else: + # Read file and resolve entities + self.doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT) + self.root = self.doc.getRootElement() + # Create xpath context + self.xpath_context = self.doc.xpathNewContext() + # Register namespaces + for key, val in self.nsmap.iteritems(): + self.xpath_context.xpathRegisterNs(key, val) + + def __del__(self): + if self.doc is not None: + self.doc.freeDoc() + if self.xpath_context is not None: + self.xpath_context.xpathFreeContext() + +perc="%" + +def validate_all_xml(dpaths, xsdfile=default_xsd): + xmlschema_context = None + if not has_libxml2: + # Use lxml + xmlschema_context = etree.parse(xsdfile) + else: + # Use libxml2 and prepare the schema validation context + ctxt = libxml2.schemaNewParserCtxt(xsdfile) + xmlschema_context = ctxt.schemaParse() + del ctxt + + fpaths = [] + for dp in dpaths: + if dp.endswith('.xml') and isSConsXml(dp): + path='.' + fpaths.append(dp) + else: + for path, dirs, files in os.walk(dp): + for f in files: + if f.endswith('.xml'): + fp = os.path.join(path, f) + if isSConsXml(fp): + fpaths.append(fp) + + fails = [] + for idx, fp in enumerate(fpaths): + fpath = os.path.join(path, fp) + print "%.2f%s (%d/%d) %s" % (float(idx+1)*100.0/float(len(fpaths)), + perc, idx+1, len(fpaths),fp) + + if not tf.validateXml(fp, xmlschema_context): + fails.append(fp) + continue + + if has_libxml2: + # Cleanup + del xmlschema_context + + if fails: + return False + + return True + +class Item(object): + def __init__(self, name): + self.name = name + self.sort_name = name.lower() + if self.sort_name[0] == '_': + self.sort_name = self.sort_name[1:] + self.sets = [] + self.uses = [] + self.summary = None + self.arguments = None + def cmp_name(self, name): + if name[0] == '_': + name = name[1:] + return name.lower() + def __cmp__(self, other): + return cmp(self.sort_name, other.sort_name) + +class Builder(Item): + pass + +class Function(Item): + def __init__(self, name): + super(Function, self).__init__(name) + +class Tool(Item): + def __init__(self, name): + Item.__init__(self, name) + self.entity = self.name.replace('+', 'X') + +class ConstructionVariable(Item): + pass + +class Arguments(object): + def __init__(self, signature, body=None): + if not body: + body = [] + self.body = body + self.signature = signature + def __str__(self): + s = ''.join(self.body).strip() + result = [] + for m in re.findall('([a-zA-Z/_]+|[^a-zA-Z/_]+)', s): + if ' ' in m: + m = '"%s"' % m + result.append(m) + return ' '.join(result) + def append(self, data): + self.body.append(data) + +class SConsDocHandler(object): + def __init__(self): + self.builders = {} + self.functions = {} + self.tools = {} + self.cvars = {} + + def parseItems(self, domelem, xpath_context, nsmap): + items = [] + + for i in tf.findAll(domelem, "item", dbxid, xpath_context, nsmap): + txt = tf.getText(i) + if txt is not None: + txt = txt.strip() + if len(txt): + items.append(txt.strip()) + + return items + + def parseUsesSets(self, domelem, xpath_context, nsmap): + uses = [] + sets = [] + + for u in tf.findAll(domelem, "uses", dbxid, xpath_context, nsmap): + uses.extend(self.parseItems(u, xpath_context, nsmap)) + for s in tf.findAll(domelem, "sets", dbxid, xpath_context, nsmap): + sets.extend(self.parseItems(s, xpath_context, nsmap)) + + return sorted(uses), sorted(sets) + + def parseInstance(self, domelem, map, Class, + xpath_context, nsmap, include_entities=True): + name = 'unknown' + if tf.hasAttribute(domelem, 'name'): + name = tf.getAttribute(domelem, 'name') + try: + instance = map[name] + except KeyError: + instance = Class(name) + map[name] = instance + uses, sets = self.parseUsesSets(domelem, xpath_context, nsmap) + instance.uses.extend(uses) + instance.sets.extend(sets) + if include_entities: + # Parse summary and function arguments + for s in tf.findAllChildrenOf(domelem, "summary", dbxid, xpath_context, nsmap): + if instance.summary is None: + instance.summary = [] + instance.summary.append(tf.copyNode(s)) + for a in tf.findAll(domelem, "arguments", dbxid, xpath_context, nsmap): + if instance.arguments is None: + instance.arguments = [] + instance.arguments.append(tf.copyNode(a)) + + def parseDomtree(self, root, xpath_context=None, nsmap=None, include_entities=True): + # Process Builders + for b in tf.findAll(root, "builder", dbxid, xpath_context, nsmap): + self.parseInstance(b, self.builders, Builder, + xpath_context, nsmap, include_entities) + # Process Functions + for f in tf.findAll(root, "scons_function", dbxid, xpath_context, nsmap): + self.parseInstance(f, self.functions, Function, + xpath_context, nsmap, include_entities) + # Process Tools + for t in tf.findAll(root, "tool", dbxid, xpath_context, nsmap): + self.parseInstance(t, self.tools, Tool, + xpath_context, nsmap, include_entities) + # Process CVars + for c in tf.findAll(root, "cvar", dbxid, xpath_context, nsmap): + self.parseInstance(c, self.cvars, ConstructionVariable, + xpath_context, nsmap, include_entities) + + def parseContent(self, content, include_entities=True): + """ Parses the given content as XML file. This method + is used when we generate the basic lists of entities + for the builders, tools and functions. + So we usually don't bother about namespaces and resolving + entities here...this is handled in parseXmlFile below + (step 2 of the overall process). + """ + # Create doctree + t = SConsDocTree() + t.parseContent(content, include_entities) + # Parse it + self.parseDomtree(t.root, t.xpath_context, t.nsmap, include_entities) + + def parseXmlFile(self, fpath): + # Create doctree + t = SConsDocTree() + t.parseXmlFile(fpath) + # Parse it + self.parseDomtree(t.root, t.xpath_context, t.nsmap) + +# lifted from Ka-Ping Yee's way cool pydoc module. +def importfile(path): + """Import a Python source file or compiled file given its path.""" + magic = imp.get_magic() + file = open(path, 'r') + if file.read(len(magic)) == magic: + kind = imp.PY_COMPILED + else: + kind = imp.PY_SOURCE + file.close() + filename = os.path.basename(path) + name, ext = os.path.splitext(filename) + file = open(path, 'r') + try: + module = imp.load_module(name, file, path, (ext, 'r', kind)) + except ImportError, e: + sys.stderr.write("Could not import %s: %s\n" % (path, e)) + return None + file.close() + return module + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/SConsExamples.py b/bin/SConsExamples.py new file mode 100644 index 0000000..9823a05 --- /dev/null +++ b/bin/SConsExamples.py @@ -0,0 +1,898 @@ +# !/usr/bin/env python +# +# Copyright (c) 2010 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# +# +# This script looks for some XML tags that describe SCons example +# configurations and commands to execute in those configurations, and +# uses TestCmd.py to execute the commands and insert the output from +# those commands into the XML that we output. This way, we can run a +# script and update all of our example documentation output without +# a lot of laborious by-hand checking. +# +# An "SCons example" looks like this, and essentially describes a set of +# input files (program source files as well as SConscript files): +# +# <scons_example name="ex1"> +# <file name="SConstruct" printme="1"> +# env = Environment() +# env.Program('foo') +# </file> +# <file name="foo.c"> +# int main() { printf("foo.c\n"); } +# </file> +# </scons_example> +# +# The <file> contents within the <scons_example> tag will get written +# into a temporary directory whenever example output needs to be +# generated. By default, the <file> contents are not inserted into text +# directly, unless you set the "printme" attribute on one or more files, +# in which case they will get inserted within a <programlisting> tag. +# This makes it easy to define the example at the appropriate +# point in the text where you intend to show the SConstruct file. +# +# Note that you should usually give the <scons_example> a "name" +# attribute so that you can refer to the example configuration later to +# run SCons and generate output. +# +# If you just want to show a file's contents without worry about running +# SCons, there's a shorter <sconstruct> tag: +# +# <sconstruct> +# env = Environment() +# env.Program('foo') +# </sconstruct> +# +# This is essentially equivalent to <scons_example><file printme="1">, +# but it's more straightforward. +# +# SCons output is generated from the following sort of tag: +# +# <scons_output example="ex1" os="posix"> +# <scons_output_command suffix="1">scons -Q foo</scons_output_command> +# <scons_output_command suffix="2">scons -Q foo</scons_output_command> +# </scons_output> +# +# You tell it which example to use with the "example" attribute, and then +# give it a list of <scons_output_command> tags to execute. You can also +# supply an "os" tag, which specifies the type of operating system this +# example is intended to show; if you omit this, default value is "posix". +# +# The generated XML will show the command line (with the appropriate +# command-line prompt for the operating system), execute the command in +# a temporary directory with the example files, capture the standard +# output from SCons, and insert it into the text as appropriate. +# Error output gets passed through to your error output so you +# can see if there are any problems executing the command. +# + +import os +import re +import sys +import time + +import SConsDoc +from SConsDoc import tf as stf + +# +# The available types for ExampleFile entries +# +FT_FILE = 0 # a physical file (=<file>) +FT_FILEREF = 1 # a reference (=<scons_example_file>) + +class ExampleFile: + def __init__(self, type_=FT_FILE): + self.type = type_ + self.name = '' + self.content = '' + self.chmod = '' + + def isFileRef(self): + return self.type == FT_FILEREF + +class ExampleFolder: + def __init__(self): + self.name = '' + self.chmod = '' + +class ExampleCommand: + def __init__(self): + self.edit = None + self.environment = '' + self.output = '' + self.cmd = '' + +class ExampleOutput: + def __init__(self): + self.name = '' + self.tools = '' + self.os = 'posix' + self.preserve = None + self.suffix = '' + self.commands = [] + +class ExampleInfo: + def __init__(self): + self.name = '' + self.files = [] + self.folders = [] + self.outputs = [] + + def getFileContents(self, fname): + for f in self.files: + if fname == f.name and not f.isFileRef(): + return f.content + + return '' + +def readExampleInfos(fpath, examples): + """ Add the example infos for the file fpath to the + global dictionary examples. + """ + + # Create doctree + t = SConsDoc.SConsDocTree() + t.parseXmlFile(fpath) + + # Parse scons_examples + for e in stf.findAll(t.root, "scons_example", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + n = '' + if stf.hasAttribute(e, 'name'): + n = stf.getAttribute(e, 'name') + if n and n not in examples: + i = ExampleInfo() + i.name = n + examples[n] = i + + # Parse file and directory entries + for f in stf.findAll(e, "file", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + fi = ExampleFile() + if stf.hasAttribute(f, 'name'): + fi.name = stf.getAttribute(f, 'name') + if stf.hasAttribute(f, 'chmod'): + fi.chmod = stf.getAttribute(f, 'chmod') + fi.content = stf.getText(f) + examples[n].files.append(fi) + for d in stf.findAll(e, "directory", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + di = ExampleFolder() + if stf.hasAttribute(d, 'name'): + di.name = stf.getAttribute(d, 'name') + if stf.hasAttribute(d, 'chmod'): + di.chmod = stf.getAttribute(d, 'chmod') + examples[n].folders.append(di) + + + # Parse scons_example_files + for f in stf.findAll(t.root, "scons_example_file", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + if stf.hasAttribute(f, 'example'): + e = stf.getAttribute(f, 'example') + else: + continue + fi = ExampleFile(FT_FILEREF) + if stf.hasAttribute(f, 'name'): + fi.name = stf.getAttribute(f, 'name') + if stf.hasAttribute(f, 'chmod'): + fi.chmod = stf.getAttribute(f, 'chmod') + fi.content = stf.getText(f) + examples[e].files.append(fi) + + + # Parse scons_output + for o in stf.findAll(t.root, "scons_output", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + if stf.hasAttribute(o, 'example'): + n = stf.getAttribute(o, 'example') + else: + continue + + eout = ExampleOutput() + if stf.hasAttribute(o, 'name'): + eout.name = stf.getAttribute(o, 'name') + if stf.hasAttribute(o, 'tools'): + eout.tools = stf.getAttribute(o, 'tools') + if stf.hasAttribute(o, 'os'): + eout.os = stf.getAttribute(o, 'os') + if stf.hasAttribute(o, 'suffix'): + eout.suffix = stf.getAttribute(o, 'suffix') + + for c in stf.findAll(o, "scons_output_command", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + oc = ExampleCommand() + if stf.hasAttribute(c, 'edit'): + oc.edit = stf.getAttribute(c, 'edit') + if stf.hasAttribute(c, 'environment'): + oc.environment = stf.getAttribute(c, 'environment') + if stf.hasAttribute(c, 'output'): + oc.output = stf.getAttribute(c, 'output') + if stf.hasAttribute(c, 'cmd'): + oc.cmd = stf.getAttribute(c, 'cmd') + else: + oc.cmd = stf.getText(c) + + eout.commands.append(oc) + + examples[n].outputs.append(eout) + +def readAllExampleInfos(dpath): + """ Scan for XML files in the given directory and + collect together all relevant infos (files/folders, + output commands) in a map, which gets returned. + """ + examples = {} + for path, dirs, files in os.walk(dpath): + for f in files: + if f.endswith('.xml'): + fpath = os.path.join(path, f) + if SConsDoc.isSConsXml(fpath): + readExampleInfos(fpath, examples) + + return examples + +generated_examples = os.path.join('doc', 'generated', 'examples') + +def ensureExampleOutputsExist(dpath): + """ Scan for XML files in the given directory and + ensure that for every example output we have a + corresponding output file in the 'generated/examples' + folder. + """ + # Ensure that the output folder exists + if not os.path.isdir(generated_examples): + os.mkdir(generated_examples) + + examples = readAllExampleInfos(dpath) + for key, value in examples.iteritems(): + # Process all scons_output tags + for o in value.outputs: + cpath = os.path.join(generated_examples, + key + '_' + o.suffix + '.xml') + if not os.path.isfile(cpath): + # Start new XML file + s = stf.newXmlTree("screen") + stf.setText(s, "NO OUTPUT YET! Run the script to generate/update all examples.") + # Write file + stf.writeTree(s, cpath) + + # Process all scons_example_file tags + for r in value.files: + if r.isFileRef(): + # Get file's content + content = value.getFileContents(r.name) + fpath = os.path.join(generated_examples, + key + '_' + r.name.replace("/", "_")) + # Write file + f = open(fpath, 'w') + f.write("%s\n" % content) + f.close() + +perc = "%" + +def createAllExampleOutputs(dpath): + """ Scan for XML files in the given directory and + creates all output files for every example in + the 'generated/examples' folder. + """ + # Ensure that the output folder exists + if not os.path.isdir(generated_examples): + os.mkdir(generated_examples) + + examples = readAllExampleInfos(dpath) + total = len(examples) + idx = 0 + for key, value in examples.iteritems(): + # Process all scons_output tags + print "%.2f%s (%d/%d) %s" % (float(idx + 1) * 100.0 / float(total), + perc, idx + 1, total, key) + + create_scons_output(value) + # Process all scons_example_file tags + for r in value.files: + if r.isFileRef(): + # Get file's content + content = value.getFileContents(r.name) + fpath = os.path.join(generated_examples, + key + '_' + r.name.replace("/", "_")) + # Write file + f = open(fpath, 'w') + f.write("%s\n" % content) + f.close() + idx += 1 + +def collectSConsExampleNames(fpath): + """ Return a set() of example names, used in the given file fpath. + """ + names = set() + suffixes = {} + failed_suffixes = False + + # Create doctree + t = SConsDoc.SConsDocTree() + t.parseXmlFile(fpath) + + # Parse it + for e in stf.findAll(t.root, "scons_example", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + n = '' + if stf.hasAttribute(e, 'name'): + n = stf.getAttribute(e, 'name') + if n: + names.add(n) + if n not in suffixes: + suffixes[n] = [] + else: + print "Error: Example in file '%s' is missing a name!" % fpath + failed_suffixes = True + + for o in stf.findAll(t.root, "scons_output", SConsDoc.dbxid, + t.xpath_context, t.nsmap): + n = '' + if stf.hasAttribute(o, 'example'): + n = stf.getAttribute(o, 'example') + else: + print "Error: scons_output in file '%s' is missing an example name!" % fpath + failed_suffixes = True + + if n not in suffixes: + print "Error: scons_output in file '%s' is referencing non-existent example '%s'!" % (fpath, n) + failed_suffixes = True + continue + + s = '' + if stf.hasAttribute(o, 'suffix'): + s = stf.getAttribute(o, 'suffix') + else: + print "Error: scons_output in file '%s' (example '%s') is missing a suffix!" % (fpath, n) + failed_suffixes = True + + if s not in suffixes[n]: + suffixes[n].append(s) + else: + print "Error: scons_output in file '%s' (example '%s') is using a duplicate suffix '%s'!" % (fpath, n, s) + failed_suffixes = True + + return names, failed_suffixes + +def exampleNamesAreUnique(dpath): + """ Scan for XML files in the given directory and + check whether the scons_example names are unique. + """ + unique = True + allnames = set() + for path, dirs, files in os.walk(dpath): + for f in files: + if f.endswith('.xml'): + fpath = os.path.join(path, f) + if SConsDoc.isSConsXml(fpath): + names, failed_suffixes = collectSConsExampleNames(fpath) + if failed_suffixes: + unique = False + i = allnames.intersection(names) + if i: + print "Not unique in %s are: %s" % (fpath, ', '.join(i)) + unique = False + + allnames |= names + + return unique + +# ############################################################### +# +# In the second half of this module (starting here) +# we define the variables and functions that are required +# to actually run the examples, collect their output and +# write it into the files in doc/generated/examples... +# which then get included by our UserGuide. +# +# ############################################################### + +sys.path.append(os.path.join(os.getcwd(), 'QMTest')) +sys.path.append(os.path.join(os.getcwd(), 'build', 'QMTest')) + +scons_py = os.path.join('bootstrap', 'src', 'script', 'scons.py') +if not os.path.exists(scons_py): + scons_py = os.path.join('src', 'script', 'scons.py') + +scons_lib_dir = os.path.join(os.getcwd(), 'bootstrap', 'src', 'engine') +if not os.path.exists(scons_lib_dir): + scons_lib_dir = os.path.join(os.getcwd(), 'src', 'engine') + +os.environ['SCONS_LIB_DIR'] = scons_lib_dir + +import TestCmd + +Prompt = { + 'posix' : '% ', + 'win32' : 'C:\\>' +} + +# The magick SCons hackery that makes this work. +# +# So that our examples can still use the default SConstruct file, we +# actually feed the following into SCons via stdin and then have it +# SConscript() the SConstruct file. This stdin wrapper creates a set +# of ToolSurrogates for the tools for the appropriate platform. These +# Surrogates print output like the real tools and behave like them +# without actually having to be on the right platform or have the right +# tool installed. +# +# The upshot: The wrapper transparently changes the world out from +# under the top-level SConstruct file in an example just so we can get +# the command output. + +Stdin = """\ +import os +import re +import SCons.Action +import SCons.Defaults +import SCons.Node.FS + +platform = '%(osname)s' + +Sep = { + 'posix' : '/', + 'win32' : '\\\\', +}[platform] + + +# Slip our own __str__() method into the EntryProxy class used to expand +# $TARGET{S} and $SOURCE{S} to translate the path-name separators from +# what's appropriate for the system we're running on to what's appropriate +# for the example system. +orig = SCons.Node.FS.EntryProxy +class MyEntryProxy(orig): + def __str__(self): + return str(self._subject).replace(os.sep, Sep) +SCons.Node.FS.EntryProxy = MyEntryProxy + +# Slip our own RDirs() method into the Node.FS.File class so that the +# expansions of $_{CPPINC,F77INC,LIBDIR}FLAGS will have the path-name +# separators translated from what's appropriate for the system we're +# running on to what's appropriate for the example system. +orig_RDirs = SCons.Node.FS.File.RDirs +def my_RDirs(self, pathlist, orig_RDirs=orig_RDirs): + return [str(x).replace(os.sep, Sep) for x in orig_RDirs(self, pathlist)] +SCons.Node.FS.File.RDirs = my_RDirs + +class Curry(object): + def __init__(self, fun, *args, **kwargs): + self.fun = fun + self.pending = args[:] + self.kwargs = kwargs.copy() + + def __call__(self, *args, **kwargs): + if kwargs and self.kwargs: + kw = self.kwargs.copy() + kw.update(kwargs) + else: + kw = kwargs or self.kwargs + + return self.fun(*self.pending + args, **kw) + +def Str(target, source, env, cmd=""): + result = [] + for cmd in env.subst_list(cmd, target=target, source=source): + result.append(' '.join(map(str, cmd))) + return '\\n'.join(result) + +class ToolSurrogate(object): + def __init__(self, tool, variable, func, varlist): + self.tool = tool + if not isinstance(variable, list): + variable = [variable] + self.variable = variable + self.func = func + self.varlist = varlist + def __call__(self, env): + t = Tool(self.tool) + t.generate(env) + for v in self.variable: + orig = env[v] + try: + strfunction = orig.strfunction + except AttributeError: + strfunction = Curry(Str, cmd=orig) + # Don't call Action() through its global function name, because + # that leads to infinite recursion in trying to initialize the + # Default Environment. + env[v] = SCons.Action.Action(self.func, + strfunction=strfunction, + varlist=self.varlist) + def __repr__(self): + # This is for the benefit of printing the 'TOOLS' + # variable through env.Dump(). + return repr(self.tool) + +def Null(target, source, env): + pass + +def Cat(target, source, env): + target = str(target[0]) + f = open(target, "wb") + for src in map(str, source): + f.write(open(src, "rb").read()) + f.close() + +def CCCom(target, source, env): + target = str(target[0]) + fp = open(target, "wb") + def process(source_file, fp=fp): + for line in open(source_file, "rb").readlines(): + m = re.match(r'#include\s[<"]([^<"]+)[>"]', line) + if m: + include = m.group(1) + for d in [str(env.Dir('$CPPPATH')), '.']: + f = os.path.join(d, include) + if os.path.exists(f): + process(f) + break + elif line[:11] != "STRIP CCCOM": + fp.write(line) + for src in map(str, source): + process(src) + fp.write('debug = ' + ARGUMENTS.get('debug', '0') + '\\n') + fp.close() + +public_class_re = re.compile('^public class (\S+)', re.MULTILINE) + +def JavaCCom(target, source, env): + # This is a fake Java compiler that just looks for + # public class FooBar + # lines in the source file(s) and spits those out + # to .class files named after the class. + tlist = list(map(str, target)) + not_copied = {} + for t in tlist: + not_copied[t] = 1 + for src in map(str, source): + contents = open(src, "rb").read() + classes = public_class_re.findall(contents) + for c in classes: + for t in [x for x in tlist if x.find(c) != -1]: + open(t, "wb").write(contents) + del not_copied[t] + for t in not_copied.keys(): + open(t, "wb").write("\\n") + +def JavaHCom(target, source, env): + tlist = map(str, target) + slist = map(str, source) + for t, s in zip(tlist, slist): + open(t, "wb").write(open(s, "rb").read()) + +def JarCom(target, source, env): + target = str(target[0]) + class_files = [] + for src in map(str, source): + for dirpath, dirnames, filenames in os.walk(src): + class_files.extend([ os.path.join(dirpath, f) + for f in filenames if f.endswith('.class') ]) + f = open(target, "wb") + for cf in class_files: + f.write(open(cf, "rb").read()) + f.close() + +# XXX Adding COLOR, COLORS and PACKAGE to the 'cc' varlist(s) by hand +# here is bogus. It's for the benefit of doc/user/command-line.in, which +# uses examples that want to rebuild based on changes to these variables. +# It would be better to figure out a way to do it based on the content of +# the generated command-line, or else find a way to let the example markup +# language in doc/user/command-line.in tell this script what variables to +# add, but that's more difficult than I want to figure out how to do right +# now, so let's just use the simple brute force approach for the moment. + +ToolList = { + 'posix' : [('cc', ['CCCOM', 'SHCCCOM'], CCCom, ['CCFLAGS', 'CPPDEFINES', 'COLOR', 'COLORS', 'PACKAGE']), + ('link', ['LINKCOM', 'SHLINKCOM'], Cat, []), + ('ar', ['ARCOM', 'RANLIBCOM'], Cat, []), + ('tar', 'TARCOM', Null, []), + ('zip', 'ZIPCOM', Null, []), + ('BitKeeper', 'BITKEEPERCOM', Cat, []), + ('CVS', 'CVSCOM', Cat, []), + ('RCS', 'RCS_COCOM', Cat, []), + ('SCCS', 'SCCSCOM', Cat, []), + ('javac', 'JAVACCOM', JavaCCom, []), + ('javah', 'JAVAHCOM', JavaHCom, []), + ('jar', 'JARCOM', JarCom, []), + ('rmic', 'RMICCOM', Cat, []), + ], + 'win32' : [('msvc', ['CCCOM', 'SHCCCOM', 'RCCOM'], CCCom, ['CCFLAGS', 'CPPDEFINES', 'COLOR', 'COLORS', 'PACKAGE']), + ('mslink', ['LINKCOM', 'SHLINKCOM'], Cat, []), + ('mslib', 'ARCOM', Cat, []), + ('tar', 'TARCOM', Null, []), + ('zip', 'ZIPCOM', Null, []), + ('BitKeeper', 'BITKEEPERCOM', Cat, []), + ('CVS', 'CVSCOM', Cat, []), + ('RCS', 'RCS_COCOM', Cat, []), + ('SCCS', 'SCCSCOM', Cat, []), + ('javac', 'JAVACCOM', JavaCCom, []), + ('javah', 'JAVAHCOM', JavaHCom, []), + ('jar', 'JARCOM', JarCom, []), + ('rmic', 'RMICCOM', Cat, []), + ], +} + +toollist = ToolList[platform] +filter_tools = '%(tools)s'.split() +if filter_tools: + toollist = [x for x in toollist if x[0] in filter_tools] + +toollist = [ToolSurrogate(*t) for t in toollist] + +toollist.append('install') + +def surrogate_spawn(sh, escape, cmd, args, env): + pass + +def surrogate_pspawn(sh, escape, cmd, args, env, stdout, stderr): + pass + +SCons.Defaults.ConstructionEnvironment.update({ + 'PLATFORM' : platform, + 'TOOLS' : toollist, + 'SPAWN' : surrogate_spawn, + 'PSPAWN' : surrogate_pspawn, +}) + +SConscript('SConstruct') +""" + +# "Commands" that we will execute in our examples. +def command_scons(args, c, test, dict): + save_vals = {} + delete_keys = [] + try: + ce = c.environment + except AttributeError: + pass + else: + for arg in c.environment.split(): + key, val = arg.split('=') + try: + save_vals[key] = os.environ[key] + except KeyError: + delete_keys.append(key) + os.environ[key] = val + test.run(interpreter=sys.executable, + program=scons_py, + # We use ToolSurrogates to capture win32 output by "building" + # examples using a fake win32 tool chain. Suppress the + # warnings that come from the new revamped VS support so + # we can build doc on (Linux) systems that don't have + # Visual C installed. + arguments='--warn=no-visual-c-missing -f - ' + ' '.join(args), + chdir=test.workpath('WORK'), + stdin=Stdin % dict) + os.environ.update(save_vals) + for key in delete_keys: + del(os.environ[key]) + out = test.stdout() + out = out.replace(test.workpath('ROOT'), '') + out = out.replace(test.workpath('WORK/SConstruct'), + '/home/my/project/SConstruct') + lines = out.split('\n') + if lines: + while lines[-1] == '': + lines = lines[:-1] + # err = test.stderr() + # if err: + # sys.stderr.write(err) + return lines + +def command_touch(args, c, test, dict): + if args[0] == '-t': + t = int(time.mktime(time.strptime(args[1], '%Y%m%d%H%M'))) + times = (t, t) + args = args[2:] + else: + time.sleep(1) + times = None + for file in args: + if not os.path.isabs(file): + file = os.path.join(test.workpath('WORK'), file) + if not os.path.exists(file): + open(file, 'wb') + os.utime(file, times) + return [] + +def command_edit(args, c, test, dict): + if c.edit is None: + add_string = 'void edit(void) { ; }\n' + else: + add_string = c.edit[:] + if add_string[-1] != '\n': + add_string = add_string + '\n' + for file in args: + if not os.path.isabs(file): + file = os.path.join(test.workpath('WORK'), file) + contents = open(file, 'rb').read() + open(file, 'wb').write(contents + add_string) + return [] + +def command_ls(args, c, test, dict): + def ls(a): + return [' '.join(sorted([x for x in os.listdir(a) if x[0] != '.']))] + if args: + l = [] + for a in args: + l.extend(ls(test.workpath('WORK', a))) + return l + else: + return ls(test.workpath('WORK')) + +def command_sleep(args, c, test, dict): + time.sleep(int(args[0])) + +CommandDict = { + 'scons' : command_scons, + 'touch' : command_touch, + 'edit' : command_edit, + 'ls' : command_ls, + 'sleep' : command_sleep, +} + +def ExecuteCommand(args, c, t, dict): + try: + func = CommandDict[args[0]] + except KeyError: + func = lambda args, c, t, dict: [] + return func(args[1:], c, t, dict) + + +def create_scons_output(e): + # The real raison d'etre for this script, this is where we + # actually execute SCons to fetch the output. + + # Loop over all outputs for the example + for o in e.outputs: + # Create new test directory + t = TestCmd.TestCmd(workdir='', combine=1) + if o.preserve: + t.preserve() + t.subdir('ROOT', 'WORK') + t.rootpath = t.workpath('ROOT').replace('\\', '\\\\') + + for d in e.folders: + dir = t.workpath('WORK', d.name) + if not os.path.exists(dir): + os.makedirs(dir) + + for f in e.files: + if f.isFileRef(): + continue + # + # Left-align file's contents, starting on the first + # non-empty line + # + data = f.content.split('\n') + i = 0 + # Skip empty lines + while data[i] == '': + i = i + 1 + lines = data[i:] + i = 0 + # Scan first line for the number of spaces + # that this block is indented + while lines[0][i] == ' ': + i = i + 1 + # Left-align block + lines = [l[i:] for l in lines] + path = f.name.replace('__ROOT__', t.rootpath) + if not os.path.isabs(path): + path = t.workpath('WORK', path) + dir, name = os.path.split(path) + if dir and not os.path.exists(dir): + os.makedirs(dir) + content = '\n'.join(lines) + content = content.replace('__ROOT__', t.rootpath) + path = t.workpath('WORK', path) + t.write(path, content) + if hasattr(f, 'chmod'): + if len(f.chmod): + os.chmod(path, int(f.chmod, 0)) + + # Regular expressions for making the doc output consistent, + # regardless of reported addresses or Python version. + + # Massage addresses in object repr strings to a constant. + address_re = re.compile(r' at 0x[0-9a-fA-F]*\>') + + # Massage file names in stack traces (sometimes reported as absolute + # paths) to a consistent relative path. + engine_re = re.compile(r' File ".*/src/engine/SCons/') + + # Python 2.5 changed the stack trace when the module is read + # from standard input from read "... line 7, in ?" to + # "... line 7, in <module>". + file_re = re.compile(r'^( *File ".*", line \d+, in) \?$', re.M) + + # Python 2.6 made UserList a new-style class, which changes the + # AttributeError message generated by our NodeList subclass. + nodelist_re = re.compile(r'(AttributeError:) NodeList instance (has no attribute \S+)') + + # Root element for our subtree + sroot = stf.newEtreeNode("screen", True) + curchild = None + content = "" + for c in o.commands: + content += Prompt[o.os] + if curchild is not None: + if not c.output: + # Append content as tail + curchild.tail = content + content = "\n" + # Add new child for userinput tag + curchild = stf.newEtreeNode("userinput") + d = c.cmd.replace('__ROOT__', '') + curchild.text = d + sroot.append(curchild) + else: + content += c.output + '\n' + else: + if not c.output: + # Add first text to root + sroot.text = content + content = "\n" + # Add new child for userinput tag + curchild = stf.newEtreeNode("userinput") + d = c.cmd.replace('__ROOT__', '') + curchild.text = d + sroot.append(curchild) + else: + content += c.output + '\n' + # Execute command and capture its output + cmd_work = c.cmd.replace('__ROOT__', t.workpath('ROOT')) + args = cmd_work.split() + lines = ExecuteCommand(args, c, t, {'osname':o.os, 'tools':o.tools}) + if not c.output and lines: + ncontent = '\n'.join(lines) + ncontent = address_re.sub(r' at 0x700000>', ncontent) + ncontent = engine_re.sub(r' File "bootstrap/src/engine/SCons/', ncontent) + ncontent = file_re.sub(r'\1 <module>', ncontent) + ncontent = nodelist_re.sub(r"\1 'NodeList' object \2", ncontent) + ncontent = ncontent.replace('__ROOT__', '') + content += ncontent + '\n' + # Add last piece of content + if len(content): + if curchild is not None: + curchild.tail = content + else: + sroot.text = content + + # Construct filename + fpath = os.path.join(generated_examples, + e.name + '_' + o.suffix + '.xml') + # Expand Element tree + s = stf.decorateWithHeader(stf.convertElementTree(sroot)[0]) + # Write it to file + stf.writeTree(s, fpath) + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/ae-cvs-ci b/bin/ae-cvs-ci new file mode 100755 index 0000000..47c8073 --- /dev/null +++ b/bin/ae-cvs-ci @@ -0,0 +1,204 @@ +# +# aegis - project change supervisor +# Copyright (C) 2004 Peter Miller; +# All rights reserved. +# +# As a specific exception to the GPL, you are allowed to copy +# this source file into your own project and modify it, without +# releasing your project under the GPL, unless there is some other +# file or condition which would require it. +# +# MANIFEST: shell script to commit changes to CVS +# +# It is assumed that your CVSROOT and CVS_RSH environment variables have +# already been set appropriately. +# +# This script is expected to be run as by integrate_pass_notify_command +# and as such the baseline has already assumed the shape asked for by +# the change. +# +# integrate_pass_notify_command = +# "$bin/ae-cvs-ci $project $change"; +# +# Alternatively, you may wish to tailor this script to the individual +# needs of your project. Make it a source file, e.g. "etc/ae-cvs-ci.sh" +# and then use the following: +# +# integrate_pass_notify_command = +# "$sh ${s etc/ae-cvs-ci} $project $change"; +# + +USAGE="Usage: $0 <project> <change>" + +PRINT="echo" +EXECUTE="eval" + +while getopts "hnq" FLAG +do + case ${FLAG} in + h ) + echo "${USAGE}" + exit 0 + ;; + n ) + EXECUTE=":" + ;; + q ) + PRINT=":" + ;; + * ) + echo "$0: unknown option ${FLAG}" >&2 + exit 1 + ;; + esac +done + +shift `expr ${OPTIND} - 1` + +case $# in +2) + project=$1 + change=$2 + ;; +*) + echo "${USAGE}" 1>&2 + exit 1 + ;; +esac + +here=`pwd` + +AEGIS_PROJECT=$project +export AEGIS_PROJECT +AEGIS_CHANGE=$change +export AEGIS_CHANGE + +module=`echo $project | sed 's|[.].*||'` + +baseline=`aegis -cd -bl` + +if test X${TMPDIR} = X; then TMPDIR=/var/tmp; fi + +TMP=${TMPDIR}/ae-cvs-ci.$$ +mkdir ${TMP} +cd ${TMP} + +PWD=`pwd` +if test X${PWD} != X${TMP}; then + echo "$0: ended up in ${PWD}, not ${TMP}" >&2 + exit 1 +fi + +fail() +{ + set +x + cd $here + rm -rf ${TMP} + echo "FAILED" 1>&2 + exit 1 +} +trap "fail" 1 2 3 15 + +Command() +{ + ${PRINT} "$*" + ${EXECUTE} "$*" +} + +# +# Create a new CVS work area. +# +# Note: this assumes the module is checked-out into a directory of the +# same name. Is there a way to ask CVS where is is going to put a +# modules, so we can always get the "cd" right? +# +${PRINT} cvs co $module +${EXECUTE} cvs co $module > LOG 2>&1 +if test $? -ne 0; then cat LOG; fail; fi +${EXECUTE} cd $module + +# +# Now we need to extract the sources from Aegis and drop them into the +# CVS work area. There are two ways to do this. +# +# The first way is to use the generated tarball. +# This has the advantage that it has the Makefile.in file in it, and +# will work immediately. +# +# The second way is to use aetar, which will give exact sources, and +# omit all derived files. This will *not* include the Makefile.in, +# and so will not be readily compilable. +# +# gunzip < $baseline/export/${project}.tar.gz | tardy -rp ${project} | tar xf - +aetar -send -comp-alg=gzip -o - | tar xzf - + +# +# If any new directories have been created we will need to add them +# to CVS before we can add the new files which we know are in them, +# or they would not have been created. Do this only if the -n option +# isn't used, because if it is, we won't have actually checked out the +# source and we'd erroneously report that all of them need to be added. +# +if test "X${EXECUTE}" != "X:" +then + find . \( -name CVS -o -name Attic \) -prune -o -type d -print | + xargs --max-args=1 | + while read dir + do + if [ ! -d "$dir/CVS" ] + then + Command cvs add "$dir" + fi + done +fi + +# +# Use the Aegis meta-data to perform some CVS commands that CVS can't +# figure out for itself. +# +aegis -l cf -unf | sed 's| -> [0-9][0-9.]*||' | +while read usage action rev filename +do + if test "x$filename" = "x" + then + filename="$rev" + fi + case $action in + create) + Command cvs add $filename + ;; + remove) + Command rm -f $filename + Command cvs remove $filename + ;; + *) + ;; + esac +done + +# +# Extract the brief description. We'd like to do this using aesub +# or something, like so: +# +# message=`aesub '${version} - ${change description}'` +# +# but the expansion of ${change description} has a lame hard-coded max of +# 80 characters, so we have to do this by hand. (This has the slight +# benefit of preserving backslashes in front of any double-quotes in +# the text; that will have to be handled if we go back to using aesub.) +# +description=`aegis -ca -l | sed -n 's/brief_description = "\(.*\)";$/\1/p'` +version=`aesub '${version}'` +message="$version - $description" + +# +# Now commit all the changes. +# +Command cvs -q commit -m \"$message\" + +# +# All done. Clean up and go home. +# +cd $here +rm -rf ${TMP} +exit 0 diff --git a/bin/ae-svn-ci b/bin/ae-svn-ci new file mode 100755 index 0000000..301d890 --- /dev/null +++ b/bin/ae-svn-ci @@ -0,0 +1,240 @@ +# +# aegis - project change supervisor +# Copyright (C) 2004 Peter Miller; +# All rights reserved. +# +# As a specific exception to the GPL, you are allowed to copy +# this source file into your own project and modify it, without +# releasing your project under the GPL, unless there is some other +# file or condition which would require it. +# +# MANIFEST: shell script to commit changes to Subversion +# +# This script is expected to be run by the integrate_pass_notify_command +# and as such the baseline has already assumed the shape asked for by +# the change. +# +# integrate_pass_notify_command = +# "$bin/ae-svn-ci $project $change http://svn.site.com/svn/trunk --username svn_user"; +# +# Alternatively, you may wish to tailor this script to the individual +# needs of your project. Make it a source file, e.g. "etc/ae-svn-ci.sh" +# and then use the following: +# +# integrate_pass_notify_command = +# "$sh ${s etc/ae-svn-ci} $project $change http://svn.site.com/svn/trunk --username svn_user"; +# + +USAGE="Usage: $0 [-hnq] <project> <change> <url> [<co_options>]" + +PRINT="echo" +EXECUTE="eval" + +while getopts "hnq" FLAG +do + case ${FLAG} in + h ) + echo "${USAGE}" + exit 0 + ;; + n ) + EXECUTE=":" + ;; + q ) + PRINT=":" + ;; + * ) + echo "$0: unknown option ${FLAG}" >&2 + exit 1 + ;; + esac +done + +shift `expr ${OPTIND} - 1` + +case $# in +[012]) + echo "${USAGE}" 1>&2 + exit 1 + ;; +*) + project=$1 + change=$2 + svn_url=$3 + shift 3 + svn_co_flags=$* + ;; +esac + +here=`pwd` + +AEGIS_PROJECT=$project +export AEGIS_PROJECT +AEGIS_CHANGE=$change +export AEGIS_CHANGE + +module=`echo $project | sed 's|[.].*||'` + +baseline=`aegis -cd -bl` + +if test X${TMPDIR} = X; then TMPDIR=/var/tmp; fi + +TMP=${TMPDIR}/ae-svn-ci.$$ +mkdir ${TMP} +cd ${TMP} + +PWD=`pwd` +if test X${PWD} != X${TMP}; then + echo "$0: ended up in ${PWD}, not ${TMP}" >&2 + exit 1 +fi + +fail() +{ + set +x + cd $here + rm -rf ${TMP} + echo "FAILED" 1>&2 + exit 1 +} +trap "fail" 1 2 3 15 + +Command() +{ + ${PRINT} "$*" + ${EXECUTE} "$*" +} + +# +# Create a new Subversion work area. +# +# Note: this assumes the module is checked-out into a directory of the +# same name. Is there a way to ask Subversion where it is going to put a +# module, so we can always get the "cd" right? +# +${PRINT} svn co $svn_url $module $svn_co_flags +${EXECUTE} svn co $svn_url $module $svn_co_flags > LOG 2>&1 +if test $? -ne 0; then cat LOG; fail; fi +${EXECUTE} cd $module + +# +# Now we need to extract the sources from Aegis and drop them into the +# Subversion work area. There are two ways to do this. +# +# The first way is to use the generated tarball. +# This has the advantage that it has the Makefile.in file in it, and +# will work immediately. +# +# The second way is to use aetar, which will give exact sources, and +# omit all derived files. This will *not* include the Makefile.in, +# and so will not be readily compilable. +# +# gunzip < $baseline/export/${project}.tar.gz | tardy -rp ${project} | tar xf - +aetar -send -comp-alg=gzip -o - | tar xzf - + +# +# If any new directories have been created we will need to add them +# to Subversion before we can add the new files which we know are in them, +# or they would not have been created. Do this only if the -n option +# isn't used, because if it is, we won't have actually checked out the +# source and we'd erroneously report that all of them need to be added. +# +if test "X${EXECUTE}" != "X:" +then + find . -name .svn -prune -o -type d -print | + xargs --max-args=1 | + while read dir + do + if [ ! -d "$dir/.svn" ] + then + Command svn add -N "$dir" + fi + done +fi + +# +# Use the Aegis meta-data to perform some commands that Subversion can't +# figure out for itself. We use an inline "aer" report script to identify +# when a remove-create pair are actually due to a move. +# +aegis -rpt -nph -f - <<_EOF_ | +auto cs; +cs = project[project_name()].state.branch.change[change_number()]; + +columns({width = 1000;}); + +auto file, moved; +for (file in cs.src) +{ + if (file.move != "") + moved[file.move] = 1; +} + +auto action; +for (file in cs.src) +{ + if (file.action == "remove" && file.move != "") + action = "move"; + else + action = file.action; + /* + * Suppress printing of any files created as the result of a move. + * These are printed as the destination when printing the line for + * the file that was *removed* as a result of the move. + */ + if (action != "create" || ! moved[file.file_name]) + print(sprintf("%s %s \\"%s\\" \\"%s\\"", file.usage, action, file.file_name, file.move)); +} +_EOF_ +while read line +do + eval set -- "$line" + usage="$1" + action="$2" + srcfile="$3" + dstfile="$4" + case $action in + create) + Command svn add $srcfile + ;; + remove) + Command rm -f $srcfile + Command svn remove $srcfile + ;; + move) + Command mv $dstfile $dstfile.move + Command svn move $srcfile $dstfile + Command cp $dstfile.move $dstfile + Command rm -f $dstfile.move + ;; + *) + ;; + esac +done + +# +# Extract the brief description. We'd like to do this using aesub +# or something, like so: +# +# message=`aesub '${version} - ${change description}'` +# +# but the expansion of ${change description} has a lame hard-coded max of +# 80 characters, so we have to do this by hand. (This has the slight +# benefit of preserving backslashes in front of any double-quotes in +# the text; that will have to be handled if we go back to using aesub.) +# +description=`aegis -ca -l | sed -n 's/brief_description = "\(.*\)";$/\1/p'` +version=`aesub '${version}'` +message="$version - $description" + +# +# Now commit all the changes. +# +Command svn commit -m \"$message\" + +# +# All done. Clean up and go home. +# +cd $here +rm -rf ${TMP} +exit 0 diff --git a/bin/ae2cvs b/bin/ae2cvs new file mode 100644 index 0000000..e7cb22b --- /dev/null +++ b/bin/ae2cvs @@ -0,0 +1,579 @@ +#! /usr/bin/env perl + +$revision = "src/ae2cvs.pl 0.04.D001 2005/08/14 15:13:36 knight"; + +$copyright = "Copyright 2001, 2002, 2003, 2004, 2005 Steven Knight."; + +# +# All rights reserved. This program is free software; you can +# redistribute and/or modify under the same terms as Perl itself. +# + +use strict; +use File::Find; +use File::Spec; +use Pod::Usage (); + +use vars qw( @add_list @args @cleanup @copy_list @libraries + @mkdir_list @remove_list + %seen_dir + $ae_copy $aedir $aedist + $cnum $comment $commit $common $copyright + $cvs_command $cvsmod $cvsroot + $delta $description $exec $help $indent $infile + $proj $pwd $quiet $revision + $summary $usedir $usepath ); + +$aedist = 1; +$cvsroot = undef; +$exec = undef; +$indent = ""; + +sub version { + print "ae2cvs: $revision\n"; + print "$copyright\n"; + exit 0; +} + +{ + use Getopt::Long; + + Getopt::Long::Configure('no_ignore_case'); + + my $ret = GetOptions ( + "aedist" => sub { $aedist = 1 }, + "aegis" => sub { $aedist = 0 }, + "change=i" => \$cnum, + "d=s" => \$cvsroot, + "file=s" => \$infile, + "help|?" => \$help, + "library=s" => \@libraries, + "module=s" => \$cvsmod, + "noexecute" => sub { $exec = 0 }, + "project=s" => \$proj, + "quiet" => \$quiet, + "usedir=s" => \$usedir, + "v|version" => \&version, + "x|execute" => sub { $exec++ if ! defined $exec || $exec != 0 }, + "X|EXECUTE" => sub { $exec = 2 if ! defined $exec || $exec != 0 }, + ); + + Pod::Usage::pod2usage(-verbose => 0) if $help || ! $ret; + + $exec = 0 if ! defined $exec; +} + +$cvs_command = $cvsroot ? "cvs -d $cvsroot -Q" : "cvs -Q"; + +# +# Wrap up the $quiet logic in one place. +# +sub printit { + return if $quiet; + my $string = join('', @_); + $string =~ s/^/$indent/msg if $indent; + print $string; +} + +# +# Wrappers for executing various builtin Perl functions in +# accordance with the -n, -q and -x options. +# +sub execute { + my $cmd = shift; + printit "$cmd\n"; + if (! $exec) { + return 1; + } + ! system($cmd); +} + +sub _copy { + my ($source, $dest) = @_; + printit "cp $source $dest\n"; + if ($exec) { + use File::Copy; + copy($source, $dest); + } +} + +sub _chdir { + my $dir = shift; + printit "cd $dir\n"; + if ($exec) { + chdir($dir) || die "ae2cvs: could not chdir($dir): $!"; + } +} + +sub _mkdir { + my $dir = shift; + printit "mkdir $dir\n"; + if ($exec) { + mkdir($dir); + } +} + +# +# Put some input data through an external filter and capture the output. +# +sub filter { + my ($cmd, $input) = @_; + + use FileHandle; + use IPC::Open2; + + my $pid = open2(*READ, *WRITE, $cmd) || die "Cannot exec '$cmd': $!\n"; + print WRITE $input; + close(WRITE); + my $output = join('', <READ>); + close(READ); + return $output; +} + +# +# Parse a change description, in both 'aegis -l cd" and "aedist" formats. +# +# Returns an array containing the project name, the change number +# (if any), the delta number (if any), the SUMMARY, the DESCRIPTION +# and the lines describing the files in the change. +# +sub parse_change { + my $output = shift; + + my ($p, $c, $d, $c_or_d, $sum, $desc, $filesection, @flines); + + # The project name line comes after NAME in "aegis -l cd" format, + # and PROJECT in "aedist" format. In both cases, the project name + # and the change/delta name are separated a comma. + ($p = $output) =~ s/(?:NAME|PROJECT)\n([^\n]*)\n.*/$1/ms; + ($p, $c_or_d) = (split(/,/, $p)); + + # In "aegis -l cd" format, the project name actually comes after + # the string "Project" and is itself enclosed in double quotes. + $p =~ s/Project "([^"]*)"/$1/; + + # The change or delta string was the right-hand side of the comma. + # "aegis -l cd" format spells it "Change 123." or "Delta 123." while + # "aedist" format spells it "change 123." + if ($c_or_d =~ /\s*[Cc]hange (\d+).*/) { $c = $1 }; + if ($c_or_d =~ /\s*[Dd]elta (\d+).*/) { $d = $1 }; + + # The SUMMARY line is always followed the DESCRIPTION section. + # It seems to always be a single line, but we grab everything in + # between just in case. + ($sum = $output) =~ s/.*\nSUMMARY\n//ms; + $sum =~ s/\nDESCRIPTION\n.*//ms; + + # The DESCRIPTION section is followed ARCHITECTURE in "aegis -l cd" + # format and by CAUSE in "aedist" format. Explicitly under it if the + # string is only "none," which means they didn't supply a description. + ($desc = $output) =~ s/.*\nDESCRIPTION\n//ms; + $desc =~ s/\n(ARCHITECTURE|CAUSE)\n.*//ms; + chomp($desc); + if ($desc eq "none" || $desc eq "none\n") { $desc = undef } + + # The FILES section is followed by HISTORY in "aegis -l cd" format. + # It seems to be the last section in "aedist" format, but stripping + # a non-existent HISTORY section doesn't hurt. + ($filesection = $output) =~ s/.*\nFILES\n//ms; + $filesection =~ s/\nHISTORY\n.*//ms; + + @flines = split(/\n/, $filesection); + + ($p, $c, $d, $sum, $desc, \@flines) +} + +# +# +# +$pwd = Cwd::cwd(); + +# +# Fetch the file list either from our aedist input +# or directly from the project itself. +# +my @filelines; +if ($aedist) { + local ($/); + undef $/; + my $infile_redir = ""; + my $contents; + if (! $infile || $infile eq "-") { + $contents = join('', <STDIN>); + } else { + open(FILE, "<$infile") || die "Cannot open '$infile': $!\n"; + binmode(FILE); + $contents = join('', <FILE>); + close(FILE); + if (! File::Spec->file_name_is_absolute($infile)) { + $infile = File::Spec->catfile($pwd, $infile); + } + $infile_redir = " < $infile"; + } + + my $output = filter("aedist -l -unf", $contents); + my ($p, $c, $d, $s, $desc, $fl) = parse_change($output); + + $proj = $p if ! defined $proj; + $summary = $s; + $description = $desc; + @filelines = @$fl; + + if (! $exec) { + printit qq(MYTMP="/tmp/ae2cvs-ae.\$\$"\n), + qq(mkdir \$MYTMP\n), + qq(cd \$MYTMP\n); + printit q(perl -MMIME::Base64 -e 'undef $/; ($c = <>) =~ s/.*\n\n//ms; print decode_base64($c)'), + $infile_redir, + qq( | zcat), + qq( | cpio -i -d --quiet\n); + $aedir = '$MYTMP'; + push(@cleanup, $aedir); + } else { + $aedir = File::Spec->catfile(File::Spec->tmpdir, "ae2cvs-ae.$$"); + _mkdir($aedir); + push(@cleanup, $aedir); + _chdir($aedir); + + use MIME::Base64; + + $contents =~ s/.*\n\n//ms; + $contents = filter("zcat", decode_base64($contents)); + + open(CPIO, "|cpio -i -d --quiet"); + print CPIO $contents; + close(CPIO); + } + + $ae_copy = sub { + foreach my $dest (@_) { + my $source = File::Spec->catfile($aedir, "src", $dest); + execute(qq(cp $source $dest)); + } + } +} else { + $cnum = $ENV{AEGIS_CHANGE} if ! defined $cnum; + $proj = $ENV{AEGIS_PROJECT} if ! defined $proj; + + $common = "-lib " . join(" -lib ", @libraries) if @libraries; + $common = "$common -proj $proj" if $proj; + + my $output = `aegis -l cd $cnum -unf $common`; + my ($p, $c, $d, $s, $desc, $fl) = parse_change($output); + + $delta = $d; + $summary = $s; + $description = $desc; + @filelines = @$fl; + + if (! $delta) { + print STDERR "ae2cvs: No delta number, exiting.\n"; + exit 1; + } + + $ae_copy = sub { + execute(qq(aegis -cp -ind -delta $delta $common @_)); + } +} + +if (! $usedir) { + $usedir = File::Spec->catfile(File::Spec->tmpdir, "ae2cvs.$$"); + _mkdir($usedir); + push(@cleanup, $usedir); +} + +_chdir($usedir); + +$usepath = $usedir; +if (! File::Spec->file_name_is_absolute($usepath)) { + $usepath = File::Spec->catfile($pwd, $usepath); +} + +if (! -d File::Spec->catfile($usedir, "CVS")) { + $cvsmod = (split(/\./, $proj))[0] if ! defined $cvsmod; + + execute(qq($cvs_command co $cvsmod)); + + _chdir($cvsmod); + + $usepath = File::Spec->catfile($usepath, $cvsmod); +} + +# +# Figure out what we have to do to accomplish everything. +# +foreach (@filelines) { + my @arr = split(/\s+/, $_); + my $type = shift @arr; # source / test + my $act = shift @arr; # modify / create + my $file = pop @arr; + + if ($act eq "create" or $act eq "modify") { + # XXX Do we really only need to do this for + # ($act eq "create") files? + my (undef, $dirs, undef) = File::Spec->splitpath($file); + my $absdir = $usepath; + my $reldir; + my $d; + foreach $d (File::Spec->splitdir($dirs)) { + next if ! $d; + $absdir = File::Spec->catdir($absdir, $d); + $reldir = $reldir ? File::Spec->catdir($reldir, $d) : $d; + if (! -d $absdir && ! $seen_dir{$reldir}) { + $seen_dir{$reldir} = 1; + push(@mkdir_list, $reldir); + } + } + + push(@copy_list, $file); + + if ($act eq "create") { + push(@add_list, $file); + } + } elsif ($act eq "remove") { + push(@remove_list, $file); + } else { + print STDERR "Unsure how to '$act' the '$file' file.\n"; + } +} + +# Now go through and mkdir() the directories, +# adding them to the CVS tree as we do. +if (@mkdir_list) { + if (! $exec) { + printit qq(# The following "mkdir" and "cvs -Q add" calls are not\n), + qq(# necessary for any directories that already exist in the\n), + qq(# CVS tree but which aren't present locally.\n); + } + foreach (@mkdir_list) { + if (! $exec) { + printit qq(if test ! -d $_; then\n); + $indent = " "; + } + _mkdir($_); + execute(qq($cvs_command add $_)); + if (! $exec) { + $indent = ""; + printit qq(fi\n); + } + } + if (! $exec) { + printit qq(# End of directory creation.\n); + } +} + +# Copy in any files in the change, before we try to "cvs add" them. +$ae_copy->(@copy_list) if @copy_list; + +if (@add_list) { + execute(qq($cvs_command add @add_list)); +} + +if (@remove_list) { + execute(qq(rm -f @remove_list)); + execute(qq($cvs_command remove @remove_list)); +} + +# Last, commit the whole bunch. +$comment = $summary; +$comment .= "\n" . $description if $description; +$commit = qq($cvs_command commit -m '$comment' .); +if ($exec == 1) { + printit qq(# Execute the following to commit the changes:\n), + qq(# $commit\n); +} else { + execute($commit); +} + +_chdir($pwd); + +# +# Directory cleanup. +# +sub END { + my $dir; + foreach $dir (@cleanup) { + printit "rm -rf $dir\n"; + if ($exec) { + finddepth(sub { + # print STDERR "unlink($_)\n" if (!-d $_); + # print STDERR "rmdir($_)\n" if (-d $_ && $_ ne "."); + unlink($_) if (!-d $_); + rmdir($_) if (-d $_ && $_ ne "."); + 1; + }, $dir); + rmdir($dir) || print STDERR "Could not remove $dir: $!\n"; + } + } +} + +__END__; + +=head1 NAME + +ae2cvs - convert an Aegis change set to CVS commands + +=head1 SYNOPSIS + +ae2cvs [-aedist|-aegis] [-c change] [-d cvs_root] [-f file] [-l lib] + [-m module] [-n] [-p proj] [-q] [-u dir] [-v] [-x] [-X] + + -aedist use aedist format from input (default) + -aegis query aegis repository directly + -c change change number + -d cvs_root CVS root directory + -f file read aedist from file ('-' == stdin) + -l lib Aegis library directory + -m module CVS module + -n no execute + -p proj project name + -q quiet, don't print commands + -u dir use dir for CVS checkin + -v print version string and exit + -x execute the commands, but don't commit; + two or more -x options commit changes + -X execute the commands and commit changes + +=head1 DESCRIPTION + +The C<ae2cvs> utility can convert an Aegis change into a set of CVS (and +other) commands to make the corresponding change(s) to a carbon-copy CVS +repository. This can be used to keep a front-end CVS repository in sync +with changes made to an Aegis project, either manually or automatically +using the C<integrate_pass_notify_command> attribute of the Aegis +project. + +By default, C<ae2cvs> makes no changes to any software, and only prints +out the necessary commands. These commands can be examined first for +safety, and then fed to any Bourne shell variant (sh, ksh, or bash) to +make the actual CVS changes. + +An option exists to have C<ae2cvs> execute the commands directly. + +=head1 OPTIONS + +The C<ae2cvs> utility supports the following options: + +=over 4 + +=item -aedist + +Reads an aedist change set. +By default, the change set is read from standard input, +or a file specified with the C<-f> option. + +=item -aegis + +Reads the change directly from the Aegis repository +by executing the proper C<aegis> commands. + +=item -c change + +Specify the Aegis change number to be used. +The value of the C<AEGIS_CHANGE> environment variable +is used by default. + +=item -d cvsroot + +Specify the CVS root directory to be used. +This option is passed explicitly to each executed C<cvs> command. +The default behavior is to omit any C<-d> options +and let the executed C<cvs> commands use the +C<CVSROOT> environment variable as they normally would. + +=item -f file + +Reads the aedist change set from the specified C<file>, +or from standard input if C<file> is C<'-'>. + +=item -l lib + +Specifies an Aegis library directory to be searched for global states +files and user state files. + +=item -m module + +Specifies the name of the CVS module to be brought up-to-date. +The default is to use the Aegis project name, +minus any branch numbers; +for example, given an Aegis project name of C<foo-cmd.0.1>, +the default CVS module name is C<foo-cmd>. + +=item -n + +No execute. Commands are printed (including a command for a final +commit of changes), but not executed. This is the default. + +=item -p proj + +Specifies the name of the Aegis project from which this change is taken. +The value of the C<AEGIS_PROJECT> environment variable +is used by default. + +=item -q + +Quiet. Commands are not printed. + +=item -u dir + +Use the already checked-out CVS tree that exists at C<dir> +for the checkins and commits. +The default is to use a separately-created temporary directory. + +=item -v + +Print the version string and exit. + +=item -x + +Execute the commands to bring the CVS repository up to date, +except for the final commit of the changes. Two or more +C<-x> options will cause the change to be committed. + +=item -X + +Execute the commands to bring the CVS repository up to date, +including the final commit of the changes. + +=back + +=head1 ENVIRONMENT VARIABLES + +=over 4 + +=item AE2CVS_FLAGS + +Specifies any options to be used to initialize +the C<ae2cvs> utility. +Options on the command line override these values. + +=back + +=head1 AUTHOR + +Steven Knight (knight at baldmt dot com) + +=head1 BUGS + +If errors occur during the execution of the Aegis or CVS commands, and +the -X option is used, a partial change (consisting of those files for +which the command(s) succeeded) will be committed. It would be safer to +generate code to detect the error and print a warning. + +When a file has been deleted in Aegis, the standard whiteout file can +cause a regex failure in this script. It doesn't necessarily happen all +the time, though, so this needs more investigation. + +=head1 TODO + +Add an explicit test for using ae2cvs in the Aegis +integrate_pass_notify_command field to support fully keeping a +repository in sync automatically. + +=head1 COPYRIGHT + +Copyright 2001, 2002, 2003, 2004, 2005 Steven Knight. + +=head1 SEE ALSO + +aegis(1), cvs(1) diff --git a/bin/calibrate.py b/bin/calibrate.py new file mode 100644 index 0000000..8ed2ece --- /dev/null +++ b/bin/calibrate.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# +# Copyright (c) 2009 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +from __future__ import division + +import optparse +import os +import re +import subprocess +import sys + +variable_re = re.compile('^VARIABLE: (.*)$', re.M) +elapsed_re = re.compile('^ELAPSED: (.*)$', re.M) + +def main(argv=None): + if argv is None: + argv = sys.argv + + parser = optparse.OptionParser(usage="calibrate.py [-h] [-p PACKAGE], [--min time] [--max time] timings/*/*-run.py") + parser.add_option('--min', type='float', default=9.5, + help="minimum acceptable execution time (default 9.5)") + parser.add_option('--max', type='float', default=10.00, + help="maximum acceptable execution time (default 10.00)") + parser.add_option('-p', '--package', type="string", + help="package type") + opts, args = parser.parse_args(argv[1:]) + + os.environ['TIMESCONS_CALIBRATE'] = '1' + + for arg in args: + if len(args) > 1: + print arg + ':' + + command = [sys.executable, 'runtest.py'] + if opts.package: + command.extend(['-p', opts.package]) + command.append(arg) + + run = 1 + good = 0 + while good < 3: + p = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = p.communicate()[0] + vm = variable_re.search(output) + em = elapsed_re.search(output) + try: + elapsed = float(em.group(1)) + except AttributeError: + print output + raise + print "run %3d: %7.3f: %s" % (run, elapsed, ' '.join(vm.groups())) + if opts.min < elapsed and elapsed < opts.max: + good += 1 + else: + good = 0 + for v in vm.groups(): + var, value = v.split('=', 1) + value = int((int(value) * opts.max) // elapsed) + os.environ[var] = str(value) + run += 1 + + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/caller-tree.py b/bin/caller-tree.py new file mode 100644 index 0000000..03c1616 --- /dev/null +++ b/bin/caller-tree.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# +# Quick script to process the *summary* output from SCons.Debug.caller() +# and print indented calling trees with call counts. +# +# The way to use this is to add something like the following to a function +# for which you want information about who calls it and how many times: +# +# from SCons.Debug import caller +# caller(0, 1, 2, 3, 4, 5) +# +# Each integer represents how many stack frames back SCons will go +# and capture the calling information, so in the above example it will +# capture the calls six levels up the stack in a central dictionary. +# +# At the end of any run where SCons.Debug.caller() is used, SCons will +# print a summary of the calls and counts that looks like the following: +# +# Callers of Node/__init__.py:629(calc_signature): +# 1 Node/__init__.py:683(calc_signature) +# Callers of Node/__init__.py:676(gen_binfo): +# 6 Node/FS.py:2035(current) +# 1 Node/__init__.py:722(get_bsig) +# +# If you cut-and-paste that summary output and feed it to this script +# on standard input, it will figure out how these entries hook up and +# print a calling tree for each one looking something like: +# +# Node/__init__.py:676(gen_binfo) +# Node/FS.py:2035(current) 6 +# Taskmaster.py:253(make_ready_current) 18 +# Script/Main.py:201(make_ready) 18 +# +# Note that you should *not* look at the call-count numbers in the right +# hand column as the actual number of times each line *was called by* +# the function on the next line. Rather, it's the *total* number +# of times each function was found in the call chain for any of the +# calls to SCons.Debug.caller(). If you're looking at more than one +# function at the same time, for example, their counts will intermix. +# So use this to get a *general* idea of who's calling what, not for +# fine-grained performance tuning. + +import sys + +class Entry(object): + def __init__(self, file_line_func): + self.file_line_func = file_line_func + self.called_by = [] + self.calls = [] + +AllCalls = {} + +def get_call(flf): + try: + e = AllCalls[flf] + except KeyError: + e = AllCalls[flf] = Entry(flf) + return e + +prefix = 'Callers of ' + +c = None +for line in sys.stdin.readlines(): + if line[0] == '#': + pass + elif line[:len(prefix)] == prefix: + c = get_call(line[len(prefix):-2]) + else: + num_calls, flf = line.strip().split() + e = get_call(flf) + c.called_by.append((e, num_calls)) + e.calls.append(c) + +stack = [] + +def print_entry(e, level, calls): + print '%-72s%6s' % ((' '*2*level) + e.file_line_func, calls) + if e in stack: + print (' '*2*(level+1))+'RECURSION' + print + elif e.called_by: + stack.append(e) + for c in e.called_by: + print_entry(c[0], level+1, c[1]) + stack.pop() + else: + print + +for e in [ e for e in AllCalls.values() if not e.calls ]: + print_entry(e, 0, '') + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/docs-create-example-outputs.py b/bin/docs-create-example-outputs.py new file mode 100644 index 0000000..30dc0ee --- /dev/null +++ b/bin/docs-create-example-outputs.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# +# Searches through the whole doc/user tree and creates +# all output files for the single examples. +# + +import os +import sys +import SConsExamples + +if __name__ == "__main__": + print "Checking whether all example names are unique..." + if SConsExamples.exampleNamesAreUnique(os.path.join('doc','user')): + print "OK" + else: + print "Not all example names and suffixes are unique! Please correct the errors listed above and try again." + sys.exit(0) + + SConsExamples.createAllExampleOutputs(os.path.join('doc','user')) diff --git a/bin/docs-update-generated.py b/bin/docs-update-generated.py new file mode 100644 index 0000000..66b22c0 --- /dev/null +++ b/bin/docs-update-generated.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# +# Searches through the whole source tree and updates +# the generated *.gen/*.mod files in the docs folder, keeping all +# documentation for the tools, builders and functions... +# as well as the entity declarations for them. +# Uses scons-proc.py under the hood... +# + +import os +import SConsDoc + +# Directory where all generated files are stored +gen_folder = os.path.join('doc','generated') + +def argpair(key): + """ Return the argument pair *.gen,*.mod for the given key. """ + arg = '%s,%s' % (os.path.join(gen_folder,'%s.gen' % key), + os.path.join(gen_folder,'%s.mod' % key)) + + return arg + +def generate_all(): + """ Scan for XML files in the src directory and call scons-proc.py + to generate the *.gen/*.mod files from it. + """ + flist = [] + for path, dirs, files in os.walk('src'): + for f in files: + if f.endswith('.xml'): + fpath = os.path.join(path, f) + if SConsDoc.isSConsXml(fpath): + flist.append(fpath) + + if flist: + # Does the destination folder exist + if not os.path.isdir(gen_folder): + try: + os.makedirs(gen_folder) + except: + print "Couldn't create destination folder %s! Exiting..." % gen_folder + return + # Call scons-proc.py + os.system('python %s -b %s -f %s -t %s -v %s %s' % + (os.path.join('bin','scons-proc.py'), + argpair('builders'), argpair('functions'), + argpair('tools'), argpair('variables'), ' '.join(flist))) + + +if __name__ == "__main__": + generate_all() diff --git a/bin/docs-validate.py b/bin/docs-validate.py new file mode 100644 index 0000000..c445c3f --- /dev/null +++ b/bin/docs-validate.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# +# Searches through the whole source tree and validates all +# documentation files against our own XSD in docs/xsd. +# + +import sys,os +import SConsDoc + +if __name__ == "__main__": + if len(sys.argv)>1: + if SConsDoc.validate_all_xml((sys.argv[1],)): + print "OK" + else: + print "Validation failed! Please correct the errors above and try again." + else: + if SConsDoc.validate_all_xml(['src', + os.path.join('doc','design'), + os.path.join('doc','developer'), + os.path.join('doc','man'), + os.path.join('doc','python10'), + os.path.join('doc','reference'), + os.path.join('doc','user') + ]): + print "OK" + else: + print "Validation failed! Please correct the errors above and try again." diff --git a/bin/files b/bin/files new file mode 100644 index 0000000..08b1caa --- /dev/null +++ b/bin/files @@ -0,0 +1,106 @@ +./SCons/Action.py +./SCons/Builder.py +./SCons/Conftest.py +./SCons/Debug.py +./SCons/Defaults.py +./SCons/Environment.py +./SCons/Errors.py +./SCons/Executor.py +./SCons/Job.py +./SCons/Node/Alias.py +./SCons/Node/FS.py +./SCons/Node/Python.py +./SCons/Node/__init__.py +./SCons/Options/__init__.py +./SCons/Options/BoolOption.py +./SCons/Options/EnumOption.py +./SCons/Options/ListOption.py +./SCons/Options/PackageOption.py +./SCons/Options/PathOption.py +./SCons/Platform/__init__.py +./SCons/Platform/aix.py +./SCons/Platform/cygwin.py +./SCons/Platform/hpux.py +./SCons/Platform/irix.py +./SCons/Platform/os2.py +./SCons/Platform/posix.py +./SCons/Platform/sunos.py +./SCons/Platform/win32.py +./SCons/Scanner/C.py +./SCons/Scanner/D.py +./SCons/Scanner/Fortran.py +./SCons/Scanner/IDL.py +./SCons/Scanner/Prog.py +./SCons/Scanner/__init__.py +./SCons/Script/SConscript.py +./SCons/Script/__init__.py +./SCons/Sig/MD5.py +./SCons/Sig/TimeStamp.py +./SCons/Sig/__init__.py +./SCons/Taskmaster.py +./SCons/Tool/__init__.py +./SCons/Tool/aixc++.py +./SCons/Tool/aixcc.py +./SCons/Tool/aixf77.py +./SCons/Tool/aixlink.py +./SCons/Tool/ar.py +./SCons/Tool/as.py +./SCons/Tool/bcc32.py +./SCons/Tool/c++.py +./SCons/Tool/cc.py +./SCons/Tool/CVS.py +./SCons/Tool/dmd.py +./SCons/Tool/default.py +./SCons/Tool/dvipdf.py +./SCons/Tool/dvips.py +./SCons/Tool/f77.py +./SCons/Tool/g++.py +./SCons/Tool/g77.py +./SCons/Tool/gas.py +./SCons/Tool/gcc.py +./SCons/Tool/gnulink.py +./SCons/Tool/hpc++.py +./SCons/Tool/hpcc.py +./SCons/Tool/hplink.py +./SCons/Tool/icc.py +./SCons/Tool/icl.py +./SCons/Tool/ifl.py +./SCons/Tool/ilink.py +./SCons/Tool/ilink32.py +./SCons/Tool/jar.py +./SCons/Tool/javac.py +./SCons/Tool/JavaCommon.py +./SCons/Tool/javah.py +./SCons/Tool/latex.py +./SCons/Tool/lex.py +./SCons/Tool/link.py +./SCons/Tool/m4.py +./SCons/Tool/masm.py +./SCons/Tool/midl.py +./SCons/Tool/mingw.py +./SCons/Tool/mslib.py +./SCons/Tool/mslink.py +./SCons/Tool/msvc.py +./SCons/Tool/msvs.py +./SCons/Tool/nasm.py +./SCons/Tool/pdflatex.py +./SCons/Tool/pdftex.py +./SCons/Tool/qt.py +./SCons/Tool/rmic.py +./SCons/Tool/sgiar.py +./SCons/Tool/sgic++.py +./SCons/Tool/sgicc.py +./SCons/Tool/sgilink.py +./SCons/Tool/sunar.py +./SCons/Tool/sunc++.py +./SCons/Tool/suncc.py +./SCons/Tool/sunlink.py +./SCons/Tool/swig.py +./SCons/Tool/tar.py +./SCons/Tool/tex.py +./SCons/Tool/tlib.py +./SCons/Tool/yacc.py +./SCons/Util.py +./SCons/Warnings.py +./SCons/__init__.py +./SCons/exitfuncs.py diff --git a/bin/import-test.py b/bin/import-test.py new file mode 100644 index 0000000..e37fe65 --- /dev/null +++ b/bin/import-test.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# +# Copyright (c) 2001 - 2014 The SCons Foundation +# +# tree2test.py - turn a directory tree into TestSCons code +# +# A quick script for importing directory hierarchies containing test +# cases that people supply (typically in a .zip or .tar.gz file) into a +# TestSCons.py script. No error checking or options yet, it just walks +# the first command-line argument (assumed to be the directory containing +# the test case) and spits out code looking like the following: +# +# test.subdir(['sub1'], +# ['sub1', 'sub2']) +# +# test.write(['sub1', 'file1'], """\ +# contents of file1 +# """) +# +# test.write(['sub1', 'sub2', 'file2'], """\ +# contents of file2 +# """) +# +# There's no massaging of contents, so any files that themselves contain +# """ triple-quotes will need to have their contents edited by hand. +# + +__revision__ = "bin/import-test.py 2014/09/27 12:51:43 garyo" + +import os.path +import sys + +directory = sys.argv[1] + +Top = None +TopPath = None + +class Dir(object): + def __init__(self, path): + self.path = path + self.entries = {} + def call_for_each_entry(self, func): + for name in sorted(self.entries.keys()): + func(name, self.entries[name]) + +def lookup(dirname): + global Top, TopPath + if not Top: + Top = Dir([]) + TopPath = dirname + os.sep + return Top + dirname = dirname.replace(TopPath, '') + dirs = dirname.split(os.sep) + t = Top + for d in dirs[:-1]: + t = t.entries[d] + node = t.entries[dirs[-1]] = Dir(dirs) + return node + +def collect_dirs(l, dir): + if dir.path: + l.append(dir.path) + def recurse(n, d): + if d: + collect_dirs(l, d) + dir.call_for_each_entry(recurse) + +def print_files(dir): + def print_a_file(n, d): + if not d: + l = dir.path + [n] + sys.stdout.write('\ntest.write(%s, """\\\n' % l) + p = os.path.join(directory, *l) + sys.stdout.write(open(p, 'r').read()) + sys.stdout.write('""")\n') + dir.call_for_each_entry(print_a_file) + + def recurse(n, d): + if d: + print_files(d) + dir.call_for_each_entry(recurse) + +for dirpath, dirnames, filenames in os.walk(directory): + dir = lookup(dirpath) + for f in fnames: + dir.entries[f] = None + +subdir_list = [] +collect_dirs(subdir_list, Top) +subdir_list = [ str(l) for l in subdir_list ] +sys.stdout.write('test.subdir(' + ',\n '.join(subdir_list) + ')\n') + +print_files(Top) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/install_python.py b/bin/install_python.py new file mode 100644 index 0000000..86807af --- /dev/null +++ b/bin/install_python.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# +# A script for unpacking and installing different historic versions of +# Python in a consistent manner for side-by-side development testing. +# +# This was written for a Linux system (specifically Ubuntu) but should +# be reasonably generic to any POSIX-style system with a /usr/local +# hierarchy. + +import getopt +import os +import shutil +import sys + +from Command import CommandRunner, Usage + +all_versions = [ + '2.3.7', + '2.4.5', + #'2.5.2', + '2.6', +] + +def main(argv=None): + if argv is None: + argv = sys.argv + + all = False + downloads_dir = 'Downloads' + downloads_url = 'http://www.python.org/ftp/python' + sudo = 'sudo' + prefix = '/usr/local' + + short_options = 'ad:hnp:q' + long_options = ['all', 'help', 'no-exec', 'prefix=', 'quiet'] + + helpstr = """\ +Usage: install_python.py [-ahnq] [-d DIR] [-p PREFIX] [VERSION ...] + + -a, --all Install all SCons versions. + -d DIR, --downloads=DIR Downloads directory. + -h, --help Print this help and exit + -n, --no-exec No execute, just print command lines + -p PREFIX, --prefix=PREFIX Installation prefix. + -q, --quiet Quiet, don't print command lines +""" + + try: + try: + opts, args = getopt.getopt(argv[1:], short_options, long_options) + except getopt.error, msg: + raise Usage(msg) + + for o, a in opts: + if o in ('-a', '--all'): + all = True + elif o in ('-d', '--downloads'): + downloads_dir = a + elif o in ('-h', '--help'): + print helpstr + sys.exit(0) + elif o in ('-n', '--no-exec'): + CommandRunner.execute = CommandRunner.do_not_execute + elif o in ('-p', '--prefix'): + prefix = a + elif o in ('-q', '--quiet'): + CommandRunner.display = CommandRunner.do_not_display + except Usage, err: + sys.stderr.write(str(err.msg) + '\n') + sys.stderr.write('use -h to get help\n') + return 2 + + if all: + if args: + msg = 'install-scons.py: -a and version arguments both specified' + sys.stderr.write(msg) + sys.exit(1) + + args = all_versions + + cmd = CommandRunner() + + for version in args: + python = 'Python-' + version + tar_gz = os.path.join(downloads_dir, python + '.tgz') + tar_gz_url = os.path.join(downloads_url, version, python + '.tgz') + + cmd.subst_dictionary(locals()) + + if not os.path.exists(tar_gz): + if not os.path.exists(downloads_dir): + cmd.run('mkdir %(downloads_dir)s') + cmd.run('wget -O %(tar_gz)s %(tar_gz_url)s') + + cmd.run('tar zxf %(tar_gz)s') + + cmd.run('cd %(python)s') + + cmd.run('./configure --prefix=%(prefix)s %(configureflags)s 2>&1 | tee configure.out') + cmd.run('make 2>&1 | tee make.out') + cmd.run('%(sudo)s make install') + + cmd.run('%(sudo)s rm -f %(prefix)s/bin/{idle,pydoc,python,python-config,smtpd.py}') + + cmd.run('cd ..') + + cmd.run((shutil.rmtree, python), 'rm -rf %(python)s') + +if __name__ == "__main__": + sys.exit(main()) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/install_scons.py b/bin/install_scons.py new file mode 100644 index 0000000..00129f6 --- /dev/null +++ b/bin/install_scons.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python +# +# A script for unpacking and installing different historic versions of +# SCons in a consistent manner for side-by-side development testing. +# +# This abstracts the changes we've made to the SCons setup.py scripts in +# different versions so that, no matter what version is specified, it ends +# up installing the necessary script(s) and library into version-specific +# names that won't interfere with other things. +# +# By default, we expect to extract the .tar.gz files from a Downloads +# subdirectory in the current directory. +# +# Note that this script cleans up after itself, removing the extracted +# directory in which we do the build. +# +# This was written for a Linux system (specifically Ubuntu) but should +# be reasonably generic to any POSIX-style system with a /usr/local +# hierarchy. + +import getopt +import os +import shutil +import sys +import tarfile +import urllib + +from Command import CommandRunner, Usage + +all_versions = [ + '0.01', + '0.02', + '0.03', + '0.04', + '0.05', + '0.06', + '0.07', + '0.08', + '0.09', + '0.10', + '0.11', + '0.12', + '0.13', + '0.14', + '0.90', + '0.91', + '0.92', + '0.93', + '0.94', + #'0.94.1', + '0.95', + #'0.95.1', + '0.96', + '0.96.1', + '0.96.90', + '0.96.91', + '0.96.92', + '0.96.93', + '0.96.94', + '0.96.95', + '0.96.96', + '0.97', + '0.97.0d20070809', + '0.97.0d20070918', + '0.97.0d20071212', + '0.98.0', + '0.98.1', + '0.98.2', + '0.98.3', + '0.98.4', + '0.98.5', + '1.0.0', + '1.0.0.d20080826', + '1.0.1', + '1.0.1.d20080915', + '1.0.1.d20081001', + '1.1.0', + '1.1.0.d20081104', + '1.1.0.d20081125', + '1.1.0.d20081207', + '1.2.0', + '1.2.0.d20090113', + '1.2.0.d20090223', + '1.2.0.d20090905', + '1.2.0.d20090919', + '1.2.0.d20091224', + '1.2.0.d20100117', + '1.2.0.d20100306', + '1.3.0', + '1.3.0.d20100404', + '1.3.0.d20100501', + '1.3.0.d20100523', + '1.3.0.d20100606', + '2.0.0.alpha.20100508', + '2.0.0.beta.20100531', + '2.0.0.beta.20100605', + '2.0.0.final.0', +] + +def main(argv=None): + if argv is None: + argv = sys.argv + + all = False + downloads_dir = 'Downloads' + downloads_url = 'http://downloads.sourceforge.net/scons' + if sys.platform == 'win32': + sudo = '' + prefix = sys.prefix + else: + sudo = 'sudo' + prefix = '/usr/local' + python = sys.executable + + short_options = 'ad:hnp:q' + long_options = ['all', 'help', 'no-exec', 'prefix=', 'quiet'] + + helpstr = """\ +Usage: install_scons.py [-ahnq] [-d DIR] [-p PREFIX] [VERSION ...] + + -a, --all Install all SCons versions. + -d DIR, --downloads=DIR Downloads directory. + -h, --help Print this help and exit + -n, --no-exec No execute, just print command lines + -p PREFIX, --prefix=PREFIX Installation prefix. + -q, --quiet Quiet, don't print command lines +""" + + try: + try: + opts, args = getopt.getopt(argv[1:], short_options, long_options) + except getopt.error, msg: + raise Usage(msg) + + for o, a in opts: + if o in ('-a', '--all'): + all = True + elif o in ('-d', '--downloads'): + downloads_dir = a + elif o in ('-h', '--help'): + print helpstr + sys.exit(0) + elif o in ('-n', '--no-exec'): + CommandRunner.execute = CommandRunner.do_not_execute + elif o in ('-p', '--prefix'): + prefix = a + elif o in ('-q', '--quiet'): + CommandRunner.display = CommandRunner.do_not_display + except Usage, err: + sys.stderr.write(str(err.msg) + '\n') + sys.stderr.write('use -h to get help\n') + return 2 + + if all: + if args: + msg = 'install-scons.py: -a and version arguments both specified' + sys.stderr.write(msg) + sys.exit(1) + + args = all_versions + + cmd = CommandRunner() + + for version in args: + scons = 'scons-' + version + tar_gz = os.path.join(downloads_dir, scons + '.tar.gz') + tar_gz_url = "%s/%s.tar.gz" % (downloads_url, scons) + + cmd.subst_dictionary(locals()) + + if not os.path.exists(tar_gz): + if not os.path.exists(downloads_dir): + cmd.run('mkdir %(downloads_dir)s') + cmd.run((urllib.urlretrieve, tar_gz_url, tar_gz), + 'wget -O %(tar_gz)s %(tar_gz_url)s') + + def extract(tar_gz): + tarfile.open(tar_gz, "r:gz").extractall() + cmd.run((extract, tar_gz), 'tar zxf %(tar_gz)s') + + cmd.run('cd %(scons)s') + + if version in ('0.01', '0.02', '0.03', '0.04', '0.05', + '0.06', '0.07', '0.08', '0.09', '0.10'): + + # 0.01 through 0.10 install /usr/local/bin/scons and + # /usr/local/lib/scons. The "scons" script knows how to + # look up the library in a version-specific directory, but + # we have to move both it and the library directory into + # the right version-specific name by hand. + cmd.run('%(python)s setup.py build') + cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s') + cmd.run('%(sudo)s mv %(prefix)s/bin/scons %(prefix)s/bin/scons-%(version)s') + cmd.run('%(sudo)s mv %(prefix)s/lib/scons %(prefix)s/lib/scons-%(version)s') + + elif version in ('0.11', '0.12', '0.13', '0.14', '0.90'): + + # 0.11 through 0.90 install /usr/local/bin/scons and + # /usr/local/lib/scons-%(version)s. We just need to move + # the script to a version-specific name. + cmd.run('%(python)s setup.py build') + cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s') + cmd.run('%(sudo)s mv %(prefix)s/bin/scons %(prefix)s/bin/scons-%(version)s') + + elif version in ('0.91', '0.92', '0.93', + '0.94', '0.94.1', + '0.95', '0.95.1', + '0.96', '0.96.1', '0.96.90'): + + # 0.91 through 0.96.90 install /usr/local/bin/scons, + # /usr/local/bin/sconsign and /usr/local/lib/scons-%(version)s. + # We need to move both scripts to version-specific names. + cmd.run('%(python)s setup.py build') + cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s') + cmd.run('%(sudo)s mv %(prefix)s/bin/scons %(prefix)s/bin/scons-%(version)s') + cmd.run('%(sudo)s mv %(prefix)s/bin/sconsign %(prefix)s/bin/sconsign-%(version)s') + lib_scons = os.path.join(prefix, 'lib', 'scons') + if os.path.isdir(lib_scons): + cmd.run('%(sudo)s mv %(prefix)s/lib/scons %(prefix)s/lib/scons-%(version)s') + + else: + + # Versions from 0.96.91 and later support what we want + # with a --no-scons-script option. + cmd.run('%(python)s setup.py build') + cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s --no-scons-script') + + cmd.run('cd ..') + + cmd.run((shutil.rmtree, scons), 'rm -rf %(scons)s') + +if __name__ == "__main__": + sys.exit(main()) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/linecount.py b/bin/linecount.py new file mode 100644 index 0000000..7c7e561 --- /dev/null +++ b/bin/linecount.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# +# Copyright (c) 2001 - 2014 The SCons Foundation +# +# Count statistics about SCons test and source files. This must be run +# against a fully-populated tree (for example, one that's been freshly +# checked out). +# +# A test file is anything under the src/ directory that begins with +# 'test_' or ends in 'Tests.py', or anything under the test/ directory +# that ends in '.py'. Note that runtest.py script does *not*, by default, +# consider the files that begin with 'test_' to be tests, because they're +# tests of SCons packaging and installation, not functional tests of +# SCons code. +# +# A source file is anything under the src/engine/ or src/script/ +# directories that ends in '.py' but does NOT begin with 'test_' +# or end in 'Tests.py'. +# +# We report the number of tests and sources, the total number of lines +# in each category, the number of non-blank lines, and the number of +# non-comment lines. The last figure (non-comment) lines is the most +# interesting one for most purposes. +from __future__ import division + +__revision__ = "bin/linecount.py 2014/09/27 12:51:43 garyo" + +import os.path + +fmt = "%-16s %5s %7s %9s %11s %11s" + +class Collection(object): + def __init__(self, name, files=None, pred=None): + self._name = name + if files is None: + files = [] + self.files = files + if pred is None: + pred = lambda x: True + self.pred = pred + def __call__(self, fname): + return self.pred(fname) + def __len__(self): + return len(self.files) + def collect(self, directory): + for dirpath, dirnames, filenames in os.walk(directory): + try: dirnames.remove('.svn') + except ValueError: pass + self.files.extend([ os.path.join(dirpath, f) + for f in filenames if self.pred(f) ]) + def lines(self): + try: + return self._lines + except AttributeError: + self._lines = lines = [] + for file in self.files: + file_lines = open(file).readlines() + lines.extend([s.lstrip() for s in file_lines]) + return lines + def non_blank(self): + return [s for s in self.lines() if s != ''] + def non_comment(self): + return [s for s in self.lines() if s == '' or s[0] != '#'] + def non_blank_non_comment(self): + return [s for s in self.lines() if s != '' and s[0] != '#'] + def printables(self): + return (self._name + ':', + len(self.files), + len(self.lines()), + len(self.non_blank()), + len(self.non_comment()), + len(self.non_blank_non_comment())) + +def is_Tests_py(x): + return x[-8:] == 'Tests.py' +def is_test_(x): + return x[:5] == 'test_' +def is_python(x): + return x[-3:] == '.py' +def is_source(x): + return is_python(x) and not is_Tests_py(x) and not is_test_(x) + +src_Tests_py_tests = Collection('src/ *Tests.py', pred=is_Tests_py) +src_test_tests = Collection('src/ test_*.py', pred=is_test_) +test_tests = Collection('test/ tests', pred=is_python) +sources = Collection('sources', pred=is_source) + +src_Tests_py_tests.collect('src') +src_test_tests.collect('src') +test_tests.collect('test') +sources.collect('src/engine') +sources.collect('src/script') + +src_tests = Collection('src/ tests', src_Tests_py_tests.files + + src_test_tests.files) +all_tests = Collection('all tests', src_tests.files + test_tests.files) + +def ratio(over, under): + return "%.2f" % (float(len(over)) / float(len(under))) + +print fmt % ('', '', '', '', '', 'non-blank') +print fmt % ('', 'files', 'lines', 'non-blank', 'non-comment', 'non-comment') +print +print fmt % src_Tests_py_tests.printables() +print fmt % src_test_tests.printables() +print +print fmt % src_tests.printables() +print fmt % test_tests.printables() +print +print fmt % all_tests.printables() +print fmt % sources.printables() +print +print fmt % ('ratio:', + ratio(all_tests, sources), + ratio(all_tests.lines(), sources.lines()), + ratio(all_tests.non_blank(), sources.non_blank()), + ratio(all_tests.non_comment(), sources.non_comment()), + ratio(all_tests.non_blank_non_comment(), + sources.non_blank_non_comment()) + ) diff --git a/bin/makedocs b/bin/makedocs new file mode 100644 index 0000000..2278a97 --- /dev/null +++ b/bin/makedocs @@ -0,0 +1,18 @@ +#! /bin/sh + +# This script uses HappyDoc to create the HTML class documentation for +# SCons. It must be run from the src/engine directory. + +base=`basename $PWD` +if [ "$base" != "engine" ]; then + echo "You must run this script from the engine directory." + exit +fi + +DEVDIR=../../doc/developer +if [ ! -d $DEVDIR ]; then + mkdir $DEVDIR +fi + +SRCFILE=../../bin/files +happydoc -d $DEVDIR `cat $SRCFILE` diff --git a/bin/memlogs.py b/bin/memlogs.py new file mode 100644 index 0000000..9d957c9 --- /dev/null +++ b/bin/memlogs.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# +# Copyright (c) 2005 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import getopt +import sys + +filenames = sys.argv[1:] + +if not filenames: + print """Usage: memlogs.py file [...] + +Summarizes the --debug=memory numbers from one or more build logs. +""" + sys.exit(0) + +fmt = "%12s %12s %12s %12s %s" + +print fmt % ("pre-read", "post-read", "pre-build", "post-build", "") + +for fname in sys.argv[1:]: + lines = [l for l in open(fname).readlines() if l[:7] == 'Memory '] + t = tuple([l.split()[-1] for l in lines]) + (fname,) + print fmt % t + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/memoicmp.py b/bin/memoicmp.py new file mode 100644 index 0000000..812af66 --- /dev/null +++ b/bin/memoicmp.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# A script to compare the --debug=memoizer output found in +# two different files. + +import sys + +def memoize_output(fname): + mout = {} + #lines=filter(lambda words: + # len(words) == 5 and + # words[1] == 'hits' and words[3] == 'misses', + # map(string.split, open(fname,'r').readlines())) + #for line in lines: + # mout[line[-1]] = ( int(line[0]), int(line[2]) ) + for line in open(fname,'r').readlines(): + words = line.split() + if len(words) == 5 and words[1] == 'hits' and words[3] == 'misses': + mout[words[-1]] = ( int(words[0]), int(words[2]) ) + return mout + +def memoize_cmp(filea, fileb): + ma = memoize_output(filea) + mb = memoize_output(fileb) + + print 'All output: %s / %s [delta]'%(filea, fileb) + print '----------HITS---------- ---------MISSES---------' + cfmt='%7d/%-7d [%d]' + ma_o = [] + mb_o = [] + mab = [] + for k in ma.keys(): + if k in mb.keys(): + if k not in mab: + mab.append(k) + else: + ma_o.append(k) + for k in mb.keys(): + if k in ma.keys(): + if k not in mab: + mab.append(k) + else: + mb_o.append(k) + + mab.sort() + ma_o.sort() + mb_o.sort() + + for k in mab: + hits = cfmt%(ma[k][0], mb[k][0], mb[k][0]-ma[k][0]) + miss = cfmt%(ma[k][1], mb[k][1], mb[k][1]-ma[k][1]) + print '%-24s %-24s %s'%(hits, miss, k) + + for k in ma_o: + hits = '%7d/ --'%(ma[k][0]) + miss = '%7d/ --'%(ma[k][1]) + print '%-24s %-24s %s'%(hits, miss, k) + + for k in mb_o: + hits = ' -- /%-7d'%(mb[k][0]) + miss = ' -- /%-7d'%(mb[k][1]) + print '%-24s %-24s %s'%(hits, miss, k) + + print '-'*(24+24+1+20) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print """Usage: %s file1 file2 + +Compares --debug=memomize output from file1 against file2."""%sys.argv[0] + sys.exit(1) + + memoize_cmp(sys.argv[1], sys.argv[2]) + sys.exit(0) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/objcounts.py b/bin/objcounts.py new file mode 100644 index 0000000..0662012 --- /dev/null +++ b/bin/objcounts.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# +# Copyright (c) 2005 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import re +import sys + +filenames = sys.argv[1:] + +if len(sys.argv) != 3: + print """Usage: objcounts.py file1 file2 + +Compare the --debug=object counts from two build logs. +""" + sys.exit(0) + +def fetch_counts(fname): + contents = open(fname).read() + m = re.search('\nObject counts:(\n\s[^\n]*)*', contents, re.S) + lines = m.group().split('\n') + list = [l.split() for l in lines if re.match('\s+\d', l)] + d = {} + for l in list: + d[l[-1]] = list(map(int, l[:-1])) + return d + +c1 = fetch_counts(sys.argv[1]) +c2 = fetch_counts(sys.argv[2]) + +common = {} +for k in c1.keys(): + try: + common[k] = (c1[k], c2[k]) + except KeyError: + # Transition: we added the module to the names of a bunch of + # the logged objects. Assume that c1 might be from an older log + # without the modules in the names, and look for an equivalent + # in c2. + if not '.' in k: + s = '.'+k + l = len(s) + for k2 in c2.keys(): + if k2[-l:] == s: + common[k2] = (c1[k], c2[k2]) + del c1[k] + del c2[k2] + break + else: + del c1[k] + del c2[k] + +def diffstr(c1, c2): + try: + d = c2 - c1 + except TypeError: + d = '' + else: + if d: + d = '[%+d]' % d + else: + d = '' + return " %5s/%-5s %-8s" % (c1, c2, d) + +def printline(c1, c2, classname): + print \ + diffstr(c1[2], c2[2]) + \ + diffstr(c1[3], c2[3]) + \ + ' ' + classname + +for k in sorted(common.keys()): + c = common[k] + printline(c[0], c[1], k) + +for k in sorted(list(c1.keys())): + printline(c1[k], ['--']*4, k) + +for k in sorted(list(c2.keys())): + printline(['--']*4, c2[k], k) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/restore.sh b/bin/restore.sh new file mode 100644 index 0000000..12a5026 --- /dev/null +++ b/bin/restore.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env sh +# +# Simple hack script to restore __revision__, __COPYRIGHT_, 2.3.4 +# and other similar variables to what gets checked in to source. This +# comes in handy when people send in diffs based on the released source. +# + +if test "X$*" = "X"; then + DIRS="src test" +else + DIRS="$*" +fi + +SEPARATOR="================================================================================" + +header() { + arg_space="$1 " + dots=`echo "$arg_space" | sed 's/./\./g'` + echo "$SEPARATOR" | sed "s;$dots;$arg_space;" +} + +for i in `find $DIRS -name '*.py'`; do + header $i + ed $i <<EOF +g/Copyright (c) 2001.*SCons Foundation/s//Copyright (c) 2001 - 2014 The SCons Foundation/p +w +/^__revision__ = /s/= .*/= "bin/restore.sh 2014/09/27 12:51:43 garyo"/p +w +q +EOF +done + +for i in `find $DIRS -name 'scons.bat'`; do + header $i + ed $i <<EOF +g/Copyright (c) 2001.*SCons Foundation/s//Copyright (c) 2001 - 2014 The SCons Foundation/p +w +/^@REM src\/script\/scons.bat/s/@REM .* knight/@REM bin/restore.sh 2014/09/27 12:51:43 garyo/p +w +q +EOF +done + +for i in `find $DIRS -name '__init__.py' -o -name 'scons.py' -o -name 'sconsign.py'`; do + header $i + ed $i <<EOF +/^__version__ = /s/= .*/= "2.3.4"/p +w +/^__build__ = /s/= .*/= ""/p +w +/^__buildsys__ = /s/= .*/= "lubuntu"/p +w +/^__date__ = /s/= .*/= "2014/09/27 12:51:43"/p +w +/^__developer__ = /s/= .*/= "garyo"/p +w +q +EOF +done + +for i in `find $DIRS -name 'setup.py'`; do + header $i + ed $i <<EOF +/^ *version = /s/= .*/= "2.3.4",/p +w +q +EOF +done + +for i in `find $DIRS -name '*.txt'`; do + header $i + ed $i <<EOF +g/Copyright (c) 2001.*SCons Foundation/s//Copyright (c) 2001 - 2014 The SCons Foundation/p +w +/# [^ ]* 0.96.[CD][0-9]* [0-9\/]* [0-9:]* knight$/s/.*/# bin/restore.sh 2014/09/27 12:51:43 garyo/p +w +/Version [0-9][0-9]*\.[0-9][0-9]*/s//Version 2.3.4/p +w +q +EOF +done + +for i in `find $DIRS -name '*.xml'`; do + header $i + ed $i <<EOF +g/Copyright (c) 2001.*SCons Foundation/s//Copyright (c) 2001 - 2014 The SCons Foundation/p +w +q +EOF +done diff --git a/bin/rsync-sourceforge b/bin/rsync-sourceforge new file mode 100644 index 0000000..de44e3b --- /dev/null +++ b/bin/rsync-sourceforge @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Sync this directory tree with sourceforge. +# +# Cribbed and modified from Peter Miller's same-named script in +# /home/groups/a/ae/aegis/aegis at SourceForge. +# +# Guide to what this does with rsync: +# +# --rsh=ssh use ssh for the transfer +# -l copy symlinks as symlinks +# -p preserve permissions +# -r recursive +# -t preserve times +# -z compress data +# --stats file transfer statistics +# --exclude exclude files matching the pattern +# --delete delete files that don't exist locally +# --delete-excluded delete files that match the --exclude patterns +# --progress show progress during the transfer +# -v verbose +# +LOCAL=/home/scons/scons +REMOTE=/home/groups/s/sc/scons/scons +/usr/bin/rsync --rsh=ssh -l -p -r -t -z --stats \ + --exclude build \ + --exclude "*,D" \ + --exclude "*.pyc" \ + --exclude aegis.log \ + --delete --delete-excluded \ + --progress -v \ + ${LOCAL}/. scons.sourceforge.net:${REMOTE}/. diff --git a/bin/scons-cdist b/bin/scons-cdist new file mode 100644 index 0000000..58b1bae --- /dev/null +++ b/bin/scons-cdist @@ -0,0 +1,272 @@ +#!/bin/sh +# +# Copyright (c) 2005 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +PROG=`basename $0` +NOARGFLAGS="afhlnqrstz" +ARGFLAGS="p:" +ALLFLAGS="${NOARGFLAGS}${ARGFLAGS}" +USAGE="Usage: ${PROG} [-${NOARGFLAGS}] [-p project] change" + +HELP="$USAGE + + -a Update the latest Aegis baseline (aedist) file. + -f Force update, skipping up-front sanity check. + -h Print this help message and exit. + -l Update the local CVS repository. + -n Don't execute, just echo commands. + -p project Set the Aegis project. + -q Quiet, don't print commands before executing them. + -r Rsync the Aegis repository to SourceForge. + -s Update the sourceforge.net CVS repository. + -t Update the tigris.org CVS repository. + -z Update the latest .tar.gz and .zip files. +" + +DO="" +PRINT="echo" +EXECUTE="eval" +SANITY_CHECK="yes" + +while getopts $ALLFLAGS FLAG; do + case $FLAG in + a | l | r | s | t | z ) + DO="${DO}${FLAG}" + ;; + f ) + SANITY_CHECK="no" + ;; + h ) + echo "${HELP}" + exit 0 + ;; + n ) + EXECUTE=":" + ;; + p ) + AEGIS_PROJECT="${OPTARG}" + ;; + q ) + PRINT=":" + ;; + * ) + echo "FLAG = ${FLAG}" >&2 + echo "${USAGE}" >&2 + exit 1 + ;; + esac +done + +shift `expr ${OPTIND} - 1` + +if test "X$1" = "X"; then + echo "${USAGE}" >&2 + exit 1 +fi + +if test "X${AEGIS_PROJECT}" = "X"; then + echo "$PROG: No AEGIS_PROJECT set." >&2 + echo "${USAGE}" >&2 + exit 1 +fi + +if test "X$DO" = "X"; then + DO="alrstz" +fi + +cmd() +{ + $PRINT "$*" + $EXECUTE "$*" +} + +CHANGE=$1 + +if test "X${SANITY_CHECK}" = "Xyes"; then + SCM="cvs" + SCMROOT="/home/scons/CVSROOT/scons" + DELTA=`aegis -l -ter cd ${CHANGE} | sed -n 's/.*, Delta \([0-9]*\)\./\1/p'` + if test "x${DELTA}" = "x"; then + echo "${PROG}: Could not find delta for change ${CHANGE}." >&2 + echo "Has this finished integrating? Change ${CHANGE} not distributed." >&2 + exit 1 + fi + PREV_DELTA=`expr ${DELTA} - 1` + COMMAND="scons-scmcheck -D ${PREV_DELTA} -d q -p ${AEGIS_PROJECT} -s ${SCM} ${SCMROOT}" + $PRINT "${COMMAND}" + OUTPUT=`${COMMAND}` + if test "X${OUTPUT}" != "X"; then + echo "${PROG}: ${SCMROOT} is not up to date:" >&2 + echo "${OUTPUT}" >& 2 + echo "Did you skip any changes? Change ${CHANGE} not distributed." >&2 + exit 1 + fi +fi + +if test X$EXECUTE != "X:" -a "X$SSH_AGENT_PID" = "X"; then + eval `ssh-agent` + ssh-add + trap 'eval `ssh-agent -k`; exit' 0 1 2 3 15 +fi + +cd + +BASELINE=`aesub -p ${AEGIS_PROJECT} -c ${CHANGE} '${Project trunk_name}'` + +TMPBLAE="/tmp/${BASELINE}.ae" +TMPCAE="/tmp/${AEGIS_PROJECT}.C${CHANGE}.ae" + +# Original values for SourceForge. +#SFLOGIN="stevenknight" +#SFHOST="scons.sourceforge.net" +#SFDEST="/home/groups/s/sc/scons/htdocs" + +SCONSLOGIN="scons" +SCONSHOST="manam.pair.com" +#SCONSDEST="public_html/production" +SCONSDEST="public_ftp" + +# +# Copy the baseline .ae to the constant location on SourceForge. +# +case "${DO}" in +*a* ) + cmd "aedist -s -bl -p ${AEGIS_PROJECT} > ${TMPBLAE}" + cmd "scp ${TMPBLAE} ${SCONSLOGIN}@${SCONSHOST}:${SCONSDEST}/${BASELINE}.ae" + cmd "rm ${TMPBLAE}" + ;; +esac + +# +# Copy the latest .tar.gz and .zip files to the constant location on +# SourceForge. +# +case "${DO}" in +*z* ) + BUILD_DIST=`aegis -p ${AEGIS_PROJECT} -cd -bl`/build/dist + SCONS_SRC_TAR_GZ=`echo ${AEGIS_PROJECT} | sed 's/scons./scons-src-/'`*.tar.gz + SCONS_SRC_ZIP=`echo ${AEGIS_PROJECT} | sed 's/scons./scons-src-/'`*.zip + cmd "scp ${BUILD_DIST}/${SCONS_SRC_TAR_GZ} ${SCONSLOGIN}@${SCONSHOST}:${SCONSDEST}/scons-src-latest.tar.gz" + cmd "scp ${BUILD_DIST}/${SCONS_SRC_ZIP} ${SCONSLOGIN}@${SCONSHOST}:${SCONSDEST}/scons-src-latest.zip" +esac + +# +# Sync Aegis tree with SourceForge. +# +# Cribbed and modified from Peter Miller's same-named script in +# /home/groups/a/ae/aegis/aegis at SourceForge. +# +# Guide to what this does with rsync: +# +# --rsh=ssh use ssh for the transfer +# -l copy symlinks as symlinks +# -p preserve permissions +# -r recursive +# -t preserve times +# -z compress data +# --stats file transfer statistics +# --exclude exclude files matching the pattern +# --delete delete files that don't exist locally +# --delete-excluded delete files that match the --exclude patterns +# --progress show progress during the transfer +# -v verbose +# +# We no longer use the --stats option. +# +case "${DO}" in +*r* ) + LOCAL=/home/scons/scons + REMOTE=/home/groups/s/sc/scons/scons + cmd "/usr/bin/rsync --rsh='ssh -l stevenknight' \ + -l -p -r -t -z \ + --exclude build \ + --exclude '*,D' \ + --exclude '*.pyc' \ + --exclude aegis.log \ + --exclude '.sconsign*' \ + --delete --delete-excluded \ + --progress -v \ + ${LOCAL}/. scons.sourceforge.net:${REMOTE}/." + ;; +esac + +# +# Sync the CVS tree with the local repository. +# +case "${DO}" in +*l* ) + ( + export CVSROOT=/home/scons/CVSROOT/scons + #cmd "ae2cvs -X -aegis -p ${AEGIS_PROJECT} -c ${CHANGE} -u $HOME/SCons/baldmt.com/scons" + cmd "ae-cvs-ci ${AEGIS_PROJECT} ${CHANGE}" + ) + ;; +esac + +# +# Sync the Subversion tree with Tigris.org. +# +case "${DO}" in +*t* ) + ( + SVN=http://scons.tigris.org/svn/scons + case ${AEGIS_PROJECT} in + scons.0.96 ) + SVN_URL=${SVN}/branches/core + ;; + scons.0.96.513 ) + SVN_URL=${SVN}/branches/sigrefactor + ;; + * ) + echo "$PROG: Don't know SVN branch for '${AEGIS_PROJECT}'" >&2 + exit 1 + ;; + esac + SVN_CO_FLAGS="--username stevenknight" + #cmd "ae2cvs -X -aegis -p ${AEGIS_PROJECT} -c ${CHANGE} -u $HOME/SCons/tigris.org/scons" + cmd "ae-svn-ci ${AEGIS_PROJECT} ${CHANGE} ${SVN_URL} ${SVN_CO_FLAGS}" + ) + ;; +esac + +# +# Sync the CVS tree with SourceForge. +# +case "${DO}" in +*s* ) + ( + export CVS_RSH=ssh + export CVSROOT=:ext:stevenknight@scons.cvs.sourceforge.net:/cvsroot/scons + #cmd "ae2cvs -X -aegis -p ${AEGIS_PROJECT} -c ${CHANGE} -u $HOME/SCons/sourceforge.net/scons" + cmd "ae-cvs-ci ${AEGIS_PROJECT} ${CHANGE}" + ) + ;; +esac + +# +# Send the change .ae to the scons-aedist mailing list +# +# The subject requires editing by hand... +# +#aedist -s -p ${AEGIS_PROJECT} ${CHANGE} > ${TMPCAE} +#aegis -l -p ${AEGIS_PROJECT} -c ${CHANGE} cd | +# pine -attach_and_delete ${TMPCAE} scons-aedist@lists.sourceforge.net diff --git a/bin/scons-diff.py b/bin/scons-diff.py new file mode 100644 index 0000000..1e4bca0 --- /dev/null +++ b/bin/scons-diff.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python +# +# scons-diff.py - diff-like utility for comparing SCons trees +# +# This supports most common diff options (with some quirks, like you can't +# just say -c and have it use a default value), but canonicalizes the +# various version strings within the file like __revision__, __build__, +# etc. so that you can diff trees without having to ignore changes in +# version lines. +# + +import difflib +import getopt +import os.path +import re +import sys + +Usage = """\ +Usage: scons-diff.py [OPTIONS] dir1 dir2 +Options: + -c NUM, --context=NUM Print NUM lines of copied context. + -h, --help Print this message and exit. + -n Don't canonicalize SCons lines. + -q, --quiet Print only whether files differ. + -r, --recursive Recursively compare found subdirectories. + -s Report when two files are the same. + -u NUM, --unified=NUM Print NUM lines of unified context. +""" + +opts, args = getopt.getopt(sys.argv[1:], + 'c:dhnqrsu:', + ['context=', 'help', 'recursive', 'unified=']) + +diff_type = None +edit_type = None +context = 2 +recursive = False +report_same = False +diff_options = [] + +def diff_line(left, right): + if diff_options: + opts = ' ' + ' '.join(diff_options) + else: + opts = '' + print 'diff%s %s %s' % (opts, left, right) + +for o, a in opts: + if o in ('-c', '-u'): + diff_type = o + context = int(a) + diff_options.append(o) + elif o in ('-h', '--help'): + print Usage + sys.exit(0) + elif o in ('-n'): + diff_options.append(o) + edit_type = o + elif o in ('-q'): + diff_type = o + diff_line = lambda l, r: None + elif o in ('-r', '--recursive'): + recursive = True + diff_options.append(o) + elif o in ('-s'): + report_same = True + +try: + left, right = args +except ValueError: + sys.stderr.write(Usage) + sys.exit(1) + +def quiet_diff(a, b, fromfile='', tofile='', + fromfiledate='', tofiledate='', n=3, lineterm='\n'): + """ + A function with the same calling signature as difflib.context_diff + (diff -c) and difflib.unified_diff (diff -u) but which prints + output like the simple, unadorned 'diff" command. + """ + if a == b: + return [] + else: + return ['Files %s and %s differ\n' % (fromfile, tofile)] + +def simple_diff(a, b, fromfile='', tofile='', + fromfiledate='', tofiledate='', n=3, lineterm='\n'): + """ + A function with the same calling signature as difflib.context_diff + (diff -c) and difflib.unified_diff (diff -u) but which prints + output like the simple, unadorned 'diff" command. + """ + sm = difflib.SequenceMatcher(None, a, b) + def comma(x1, x2): + return x1+1 == x2 and str(x2) or '%s,%s' % (x1+1, x2) + result = [] + for op, a1, a2, b1, b2 in sm.get_opcodes(): + if op == 'delete': + result.append("%sd%d\n" % (comma(a1, a2), b1)) + result.extend(['< ' + l for l in a[a1:a2]]) + elif op == 'insert': + result.append("%da%s\n" % (a1, comma(b1, b2))) + result.extend(['> ' + l for l in b[b1:b2]]) + elif op == 'replace': + result.append("%sc%s\n" % (comma(a1, a2), comma(b1, b2))) + result.extend(['< ' + l for l in a[a1:a2]]) + result.append('---\n') + result.extend(['> ' + l for l in b[b1:b2]]) + return result + +diff_map = { + '-c' : difflib.context_diff, + '-q' : quiet_diff, + '-u' : difflib.unified_diff, +} + +diff_function = diff_map.get(diff_type, simple_diff) + +baseline_re = re.compile('(# |@REM )/home/\S+/baseline/') +comment_rev_re = re.compile('(# |@REM )(\S+) 0.96.[CD]\d+ \S+ \S+( knight)') +revision_re = re.compile('__revision__ = "[^"]*"') +build_re = re.compile('__build__ = "[^"]*"') +date_re = re.compile('__date__ = "[^"]*"') + +def lines_read(file): + return open(file).readlines() + +def lines_massage(file): + text = open(file).read() + text = baseline_re.sub('\\1', text) + text = comment_rev_re.sub('\\1\\2\\3', text) + text = revision_re.sub('__revision__ = "bin/scons-diff.py"', text) + text = build_re.sub('__build__ = "0.96.92.DXXX"', text) + text = date_re.sub('__date__ = "2006/08/25 02:59:00"', text) + return text.splitlines(1) + +lines_map = { + '-n' : lines_read, +} + +lines_function = lines_map.get(edit_type, lines_massage) + +def do_diff(left, right, diff_subdirs): + if os.path.isfile(left) and os.path.isfile(right): + diff_file(left, right) + elif not os.path.isdir(left): + diff_file(left, os.path.join(right, os.path.split(left)[1])) + elif not os.path.isdir(right): + diff_file(os.path.join(left, os.path.split(right)[1]), right) + elif diff_subdirs: + diff_dir(left, right) + +def diff_file(left, right): + l = lines_function(left) + r = lines_function(right) + d = diff_function(l, r, left, right, context) + try: + text = ''.join(d) + except IndexError: + sys.stderr.write('IndexError diffing %s and %s\n' % (left, right)) + else: + if text: + diff_line(left, right) + print text, + elif report_same: + print 'Files %s and %s are identical' % (left, right) + +def diff_dir(left, right): + llist = os.listdir(left) + rlist = os.listdir(right) + u = {} + for l in llist: + u[l] = 1 + for r in rlist: + u[r] = 1 + for x in sorted([ x for x in u.keys() if x[-4:] != '.pyc' ]): + if x in llist: + if x in rlist: + do_diff(os.path.join(left, x), + os.path.join(right, x), + recursive) + else: + print 'Only in %s: %s' % (left, x) + else: + print 'Only in %s: %s' % (right, x) + +do_diff(left, right, True) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/scons-proc.py b/bin/scons-proc.py new file mode 100644 index 0000000..9567db8 --- /dev/null +++ b/bin/scons-proc.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python +# +# Process a list of Python and/or XML files containing SCons documentation. +# +# This script creates formatted lists of the Builders, functions, Tools +# or construction variables documented in the specified XML files. +# +# Depending on the options, the lists are output in either +# DocBook-formatted generated XML files containing the summary text +# and/or .mod files containing the ENTITY definitions for each item. +# +import getopt +import os +import re +import string +import sys +try: + from io import StringIO # usable as of 2.6; takes unicode only +except ImportError: + # No 'io' module or no StringIO in io + exec('from cStringIO import StringIO') + +import SConsDoc +from SConsDoc import tf as stf + +base_sys_path = [os.getcwd() + '/build/test-tar-gz/lib/scons'] + sys.path + +helpstr = """\ +Usage: scons-proc.py [-b file(s)] [-f file(s)] [-t file(s)] [-v file(s)] + [infile ...] +Options: + -b file(s) dump builder information to the specified file(s) + -f file(s) dump function information to the specified file(s) + -t file(s) dump tool information to the specified file(s) + -v file(s) dump variable information to the specified file(s) + + Regard that each -[btv] argument is a pair of + comma-separated .gen,.mod file names. + +""" + +opts, args = getopt.getopt(sys.argv[1:], + "b:f:ht:v:", + ['builders=', 'help', + 'tools=', 'variables=']) + +buildersfiles = None +functionsfiles = None +toolsfiles = None +variablesfiles = None + +for o, a in opts: + if o in ['-b', '--builders']: + buildersfiles = a + elif o in ['-f', '--functions']: + functionsfiles = a + elif o in ['-h', '--help']: + sys.stdout.write(helpstr) + sys.exit(0) + elif o in ['-t', '--tools']: + toolsfiles = a + elif o in ['-v', '--variables']: + variablesfiles = a + +def parse_docs(args, include_entities=True): + h = SConsDoc.SConsDocHandler() + for f in args: + if include_entities: + try: + h.parseXmlFile(f) + except: + sys.stderr.write("error in %s\n" % f) + raise + else: + content = open(f).read() + if content: + try: + h.parseContent(content, include_entities) + except: + sys.stderr.write("error in %s\n" % f) + raise + return h + +Warning = """\ +<!-- +THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. +--> +""" + +Regular_Entities_Header = """\ +<!-- + + Regular %s entities. + +--> +""" + +Link_Entities_Header = """\ +<!-- + + Entities that are links to the %s entries in the appendix. + +--> +""" + +class SCons_XML(object): + def __init__(self, entries, **kw): + self.values = entries + for k, v in kw.items(): + setattr(self, k, v) + + def fopen(self, name): + if name == '-': + return sys.stdout + return open(name, 'w') + + def write(self, files): + gen, mod = files.split(',') + self.write_gen(gen) + self.write_mod(mod) + + def write_gen(self, filename): + if not filename: + return + # Try to split off .gen filename + if filename.count(','): + fl = filename.split(',') + filename = fl[0] + + # Start new XML file + root = stf.newXmlTree("variablelist") + + for v in self.values: + + ve = stf.newNode("varlistentry") + stf.setAttribute(ve, 'id', '%s%s' % (v.prefix, v.idfunc())) + for t in v.xml_terms(): + stf.appendNode(ve, t) + vl = stf.newNode("listitem") + added = False + if v.summary is not None: + for s in v.summary: + added = True + stf.appendNode(vl, stf.copyNode(s)) + + if len(v.sets): + added = True + vp = stf.newNode("para") + s = ['&cv-link-%s;' % x for x in v.sets] + stf.setText(vp, 'Sets: ' + ', '.join(s) + '.') + stf.appendNode(vl, vp) + if len(v.uses): + added = True + vp = stf.newNode("para") + u = ['&cv-link-%s;' % x for x in v.uses] + stf.setText(vp, 'Uses: ' + ', '.join(u) + '.') + stf.appendNode(vl, vp) + + # Still nothing added to this list item? + if not added: + # Append an empty para + vp = stf.newNode("para") + stf.appendNode(vl, vp) + + stf.appendNode(ve, vl) + stf.appendNode(root, ve) + + # Write file + f = self.fopen(filename) + stf.writeGenTree(root, f) + + def write_mod(self, filename): + try: + description = self.values[0].description + except: + description = "" + if not filename: + return + # Try to split off .mod filename + if filename.count(','): + fl = filename.split(',') + filename = fl[1] + f = self.fopen(filename) + f.write(Warning) + f.write('\n') + f.write(Regular_Entities_Header % description) + f.write('\n') + for v in self.values: + f.write('<!ENTITY %s%s "<%s xmlns=\'%s\'>%s</%s>">\n' % + (v.prefix, v.idfunc(), + v.tag, SConsDoc.dbxsd, v.entityfunc(), v.tag)) + if self.env_signatures: + f.write('\n') + for v in self.values: + f.write('<!ENTITY %senv-%s "<%s xmlns=\'%s\'>env.%s</%s>">\n' % + (v.prefix, v.idfunc(), + v.tag, SConsDoc.dbxsd, v.entityfunc(), v.tag)) + f.write('\n') + f.write(Warning) + f.write('\n') + f.write(Link_Entities_Header % description) + f.write('\n') + for v in self.values: + f.write('<!ENTITY %slink-%s "<link linkend=\'%s%s\' xmlns=\'%s\'><%s>%s</%s></link>">\n' % + (v.prefix, v.idfunc(), + v.prefix, v.idfunc(), SConsDoc.dbxsd, + v.tag, v.entityfunc(), v.tag)) + if self.env_signatures: + f.write('\n') + for v in self.values: + f.write('<!ENTITY %slink-env-%s "<link linkend=\'%s%s\' xmlns=\'%s\'><%s>env.%s</%s></link>">\n' % + (v.prefix, v.idfunc(), + v.prefix, v.idfunc(), SConsDoc.dbxsd, + v.tag, v.entityfunc(), v.tag)) + f.write('\n') + f.write(Warning) + +class Proxy(object): + def __init__(self, subject): + """Wrap an object as a Proxy object""" + self.__subject = subject + + def __getattr__(self, name): + """Retrieve an attribute from the wrapped object. If the named + attribute doesn't exist, AttributeError is raised""" + return getattr(self.__subject, name) + + def get(self): + """Retrieve the entire wrapped object""" + return self.__subject + + def __cmp__(self, other): + if issubclass(other.__class__, self.__subject.__class__): + return cmp(self.__subject, other) + return cmp(self.__dict__, other.__dict__) + +class SConsThing(Proxy): + def idfunc(self): + return self.name + + def xml_terms(self): + e = stf.newNode("term") + stf.setText(e, self.name) + return [e] + +class Builder(SConsThing): + description = 'builder' + prefix = 'b-' + tag = 'function' + + def xml_terms(self): + ta = stf.newNode("term") + b = stf.newNode(self.tag) + stf.setText(b, self.name+'()') + stf.appendNode(ta, b) + tb = stf.newNode("term") + b = stf.newNode(self.tag) + stf.setText(b, 'env.'+self.name+'()') + stf.appendNode(tb, b) + return [ta, tb] + + def entityfunc(self): + return self.name + +class Function(SConsThing): + description = 'function' + prefix = 'f-' + tag = 'function' + + def xml_terms(self): + if self.arguments is None: + a = stf.newNode("arguments") + stf.setText(a, '()') + arguments = [a] + else: + arguments = self.arguments + tlist = [] + for arg in arguments: + signature = 'both' + if stf.hasAttribute(arg, 'signature'): + signature = stf.getAttribute(arg, 'signature') + s = stf.getText(arg).strip() + if signature in ('both', 'global'): + t = stf.newNode("term") + syn = stf.newNode("literal") + stf.setText(syn, '%s%s' % (self.name, s)) + stf.appendNode(t, syn) + tlist.append(t) + if signature in ('both', 'env'): + t = stf.newNode("term") + syn = stf.newNode("literal") + stf.setText(syn, 'env.%s%s' % (self.name, s)) + stf.appendNode(t, syn) + tlist.append(t) + + if not tlist: + tlist.append(stf.newNode("term")) + return tlist + + def entityfunc(self): + return self.name + +class Tool(SConsThing): + description = 'tool' + prefix = 't-' + tag = 'literal' + + def idfunc(self): + return self.name.replace('+', 'X') + + def entityfunc(self): + return self.name + +class Variable(SConsThing): + description = 'construction variable' + prefix = 'cv-' + tag = 'envar' + + def entityfunc(self): + return '$' + self.name + +def write_output_files(h, buildersfiles, functionsfiles, + toolsfiles, variablesfiles, write_func): + if buildersfiles: + g = processor_class([ Builder(b) for b in sorted(h.builders.values()) ], + env_signatures=True) + write_func(g, buildersfiles) + + if functionsfiles: + g = processor_class([ Function(b) for b in sorted(h.functions.values()) ], + env_signatures=True) + write_func(g, functionsfiles) + + if toolsfiles: + g = processor_class([ Tool(t) for t in sorted(h.tools.values()) ], + env_signatures=False) + write_func(g, toolsfiles) + + if variablesfiles: + g = processor_class([ Variable(v) for v in sorted(h.cvars.values()) ], + env_signatures=False) + write_func(g, variablesfiles) + +processor_class = SCons_XML + +# Step 1: Creating entity files for builders, functions,... +print "Generating entity files..." +h = parse_docs(args, False) +write_output_files(h, buildersfiles, functionsfiles, toolsfiles, + variablesfiles, SCons_XML.write_mod) + +# Step 2: Validating all input files +print "Validating files against SCons XSD..." +if SConsDoc.validate_all_xml(['src']): + print "OK" +else: + print "Validation failed! Please correct the errors above and try again." + +# Step 3: Creating actual documentation snippets, using the +# fully resolved and updated entities from the *.mod files. +print "Updating documentation for builders, tools and functions..." +h = parse_docs(args, True) +write_output_files(h, buildersfiles, functionsfiles, toolsfiles, + variablesfiles, SCons_XML.write) +print "Done" + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/scons-review.sh b/bin/scons-review.sh new file mode 100755 index 0000000..f126333 --- /dev/null +++ b/bin/scons-review.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +case "$1" in +'') exec svn diff --diff-cmd diff -x -c $* ;; +-m) svn diff --diff-cmd diff -x -c $* | alpine scons-dev ;; +*) echo "Error: unknown option '$1"; exit 1 ;; +esac + +# OLD CODE FOR USE WITH AEGIS +# +#if test $# -ne 1; then +# echo "Usage: scons-review change#" >&2 +# exit 1 +#fi +#if test "X$AEGIS_PROJECT" = "X"; then +# echo "scons-review: AEGIS_PROJECT is not set" >&2 +# exit 1 +#fi +#DIR=`aegis -cd -dd $*` +#if test "X${DIR}" = "X"; then +# echo "scons-review: No Aegis directory for '$*'" >&2 +# exit 1 +#fi +#(cd ${DIR} && find * -name '*,D' | sort | xargs cat) | pine scons-dev diff --git a/bin/scons-test.py b/bin/scons-test.py new file mode 100644 index 0000000..046cf4b --- /dev/null +++ b/bin/scons-test.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python +# +# A script that takes an scons-src-{version}.zip file, unwraps it in +# a temporary location, and calls runtest.py to execute one or more of +# its tests. +# +# The default is to download the latest scons-src archive from the SCons +# web site, and to execute all of the tests. +# +# With a little more work, this will become the basis of an automated +# testing and reporting system that anyone will be able to use to +# participate in testing SCons on their system and regularly reporting +# back the results. A --xml option is a stab at gathering a lot of +# relevant information about the system, the Python version, etc., +# so that problems on different platforms can be identified sooner. +# + +import atexit +import getopt +import imp +import os +import os.path +import sys +import tempfile +import time +import zipfile + +try: + # try Python 3.x style + from urllib.request import urlretrieve +except ImportError: + # nope, must be 2.x; this hack is equivalent + import imp + # protect import from fixer + urlretrieve = imp.load_module('urllib', + *imp.find_module('urllib')).urlretrieve + +helpstr = """\ +Usage: scons-test.py [-f zipfile] [-o outdir] [-v] [--xml] [runtest arguments] +Options: + -f FILE Specify input .zip FILE name + -o DIR, --out DIR Change output directory name to DIR + -v, --verbose Print file names when extracting + --xml XML output +""" + +opts, args = getopt.getopt(sys.argv[1:], + "f:o:v", + ['file=', 'out=', 'verbose', 'xml']) + +format = None +outdir = None +printname = lambda x: x +inputfile = 'http://scons.sourceforge.net/scons-src-latest.zip' + +for o, a in opts: + if o == '-f' or o == '--file': + inputfile = a + elif o == '-o' or o == '--out': + outdir = a + elif o == '-v' or o == '--verbose': + def printname(x): + print x + elif o == '--xml': + format = o + +startdir = os.getcwd() + +tempfile.template = 'scons-test.' +tempdir = tempfile.mktemp() + +if not os.path.exists(tempdir): + os.mkdir(tempdir) + def cleanup(tempdir=tempdir): + import shutil + os.chdir(startdir) + shutil.rmtree(tempdir) + atexit.register(cleanup) + +# Fetch the input file if it happens to be across a network somewhere. +# Ohmigod, does Python make this simple... +inputfile, headers = urlretrieve(inputfile) + +# Unzip the header file in the output directory. We use our own code +# (lifted from scons-unzip.py) to make the output subdirectory name +# match the basename of the .zip file. +zf = zipfile.ZipFile(inputfile, 'r') + +if outdir is None: + name, _ = os.path.splitext(os.path.basename(inputfile)) + outdir = os.path.join(tempdir, name) + +def outname(n, outdir=outdir): + l = [] + while True: + n, tail = os.path.split(n) + if not n: + break + l.append(tail) + l.append(outdir) + l.reverse() + return os.path.join(*l) + +for name in zf.namelist(): + dest = outname(name) + dir = os.path.dirname(dest) + try: + os.makedirs(dir) + except: + pass + printname(dest) + # if the file exists, then delete it before writing + # to it so that we don't end up trying to write to a symlink: + if os.path.isfile(dest) or os.path.islink(dest): + os.unlink(dest) + if not os.path.isdir(dest): + open(dest, 'w').write(zf.read(name)) + +os.chdir(outdir) + +# Load (by hand) the SCons modules we just unwrapped so we can +# extract their version information. Note that we have to override +# SCons.Script.main() with a do_nothing() function, because loading up +# the 'scons' script will actually try to execute SCons... +src_script = os.path.join(outdir, 'src', 'script') +src_engine = os.path.join(outdir, 'src', 'engine') +src_engine_SCons = os.path.join(src_engine, 'SCons') + +fp, pname, desc = imp.find_module('SCons', [src_engine]) +SCons = imp.load_module('SCons', fp, pname, desc) + +fp, pname, desc = imp.find_module('Script', [src_engine_SCons]) +SCons.Script = imp.load_module('Script', fp, pname, desc) + +def do_nothing(): + pass +SCons.Script.main = do_nothing + +fp, pname, desc = imp.find_module('scons', [src_script]) +scons = imp.load_module('scons', fp, pname, desc) +fp.close() + +# Default is to run all the tests by passing the -a flags to runtest.py. +if not args: + runtest_args = '-a' +else: + runtest_args = ' '.join(args) + +if format == '--xml': + + print "<scons_test_run>" + print " <sys>" + sys_keys = ['byteorder', 'exec_prefix', 'executable', 'maxint', 'maxunicode', 'platform', 'prefix', 'version', 'version_info'] + for k in sys_keys: + print " <%s>%s</%s>" % (k, sys.__dict__[k], k) + print " </sys>" + + fmt = '%a %b %d %H:%M:%S %Y' + print " <time>" + print " <gmtime>%s</gmtime>" % time.strftime(fmt, time.gmtime()) + print " <localtime>%s</localtime>" % time.strftime(fmt, time.localtime()) + print " </time>" + + print " <tempdir>%s</tempdir>" % tempdir + + def print_version_info(tag, module): + print " <%s>" % tag + print " <version>%s</version>" % module.__version__ + print " <build>%s</build>" % module.__build__ + print " <buildsys>%s</buildsys>" % module.__buildsys__ + print " <date>%s</date>" % module.__date__ + print " <developer>%s</developer>" % module.__developer__ + print " </%s>" % tag + + print " <scons>" + print_version_info("script", scons) + print_version_info("engine", SCons) + print " </scons>" + + environ_keys = [ + 'PATH', + 'SCONSFLAGS', + 'SCONS_LIB_DIR', + 'PYTHON_ROOT', + 'QTDIR', + + 'COMSPEC', + 'INTEL_LICENSE_FILE', + 'INCLUDE', + 'LIB', + 'MSDEVDIR', + 'OS', + 'PATHEXT', + 'SystemRoot', + 'TEMP', + 'TMP', + 'USERNAME', + 'VXDOMNTOOLS', + 'WINDIR', + 'XYZZY' + + 'ENV', + 'HOME', + 'LANG', + 'LANGUAGE', + 'LOGNAME', + 'MACHINE', + 'OLDPWD', + 'PWD', + 'OPSYS', + 'SHELL', + 'TMPDIR', + 'USER', + ] + + print " <environment>" + for key in sorted(environ_keys): + value = os.environ.get(key) + if value: + print " <variable>" + print " <name>%s</name>" % key + print " <value>%s</value>" % value + print " </variable>" + print " </environment>" + + command = '"%s" runtest.py -q -o - --xml %s' % (sys.executable, runtest_args) + #print command + os.system(command) + print "</scons_test_run>" + +else: + + def print_version_info(tag, module): + print "\t%s: v%s.%s, %s, by %s on %s" % (tag, + module.__version__, + module.__build__, + module.__date__, + module.__developer__, + module.__buildsys__) + + print "SCons by Steven Knight et al.:" + print_version_info("script", scons) + print_version_info("engine", SCons) + + command = '"%s" runtest.py %s' % (sys.executable, runtest_args) + #print command + os.system(command) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/scons-unzip.py b/bin/scons-unzip.py new file mode 100644 index 0000000..d4ec4bf --- /dev/null +++ b/bin/scons-unzip.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# +# A quick script to unzip a .zip archive and put the files in a +# subdirectory that matches the basename of the .zip file. +# +# This is actually generic functionality, it's not SCons-specific, but +# I'm using this to make it more convenient to manage working on multiple +# changes on Windows, where I don't have access to my Aegis tools. +# + +import getopt +import os.path +import sys +import zipfile + +helpstr = """\ +Usage: scons-unzip.py [-o outdir] zipfile +Options: + -o DIR, --out DIR Change output directory name to DIR + -v, --verbose Print file names when extracting +""" + +opts, args = getopt.getopt(sys.argv[1:], + "o:v", + ['out=', 'verbose']) + +outdir = None +printname = lambda x: x + +for o, a in opts: + if o == '-o' or o == '--out': + outdir = a + elif o == '-v' or o == '--verbose': + def printname(x): + print x + +if len(args) != 1: + sys.stderr.write("scons-unzip.py: \n") + sys.exit(1) + +zf = zipfile.ZipFile(str(args[0]), 'r') + +if outdir is None: + outdir, _ = os.path.splitext(os.path.basename(args[0])) + +def outname(n, outdir=outdir): + l = [] + while True: + n, tail = os.path.split(n) + if not n: + break + l.append(tail) + l.append(outdir) + l.reverse() + return os.path.join(*l) + +for name in zf.namelist(): + dest = outname(name) + dir = os.path.dirname(dest) + try: + os.makedirs(dir) + except: + pass + printname(dest) + # if the file exists, then delete it before writing + # to it so that we don't end up trying to write to a symlink: + if os.path.isfile(dest) or os.path.islink(dest): + os.unlink(dest) + if not os.path.isdir(dest): + open(dest, 'w').write(zf.read(name)) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/scons_dev_master.py b/bin/scons_dev_master.py new file mode 100644 index 0000000..3c41ac0 --- /dev/null +++ b/bin/scons_dev_master.py @@ -0,0 +1,215 @@ +#!/bin/sh +# + +# A script for turning a generic Ubuntu system into a master for +# SCons development. + +import getopt +import sys + +from Command import CommandRunner, Usage + +INITIAL_PACKAGES = [ + 'subversion', +] + +INSTALL_PACKAGES = [ + 'wget', +] + +PYTHON_PACKAGES = [ + 'g++', + 'gcc', + 'make', + 'zlib1g-dev', +] + +BUILDING_PACKAGES = [ + 'python-libxml2', + 'python-libxslt1', + 'fop', + 'python-dev', + 'python-epydoc', + 'rpm', + 'tar', + + # additional packages that Bill Deegan's web page suggests + #'docbook-to-man', + #'docbook-xsl', + #'docbook2x', + #'tetex-bin', + #'tetex-latex', + + # for ubuntu 9.10 + # 'texlive-lang-french' + +] + +DOCUMENTATION_PACKAGES = [ + 'docbook-doc', + 'epydoc-doc', + 'gcc-doc', + 'pkg-config', + 'python-doc', + 'sun-java5-doc', + 'sun-java6-doc', + 'swig-doc', + 'texlive-doc', +] + +TESTING_PACKAGES = [ + 'bison', + 'cssc', + 'cvs', + 'flex', + 'g++', + 'gcc', + 'gcj', + 'ghostscript', +# 'libgcj7-dev', + 'm4', + 'openssh-client', + 'openssh-server', + 'python-profiler', + 'python-all-dev', + 'rcs', + 'rpm', +# 'sun-java5-jdk', + 'sun-java6-jdk', + 'swig', + 'texlive-base-bin', + 'texlive-extra-utils', + 'texlive-latex-base', + 'texlive-latex-extra', + 'zip', +] + +BUILDBOT_PACKAGES = [ + 'buildbot', + 'cron', +] + +default_args = [ + 'upgrade', + 'checkout', + 'building', + 'testing', + 'python-versions', + 'scons-versions', +] + +def main(argv=None): + if argv is None: + argv = sys.argv + + short_options = 'hnqy' + long_options = ['help', 'no-exec', 'password=', 'quiet', 'username=', + 'yes', 'assume-yes'] + + helpstr = """\ +Usage: scons_dev_master.py [-hnqy] [--password PASSWORD] [--username USER] + [ACTIONS ...] + + ACTIONS (in default order): + upgrade Upgrade the system + checkout Check out SCons + building Install packages for building SCons + testing Install packages for testing SCons + scons-versions Install versions of SCons + python-versions Install versions of Python + + ACTIONS (optional): + buildbot Install packages for running BuildBot +""" + + scons_url = 'http://scons.tigris.org/svn/scons/trunk' + sudo = 'sudo' + password = '""' + username = 'guest' + yesflag = '' + + try: + try: + opts, args = getopt.getopt(argv[1:], short_options, long_options) + except getopt.error, msg: + raise Usage(msg) + + for o, a in opts: + if o in ('-h', '--help'): + print helpstr + sys.exit(0) + elif o in ('-n', '--no-exec'): + CommandRunner.execute = CommandRunner.do_not_execute + elif o in ('--password'): + password = a + elif o in ('-q', '--quiet'): + CommandRunner.display = CommandRunner.do_not_display + elif o in ('--username'): + username = a + elif o in ('-y', '--yes', '--assume-yes'): + yesflag = o + except Usage, err: + sys.stderr.write(str(err.msg) + '\n') + sys.stderr.write('use -h to get help\n') + return 2 + + if not args: + args = default_args + + initial_packages = ' '.join(INITIAL_PACKAGES) + install_packages = ' '.join(INSTALL_PACKAGES) + building_packages = ' '.join(BUILDING_PACKAGES) + testing_packages = ' '.join(TESTING_PACKAGES) + buildbot_packages = ' '.join(BUILDBOT_PACKAGES) + python_packages = ' '.join(PYTHON_PACKAGES) + + cmd = CommandRunner(locals()) + + for arg in args: + if arg == 'upgrade': + cmd.run('%(sudo)s apt-get %(yesflag)s upgrade') + elif arg == 'checkout': + cmd.run('%(sudo)s apt-get %(yesflag)s install %(initial_packages)s') + cmd.run('svn co --username guest --password "" %(scons_url)s') + elif arg == 'building': + cmd.run('%(sudo)s apt-get %(yesflag)s install %(building_packages)s') + elif arg == 'testing': + cmd.run('%(sudo)s apt-get %(yesflag)s install %(testing_packages)s') + elif arg == 'buildbot': + cmd.run('%(sudo)s apt-get %(yesflag)s install %(buildbot_packages)s') + elif arg == 'python-versions': + if install_packages: + cmd.run('%(sudo)s apt-get %(yesflag)s install %(install_packages)s') + install_packages = None + cmd.run('%(sudo)s apt-get %(yesflag)s install %(python_packages)s') + try: + import install_python + except ImportError: + msg = 'Could not import install_python; skipping python-versions.\n' + sys.stderr.write(msg) + else: + install_python.main(['install_python.py', '-a']) + elif arg == 'scons-versions': + if install_packages: + cmd.run('%(sudo)s apt-get %(yesflag)s install %(install_packages)s') + install_packages = None + try: + import install_scons + except ImportError: + msg = 'Could not import install_scons; skipping scons-versions.\n' + sys.stderr.write(msg) + else: + install_scons.main(['install_scons.py', '-a']) + else: + msg = '%s: unknown argument %s\n' + sys.stderr.write(msg % (argv[0], repr(arg))) + sys.exit(1) + +if __name__ == "__main__": + sys.exit(main()) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/scp-sourceforge b/bin/scp-sourceforge new file mode 100755 index 0000000..ad761d1 --- /dev/null +++ b/bin/scp-sourceforge @@ -0,0 +1,38 @@ +#!/bin/bash +set -x +set -e + +if [ -z "$1" ]; then + echo usage: $0 SourceForgeUserName + exit +fi + +SF_USER=$1 + +rm -rf sf +for p in scons scons-src scons-local +do + mkdir -p sf/$p/$VERSION + cp -p src/Announce.txt \ + build/scons/CHANGES.txt \ + build/scons/RELEASE.txt \ + sf/$p/$VERSION +done + +cp -p build/dist/scons-$VERSION-1.noarch.rpm \ + build/dist/scons-$VERSION-1.src.rpm \ + build/dist/scons-$VERSION.tar.gz \ + build/dist/scons-$VERSION.win32.exe \ + build/dist/scons-$VERSION.zip \ + sf/scons/$VERSION +cp -p build/dist/scons-local-$VERSION.tar.gz \ + build/dist/scons-local-$VERSION.zip \ + sf/scons-src/$VERSION +cp -p build/dist/scons-src-$VERSION.tar.gz \ + build/dist/scons-src-$VERSION.zip \ + sf/scons-local/$VERSION + +# Transmit them in this order, since the most-recent is displayed at the top +scp -r sf/scons-local/ sf/scons-src/ sf/scons/ \ + $SF_USER,scons@frs.sourceforge.net:/home/pfs/project/s/sc/scons +rm -rf sf diff --git a/bin/sfsum b/bin/sfsum new file mode 100644 index 0000000..2dfa422 --- /dev/null +++ b/bin/sfsum @@ -0,0 +1,147 @@ +#!/usr/bin/env python +# +# sfsum.py: A script for parsing XML data exported from +# SourceForge projects. +# +# Right now, this is hard-coded to generate a summary of open bugs. +# +# XML data for SourceForge project is available for download by project +# administrators. Because it's intended for backup purposes, you have +# to slurp the whole set of data, including info about all of the closed +# items, the feature requests, etc., so it can get big. +# +# You can do this by hand (if you're an administrator) with a URL like +# this (where 30337 is the group_id for SCons): +# +# http://sourceforge.net/export/xml_export.php?group_id=30337 +# +# They also have a Perl script, called xml_export, available as part +# of a set of utilities called "adocman" which automate dealing with +# SourceForge document management from the command line. "adocman" +# is available at: +# +# https://sourceforge.net/projects/sitedocs/ +# + +import xml.sax +import xml.sax.saxutils +import sys + +SFName = { + 'Unassigned' : 'nobody', + 'Chad Austin' : 'aegis', + 'Charle Crain' : 'diewarzau', + 'Steven Knight' : 'stevenknight', + 'Steve Leblanc' : 'stevenleblanc', + 'Jeff Petkau' : 'jpet', + 'Anthony Roach' : 'anthonyroach', + 'Steven Shaw' : 'steven_shaw', + 'Terrel Shumway' : 'terrelshumway', + 'Greg Spencer' : 'greg_spencer', + 'Christoph Wiedemann' : 'wiedeman', +} + +class Artifact(object): + """Just a place to hold attributes that we find in the XML.""" + pass + +Artifacts = {} + +def nws(text): + """Normalize white space. This will become important if/when + we enhance this to search for arbitrary fields.""" + return ' '.join(text.split()) + +class ClassifyArtifacts(xml.sax.saxutils.DefaultHandler): + """ + Simple SAX subclass to classify the artifacts in SourceForge + XML output. + + This reads up the fields in an XML description and turns the field + descriptions into attributes of an Artificat object, on the fly. + Artifacts are of the following types: + + Bugs + Feature Requests + Patches + Support Requests + + We could, if we choose to, add additional types in the future + by creating additional trackers. + + This class loses some info right now because we don't pay attention + to the <messages> tag in the output, which contains a list of items + that have <field> tags in them. Right now, these just overwrite + each other in the Arifact object we create. + + We also don't pay attention to any attributes of a <field> tag other + than the "name" attribute. We'll need to extend this class if we + ever want to pay attention to those attributes. + """ + def __init__(self): + self.artifact = None + + def startElement(self, name, attrs): + self.text = "" + if name == 'artifact': + self.artifact = Artifact() + elif not self.artifact is None and name == 'field': + self.fname = attrs.get('name', None) + + def characters(self, ch): + if not self.artifact is None: + self.text = self.text + ch + + def endElement(self, name): + global Artifacts + if name == 'artifact': + type = self.artifact.artifact_type + try: + list = Artifacts[type] + except KeyError: + Artifacts[type] = list = [] + list.append(self.artifact) + self.artifact = None + elif not self.artifact is None and name == 'field': + setattr(self.artifact, self.fname, self.text) + +if __name__ == '__main__': + # Create a parser. + parser = xml.sax.make_parser() + # Tell the parser we are not interested in XML namespaces. + parser.setFeature(xml.sax.handler.feature_namespaces, 0) + + # Instantiate our handler and tell the parser to use it. + parser.setContentHandler(ClassifyArtifacts()) + + # Parse the input. + parser.parse(sys.argv[1]) + + # Hard-coded search for 'Open' bugs. This should be easily + # generalized once we figure out other things for this script to do. + bugs = [x for x in Artifacts['Bugs'] if x.status == 'Open'] + + print list(Artifacts.keys()) + + print "%d open bugs" % len(bugs) + + # Sort them into a separate list for each assignee. + Assigned = {} + for bug in bugs: + a = bug.assigned_to + try: + list = Assigned[a] + except KeyError: + Assigned[a] = list = [] + list.append(bug) + + for a in SFName.keys(): + try: + b = Assigned[SFName[a]] + except KeyError: + pass + else: + print " %s" % a + b.sort(lambda x, y: cmp(x.artifact_id, y.artifact_id)) + for bug in b: + print " %-6s %s" % (bug.artifact_id, bug.summary) diff --git a/bin/svn-bisect.py b/bin/svn-bisect.py new file mode 100755 index 0000000..77bda58 --- /dev/null +++ b/bin/svn-bisect.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# -*- Python -*- +from __future__ import division + +import sys +from math import log, ceil +from optparse import OptionParser +import subprocess + +# crack the command line +parser = OptionParser( + usage="%prog lower-revision upper-revision test_script [arg1 ...]", + description="Binary search for a bug in a SVN checkout") +(options,script_args) = parser.parse_args() + +# make sure we have sufficient parameters +if len(script_args) < 1: + parser.error("Need a lower revision") +elif len(script_args) < 2: + parser.error("Need an upper revision") +elif len(script_args) < 3: + parser.error("Need a script to run") + +# extract our starting values +lower = int(script_args[0]) +upper = int(script_args[1]) +script = script_args[2:] + +# print an error message and quit +def error(s): + print >>sys.stderr, "******", s, "******" + sys.exit(1) + +# update to the specified version and run test +def testfail(revision): + "Return true if test fails" + print "Updating to revision", revision + if subprocess.call(["svn","up","-qr",str(revision)]) != 0: + m = "SVN did not update properly to revision %d" + raise RuntimeError(m % revision) + return subprocess.call(script,shell=False) != 0 + +# confirm that the endpoints are different +print "****** Checking upper bracket", upper +upperfails = testfail(upper) +print "****** Checking lower bracket", lower +lowerfails = testfail(lower) +if upperfails == lowerfails: + error("Upper and lower revisions must bracket the failure") + +# binary search for transition +msg = "****** max %d revisions to test (bug bracketed by [%d,%d])" +while upper-lower > 1: + print msg % (ceil(log(upper-lower,2)), lower, upper) + + mid = (lower + upper)//2 + midfails = testfail(mid) + if midfails == lowerfails: + lower = mid + lowerfails = midfails + else: + upper = mid + upperfails = midfails + +# show which revision was first to fail +if upperfails != lowerfails: lower = upper +print "The error was caused by revision", lower + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/time-scons.py b/bin/time-scons.py new file mode 100644 index 0000000..b7d8ef1 --- /dev/null +++ b/bin/time-scons.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python +# +# time-scons.py: a wrapper script for running SCons timings +# +# This script exists to: +# +# 1) Wrap the invocation of runtest.py to run the actual TimeSCons +# timings consistently. It does this specifically by building +# SCons first, so .pyc compilation is not part of the timing. +# +# 2) Provide an interface for running TimeSCons timings against +# earlier revisions, before the whole TimeSCons infrastructure +# was "frozen" to provide consistent timings. This is done +# by updating the specific pieces containing the TimeSCons +# infrastructure to the earliest revision at which those pieces +# were "stable enough." +# +# By encapsulating all the logic in this script, our Buildbot +# infrastructure only needs to call this script, and we should be able +# to change what we need to in this script and have it affect the build +# automatically when the source code is updated, without having to +# restart either master or slave. + +import optparse +import os +import shutil +import subprocess +import sys +import tempfile +import xml.sax.handler + + +SubversionURL = 'http://scons.tigris.org/svn/scons' + + +# This is the baseline revision when the TimeSCons scripts first +# stabilized and collected "real," consistent timings. If we're timing +# a revision prior to this, we'll forcibly update the TimeSCons pieces +# of the tree to this revision to collect consistent timings for earlier +# revisions. +TimeSCons_revision = 4569 + +# The pieces of the TimeSCons infrastructure that are necessary to +# produce consistent timings, even when the rest of the tree is from +# an earlier revision that doesn't have these pieces. +TimeSCons_pieces = ['QMTest', 'timings', 'runtest.py'] + + +class CommandRunner(object): + """ + Executor class for commands, including "commands" implemented by + Python functions. + """ + verbose = True + active = True + + def __init__(self, dictionary={}): + self.subst_dictionary(dictionary) + + def subst_dictionary(self, dictionary): + self._subst_dictionary = dictionary + + def subst(self, string, dictionary=None): + """ + Substitutes (via the format operator) the values in the specified + dictionary into the specified command. + + The command can be an (action, string) tuple. In all cases, we + perform substitution on strings and don't worry if something isn't + a string. (It's probably a Python function to be executed.) + """ + if dictionary is None: + dictionary = self._subst_dictionary + if dictionary: + try: + string = string % dictionary + except TypeError: + pass + return string + + def display(self, command, stdout=None, stderr=None): + if not self.verbose: + return + if isinstance(command, tuple): + func = command[0] + args = command[1:] + s = '%s(%s)' % (func.__name__, ', '.join(map(repr, args))) + if isinstance(command, list): + # TODO: quote arguments containing spaces + # TODO: handle meta characters? + s = ' '.join(command) + else: + s = self.subst(command) + if not s.endswith('\n'): + s += '\n' + sys.stdout.write(s) + sys.stdout.flush() + + def execute(self, command, stdout=None, stderr=None): + """ + Executes a single command. + """ + if not self.active: + return 0 + if isinstance(command, str): + command = self.subst(command) + cmdargs = shlex.split(command) + if cmdargs[0] == 'cd': + command = (os.chdir,) + tuple(cmdargs[1:]) + if isinstance(command, tuple): + func = command[0] + args = command[1:] + return func(*args) + else: + if stdout is sys.stdout: + # Same as passing sys.stdout, except works with python2.4. + subout = None + elif stdout is None: + # Open pipe for anything else so Popen works on python2.4. + subout = subprocess.PIPE + else: + subout = stdout + if stderr is sys.stderr: + # Same as passing sys.stdout, except works with python2.4. + suberr = None + elif stderr is None: + # Merge with stdout if stderr isn't specified. + suberr = subprocess.STDOUT + else: + suberr = stderr + p = subprocess.Popen(command, + shell=(sys.platform == 'win32'), + stdout=subout, + stderr=suberr) + p.wait() + return p.returncode + + def run(self, command, display=None, stdout=None, stderr=None): + """ + Runs a single command, displaying it first. + """ + if display is None: + display = command + self.display(display) + return self.execute(command, stdout, stderr) + + def run_list(self, command_list, **kw): + """ + Runs a list of commands, stopping with the first error. + + Returns the exit status of the first failed command, or 0 on success. + """ + status = 0 + for command in command_list: + s = self.run(command, **kw) + if s and status == 0: + status = s + return 0 + + +def get_svn_revisions(branch, revisions=None): + """ + Fetch the actual SVN revisions for the given branch querying + "svn log." A string specifying a range of revisions can be + supplied to restrict the output to a subset of the entire log. + """ + command = ['svn', 'log', '--xml'] + if revisions: + command.extend(['-r', revisions]) + command.append(branch) + p = subprocess.Popen(command, stdout=subprocess.PIPE) + + class SVNLogHandler(xml.sax.handler.ContentHandler): + def __init__(self): + self.revisions = [] + def startElement(self, name, attributes): + if name == 'logentry': + self.revisions.append(int(attributes['revision'])) + + parser = xml.sax.make_parser() + handler = SVNLogHandler() + parser.setContentHandler(handler) + parser.parse(p.stdout) + return sorted(handler.revisions) + + +def prepare_commands(): + """ + Returns a list of the commands to be executed to prepare the tree + for testing. This involves building SCons, specifically the + build/scons subdirectory where our packaging build is staged, + and then running setup.py to create a local installed copy + with compiled *.pyc files. The build directory gets removed + first. + """ + commands = [] + if os.path.exists('build'): + commands.extend([ + ['mv', 'build', 'build.OLD'], + ['rm', '-rf', 'build.OLD'], + ]) + commands.append([sys.executable, 'bootstrap.py', 'build/scons']) + commands.append([sys.executable, + 'build/scons/setup.py', + 'install', + '--prefix=' + os.path.abspath('build/usr')]) + return commands + +def script_command(script): + """Returns the command to actually invoke the specified timing + script using our "built" scons.""" + return [sys.executable, 'runtest.py', '-x', 'build/usr/bin/scons', script] + +def do_revisions(cr, opts, branch, revisions, scripts): + """ + Time the SCons branch specified scripts through a list of revisions. + + We assume we're in a (temporary) directory in which we can check + out the source for the specified revisions. + """ + stdout = sys.stdout + stderr = sys.stderr + + status = 0 + + if opts.logsdir and not opts.no_exec and len(scripts) > 1: + for script in scripts: + subdir = os.path.basename(os.path.dirname(script)) + logsubdir = os.path.join(opts.origin, opts.logsdir, subdir) + if not os.path.exists(logsubdir): + os.makedirs(logsubdir) + + for this_revision in revisions: + + if opts.logsdir and not opts.no_exec: + log_name = '%s.log' % this_revision + log_file = os.path.join(opts.origin, opts.logsdir, log_name) + stdout = open(log_file, 'w') + stderr = None + + commands = [ + ['svn', 'co', '-q', '-r', str(this_revision), branch, '.'], + ] + + if int(this_revision) < int(TimeSCons_revision): + commands.append(['svn', 'up', '-q', '-r', str(TimeSCons_revision)] + + TimeSCons_pieces) + + commands.extend(prepare_commands()) + + s = cr.run_list(commands, stdout=stdout, stderr=stderr) + if s: + if status == 0: + status = s + continue + + for script in scripts: + if opts.logsdir and not opts.no_exec and len(scripts) > 1: + subdir = os.path.basename(os.path.dirname(script)) + lf = os.path.join(opts.origin, opts.logsdir, subdir, log_name) + out = open(lf, 'w') + err = None + close_out = True + else: + out = stdout + err = stderr + close_out = False + s = cr.run(script_command(script), stdout=out, stderr=err) + if s and status == 0: + status = s + if close_out: + out.close() + out = None + + if int(this_revision) < int(TimeSCons_revision): + # "Revert" the pieces that we previously updated to the + # TimeSCons_revision, so the update to the next revision + # works cleanly. + command = (['svn', 'up', '-q', '-r', str(this_revision)] + + TimeSCons_pieces) + s = cr.run(command, stdout=stdout, stderr=stderr) + if s: + if status == 0: + status = s + continue + + if stdout not in (sys.stdout, None): + stdout.close() + stdout = None + + return status + +Usage = """\ +time-scons.py [-hnq] [-r REVISION ...] [--branch BRANCH] + [--logsdir DIR] [--svn] SCRIPT ...""" + +def main(argv=None): + if argv is None: + argv = sys.argv + + parser = optparse.OptionParser(usage=Usage) + parser.add_option("--branch", metavar="BRANCH", default="trunk", + help="time revision on BRANCH") + parser.add_option("--logsdir", metavar="DIR", default='.', + help="generate separate log files for each revision") + parser.add_option("-n", "--no-exec", action="store_true", + help="no execute, just print the command line") + parser.add_option("-q", "--quiet", action="store_true", + help="quiet, don't print the command line") + parser.add_option("-r", "--revision", metavar="REVISION", + help="time specified revisions") + parser.add_option("--svn", action="store_true", + help="fetch actual revisions for BRANCH") + opts, scripts = parser.parse_args(argv[1:]) + + if not scripts: + sys.stderr.write('No scripts specified.\n') + sys.exit(1) + + CommandRunner.verbose = not opts.quiet + CommandRunner.active = not opts.no_exec + cr = CommandRunner() + + os.environ['TESTSCONS_SCONSFLAGS'] = '' + + branch = SubversionURL + '/' + opts.branch + + if opts.svn: + revisions = get_svn_revisions(branch, opts.revision) + elif opts.revision: + # TODO(sgk): parse this for SVN-style revision strings + revisions = [opts.revision] + else: + revisions = None + + if opts.logsdir and not os.path.exists(opts.logsdir): + os.makedirs(opts.logsdir) + + if revisions: + opts.origin = os.getcwd() + tempdir = tempfile.mkdtemp(prefix='time-scons-') + try: + os.chdir(tempdir) + status = do_revisions(cr, opts, branch, revisions, scripts) + finally: + os.chdir(opts.origin) + shutil.rmtree(tempdir) + else: + commands = prepare_commands() + commands.extend([ script_command(script) for script in scripts ]) + status = cr.run_list(commands, stdout=sys.stdout, stderr=sys.stderr) + + return status + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/timebuild b/bin/timebuild new file mode 100644 index 0000000..d5af983 --- /dev/null +++ b/bin/timebuild @@ -0,0 +1,65 @@ +#!/bin/sh +# +# Profile running SCons to build itself from the current package. +# +# This runs "aegis -build" to build a current scons-src-*.tar.gz +# package, unpacks it in the supplied directory name, and then +# starts a profiled run of an SCons build, followed by another. +# This results in two profiles: +# +# NAME/NAME-0.prof +# profile of a build-everything run +# +# NAME/NAME-1.prof +# profile of an all-up-to-date run +# +# This also copies the build scons-src-*.tar.gz file to the NAME +# subdirectory, and tars up everything under src/ as NAME/src.tar.gz, +# so that repeated runs with different in-progress changes can serve +# as their own crude version control, so you don't lose that exact +# combination of features which performed best. + +if test X$1 = X; then + echo "Must supply name!" >&2 + exit 1 +fi + +VERSION=0.90 + +DIR=$1 + +SRC="scons-src-$VERSION" +SRC_TAR_GZ="${SRC}.tar.gz" +B_D_SRC_TAR_GZ="build/dist/${SRC_TAR_GZ}" + +echo "Building ${B_D_SRC_TAR_GZ}: " `date` +aegis -build ${B_D_SRC_TAR_GZ} + +echo "mkdir ${DIR}: " `date` +mkdir ${DIR} + +echo "cp ${B_D_SRC_TAR_GZ} ${DIR}: " `date` +cp ${B_D_SRC_TAR_GZ} ${DIR} + +echo "tar cf ${DIR}/src.tar.gz: " `date` +tar cf ${DIR}/src.tar.gz src + +cd ${DIR} + +echo "tar zxf ${SRC_TAR_GZ}: " `date` +tar zxf ${SRC_TAR_GZ} + +cd ${SRC} + +SCRIPT="src/script/scons.py" +ARGS="version=$VERSION" + +export SCONS_LIB_DIR=`pwd`/src/engine + +echo "Build run starting: " `date` +python $SCRIPT --profile=../$DIR-0.prof $ARGS > ../$DIR-0.log 2>&1 + +echo "Up-to-date run starting: " `date` +python $SCRIPT --profile=../$DIR-1.prof $ARGS > ../$DIR-1.log 2>&1 + +echo "Finished $DIR at: " `date` diff --git a/bin/update-release-info.py b/bin/update-release-info.py new file mode 100644 index 0000000..e523ad9 --- /dev/null +++ b/bin/update-release-info.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python +""" +This program takes information about a release in the ReleaseConfig file +and inserts the information in various files. It helps to keep the files +in sync because it never forgets which files need to be updated, nor what +information should be inserted in each file. + +It takes one parameter that says what the mode of update should be, which +may be one of 'develop', 'release', or 'post'. + +In 'develop' mode, which is what someone would use as part of their own +development practices, the release type is forced to be 'alpha' and the +patch level is the string 'yyyymmdd'. Otherwise, it's the same as the +'release' mode. + +In 'release' mode, the release type is taken from ReleaseConfig and the +patch level is calculated from the release date for non-final runs and +taken from the version tuple for final runs. It then inserts information +in various files: + - The RELEASE header line in src/CHANGES.txt and src/Announce.txt. + - The version string at the top of src/RELEASE.txt. + - The version string in the 'default_version' variable in SConstruct + and QMTest/TestSCons.py. + - The copyright years in SConstruct and QMTest/TestSCons.py. + - The month and year (used for documentation) in SConstruct. + - The unsupported and deprecated Python floors in QMTest/TestSCons.py + and src/engine/SCons/Script/Main.py + - The version string in the filenames in README. + +In 'post' mode, files are prepared for the next release cycle: + - In ReleaseConfig, the version tuple is updated for the next release + by incrementing the release number (either minor or micro, depending + on the branch) and resetting release type to 'alpha'. + - A blank template replaces src/RELEASE.txt. + - A new section to accumulate changes is added to src/CHANGES.txt and + src/Announce.txt. +""" +# +# Copyright (c) 2001 - 2014 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +__revision__ = "bin/update-release-info.py 2014/09/27 12:51:43 garyo" + +import os +import sys +import time +import re + +DEBUG = os.environ.get('DEBUG', 0) + +# Evaluate parameter + +if len(sys.argv) < 2: + mode = 'develop' +else: + mode = sys.argv[1] + if mode not in ['develop', 'release', 'post']: + print("""ERROR: `%s' as a parameter is invalid; it must be one of +\tdevelop, release, or post. The default is develop.""" % mode) + sys.exit(1) + +# Get configuration information + +config = dict() +exec open('ReleaseConfig').read() in globals(), config + +try: + version_tuple = config['version_tuple'] + unsupported_version = config['unsupported_python_version'] + deprecated_version = config['deprecated_python_version'] +except KeyError: + print('''ERROR: Config file must contain at least version_tuple, +\tunsupported_python_version, and deprecated_python_version.''') + sys.exit(1) +if DEBUG: print 'version tuple', version_tuple +if DEBUG: print 'unsupported Python version', unsupported_version +if DEBUG: print 'deprecated Python version', deprecated_version + +try: + release_date = config['release_date'] +except KeyError: + release_date = time.localtime()[:6] +else: + if len(release_date) == 3: + release_date = release_date + time.localtime()[3:6] + if len(release_date) != 6: + print '''ERROR: Invalid release date''', release_date + sys.exit(1) +if DEBUG: print 'release date', release_date + +if mode == 'develop' and version_tuple[3] != 'alpha': + version_tuple == version_tuple[:3] + ('alpha', 0) +if len(version_tuple) > 3 and version_tuple[3] != 'final': + if mode == 'develop': + version_tuple = version_tuple[:4] + ('yyyymmdd',) + else: + yyyy,mm,dd,_,_,_ = release_date + version_tuple = version_tuple[:4] + ((yyyy*100 + mm)*100 + dd,) +version_string = '.'.join(map(str, version_tuple)) +if len(version_tuple) > 3: + version_type = version_tuple[3] +else: + version_type = 'final' +if DEBUG: print 'version string', version_string + +if version_type not in ['alpha', 'beta', 'candidate', 'final']: + print("""ERROR: `%s' is not a valid release type in version tuple; +\tit must be one of alpha, beta, candidate, or final""" % version_type) + sys.exit(1) + +try: + month_year = config['month_year'] +except KeyError: + if version_type == 'alpha': + month_year = 'MONTH YEAR' + else: + month_year = time.strftime('%B %Y', release_date + (0,0,0)) +if DEBUG: print 'month year', month_year + +try: + copyright_years = config['copyright_years'] +except KeyError: + copyright_years = '2001 - %d'%(release_date[0] + 1) +if DEBUG: print 'copyright years', copyright_years + +class UpdateFile(object): + """ + XXX + """ + + def __init__(self, file, orig = None): + ''' + ''' + if orig is None: orig = file + try: + self.content = open(orig, 'rU').read() + except IOError: + # Couldn't open file; don't try to write anything in __del__ + self.file = None + raise + else: + self.file = file + if file == orig: + # so we can see if it changed + self.orig = self.content + else: + # pretend file changed + self.orig = '' + + def sub(self, pattern, replacement, count = 1): + ''' + XXX + ''' + self.content = re.sub(pattern, replacement, self.content, count) + + def replace_assign(self, name, replacement, count = 1): + ''' + XXX + ''' + self.sub('\n' + name + ' = .*', '\n' + name + ' = ' + replacement) + + # Determine the pattern to match a version + + _rel_types = '(alpha|beta|candidate|final)' + match_pat = '\d+\.\d+\.\d+\.' + _rel_types + '\.(\d+|yyyymmdd)' + match_rel = re.compile(match_pat) + + def replace_version(self, replacement = version_string, count = 1): + ''' + XXX + ''' + self.content = self.match_rel.sub(replacement, self.content, count) + + # Determine the release date and the pattern to match a date + # Mon, 05 Jun 2010 21:17:15 -0700 + # NEW DATE WILL BE INSERTED HERE + + if mode == 'develop': + new_date = 'NEW DATE WILL BE INSERTED HERE' + else: + min = (time.daylight and time.altzone or time.timezone)//60 + hr = min//60 + min = -(min%60 + hr*100) + new_date = (time.strftime('%a, %d %b %Y %X', release_date + (0,0,0)) + + ' %+.4d' % min) + + _days = '(Sun|Mon|Tue|Wed|Thu|Fri|Sat)' + _months = '(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oce|Nov|Dec)' + match_date = _days+', \d\d '+_months+' \d\d\d\d \d\d:\d\d:\d\d [+-]\d\d\d\d' + match_date = re.compile(match_date) + + def replace_date(self, replacement = new_date, count = 1): + ''' + XXX + ''' + self.content = self.match_date.sub(replacement, self.content, count) + + def __del__(self): + ''' + XXX + ''' + if self.file is not None and self.content != self.orig: + print 'Updating ' + self.file + '...' + open(self.file, 'w').write(self.content) + +if mode == 'post': + # Set up for the next release series. + + if version_tuple[2]: + # micro release, increment micro value + minor = version_tuple[1] + micro = version_tuple[2] + 1 + else: + # minor release, increment minor value + minor = version_tuple[1] + 1 + micro = 0 + new_tuple = (version_tuple[0], minor, micro, 'alpha', 0) + new_version = '.'.join(map(str, new_tuple[:4])) + '.yyyymmdd' + + # Update ReleaseConfig + + t = UpdateFile('ReleaseConfig') + if DEBUG: t.file = '/tmp/ReleaseConfig' + t.replace_assign('version_tuple', str(new_tuple)) + + # Update src/CHANGES.txt + + t = UpdateFile(os.path.join('src', 'CHANGES.txt')) + if DEBUG: t.file = '/tmp/CHANGES.txt' + t.sub('(\nRELEASE .*)', r"""\nRELEASE VERSION/DATE TO BE FILLED IN LATER\n + From John Doe:\n + - Whatever John Doe did.\n +\1""") + + # Update src/RELEASE.txt + + t = UpdateFile(os.path.join('src', 'RELEASE.txt'), + os.path.join('template', 'RELEASE.txt')) + if DEBUG: t.file = '/tmp/RELEASE.txt' + t.replace_version(new_version) + + # Update src/Announce.txt + + t = UpdateFile(os.path.join('src', 'Announce.txt')) + if DEBUG: t.file = '/tmp/Announce.txt' + t.sub('\nRELEASE .*', '\nRELEASE VERSION/DATE TO BE FILLED IN LATER') + announce_pattern = """( + Please note the following important changes scheduled for the next + release: +)""" + announce_replace = (r"""\1 + -- FEATURE THAT WILL CHANGE\n + Please note the following important changes since release """ + + '.'.join(map(str, version_tuple[:3])) + ':\n') + t.sub(announce_pattern, announce_replace) + + # Write out the last update and exit + + t = None + sys.exit() + +# Update src/CHANGES.txt + +t = UpdateFile(os.path.join('src', 'CHANGES.txt')) +if DEBUG: t.file = '/tmp/CHANGES.txt' +t.sub('\nRELEASE .*', '\nRELEASE ' + version_string + ' - ' + t.new_date) + +# Update src/RELEASE.txt + +t = UpdateFile(os.path.join('src', 'RELEASE.txt')) +if DEBUG: t.file = '/tmp/RELEASE.txt' +t.replace_version() + +# Update src/Announce.txt + +t = UpdateFile(os.path.join('src', 'Announce.txt')) +if DEBUG: t.file = '/tmp/Announce.txt' +t.sub('\nRELEASE .*', '\nRELEASE ' + version_string + ' - ' + t.new_date) + +# Update SConstruct + +t = UpdateFile('SConstruct') +if DEBUG: t.file = '/tmp/SConstruct' +t.replace_assign('month_year', repr(month_year)) +t.replace_assign('copyright_years', repr(copyright_years)) +t.replace_assign('default_version', repr(version_string)) + +# Update README + +t = UpdateFile('README.rst') +if DEBUG: t.file = '/tmp/README.rst' +t.sub('-' + t.match_pat + '\.', '-' + version_string + '.', count = 0) +for suf in ['tar', 'win32', 'zip', 'rpm', 'exe', 'deb']: + t.sub('-(\d+\.\d+\.\d+)\.%s' % suf, + '-%s.%s' % (version_string, suf), + count = 0) + +# Update QMTest/TestSCons.py + +t = UpdateFile(os.path.join('QMTest', 'TestSCons.py')) +if DEBUG: t.file = '/tmp/TestSCons.py' +t.replace_assign('copyright_years', repr(copyright_years)) +t.replace_assign('default_version', repr(version_string)) +#??? t.replace_assign('SConsVersion', repr(version_string)) +t.replace_assign('python_version_unsupported', str(unsupported_version)) +t.replace_assign('python_version_deprecated', str(deprecated_version)) + +# Update Script/Main.py + +t = UpdateFile(os.path.join('src', 'engine', 'SCons', 'Script', 'Main.py')) +if DEBUG: t.file = '/tmp/Main.py' +t.replace_assign('unsupported_python_version', str(unsupported_version)) +t.replace_assign('deprecated_python_version', str(deprecated_version)) + +# Update doc/user/main.{in,xml} + +docyears = '2004 - %d' % release_date[0] +if os.path.exists(os.path.join('doc', 'user', 'main.in')): + # this is no longer used as of Dec 2013 + t = UpdateFile(os.path.join('doc', 'user', 'main.in')) + if DEBUG: t.file = '/tmp/main.in' + ## TODO debug these + #t.sub('<pubdate>[^<]*</pubdate>', '<pubdate>' + docyears + '</pubdate>') + #t.sub('<year>[^<]*</year>', '<year>' + docyears + '</year>') + +t = UpdateFile(os.path.join('doc', 'user', 'main.xml')) +if DEBUG: t.file = '/tmp/main.xml' +t.sub('<pubdate>[^<]*</pubdate>', '<pubdate>' + docyears + '</pubdate>') +t.sub('<year>[^<]*</year>', '<year>' + docyears + '</year>') + +# Write out the last update + +t = None + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/upload-release-files.sh b/bin/upload-release-files.sh new file mode 100755 index 0000000..bf09751 --- /dev/null +++ b/bin/upload-release-files.sh @@ -0,0 +1,75 @@ +#!/bin/sh + +if [ $# -lt 2 ]; then + echo Usage: $0 VERSION SF_USERNAME + exit 1 +fi + +VERSION=$1; shift +SF_USER=$1; shift + +RSYNC='rsync' +RSYNCOPTS='-v -e ssh' +SF_MACHINE='frs.sourceforge.net' +SF_TOPDIR='/home/frs/project/scons' + +# the build products are here: +cd build/dist +cp -f ../../src/CHANGES.txt ../../src/RELEASE.txt ../../src/Announce.txt . + +set -x + +# Upload main scons release files: +$RSYNC $RSYNCOPTS \ + scons-$VERSION-1.noarch.rpm \ + scons-$VERSION-1.src.rpm \ + scons-$VERSION-setup.exe \ + scons-$VERSION.tar.gz \ + scons-$VERSION.zip \ + Announce.txt CHANGES.txt RELEASE.txt \ + $SF_USER@$SF_MACHINE:$SF_TOPDIR/scons/$VERSION/ + +# Local packages: +$RSYNC $RSYNCOPTS \ + scons-local-$VERSION.tar.gz \ + scons-local-$VERSION.zip \ + Announce.txt CHANGES.txt RELEASE.txt \ + $SF_USER@$SF_MACHINE:$SF_TOPDIR/scons-local/$VERSION/ + +# Source packages: +$RSYNC $RSYNCOPTS \ + scons-src-$VERSION.tar.gz \ + scons-src-$VERSION.zip \ + Announce.txt CHANGES.txt RELEASE.txt \ + $SF_USER@$SF_MACHINE:$SF_TOPDIR/scons-src/$VERSION/ + + +# +# scons.org stuff: +# +# Doc: copy the doc tgz over; we'll unpack later +$RSYNC $RSYNCOPTS \ + scons-doc-$VERSION.tar.gz \ + scons@scons.org:public_html/production/doc/$VERSION/ +# Copy the changelog +$RSYNC $RSYNCOPTS \ + CHANGES.txt \ + scons@scons.org:public_html/production/ +# Note that Announce.txt gets copied over to RELEASE.txt. +# This should be fixed at some point. +$RSYNC $RSYNCOPTS \ + Announce.txt \ + scons@scons.org:public_html/production/RELEASE.txt +# Unpack the doc and repoint doc symlinks: +ssh scons@scons.org " + cd public_html/production/doc + cd $VERSION + tar xvf scons-doc-$VERSION.tar.gz + cd .. + rm latest; ln -s $VERSION latest + rm production; ln -s $VERSION production + for f in HTML PDF EPUB PS TEXT; do rm \$f; ln -s $VERSION/\$f \$f; done +" +echo '*****' +echo '***** Now manually update index.php, includes/versions.php and news-raw.xhtml on scons.org.' +echo '*****' diff --git a/bin/xml_export b/bin/xml_export new file mode 100644 index 0000000..bc9ccbd --- /dev/null +++ b/bin/xml_export @@ -0,0 +1,225 @@ +#!/usr/bin/perl -w +# +# xml_export - Retrieve data from the SF.net XML export for project data +# +# Copyright (C) 2002 Open Source Development Network, Inc. ("OSDN") +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the license details found +# below in the section marked "$LICENSE_TEXT". +# +# SCons: modified the following RCS Id line so it won't expand during +# our checkins. +# +# $_Id: adocman,v 1.51 2002/06/07 18:56:35 moorman Exp _$ +# +# Written by Nate Oostendorp <oostendo@sourceforge.net> +# and Jacob Moorman <moorman@sourceforge.net> +########################################################################### + +use strict; +use Alexandria::Client; +use HTTP::Request::Common; +my $client = new Alexandria::Client; + +util_verifyvariables("groupid"); + +my $res = $ua->simple_request(GET "$config{hosturl}/export/xml_export.php?group_id=$config{groupid}"); + +if (not $res->is_success()) { + die "Failed to connect: ".$res->as_string(); +} +print $res->content; + +########################################################################### + +__END__ +=head1 NAME + +xml_export - Retrieve data for a project via the SF.net XML export facility + +=head1 DESCRIPTION + +B<This program> provides a simple mechanism to download data from the +XML data export facility on SourceForge.net. This utility is needed +(in place of a downloader like wget or curl) since authentication by +a project administrator is required to access the XML export facility. + +=head1 SYNOPSIS + +xml_export [options] > output_file + + OPTIONS + --login Login to the SourceForge.net site + --logout Logout of the SourceForge.net site + --groupid=GROUPID Group ID of the project whose data you wish to export + +=head1 ERROR LEVELS + +The following error levels are returned upon exit of this program: + + 0 success + + 1 failure: general (requested DocManager operation failed) + + 2 failure: authentication failure + + 3 failure: must --login before performing this operation + + 4 failure: bad command-line option specified or variable setting problem + + 5 failure: error in accessing/creating a file or directory + + 6 failure: failed to enter requested input before timeout expired + +=head1 AUTHORITATIVE SOURCE + +The original version of B<this program> may be found in the materials +provided from the SourceForge.net Site Documentation project (sitedocs) +on the SourceForge.net site. The latest version of this program +may be found in the CVS repository for the sitedocs project on +SourceForge.net. The sitedocs project pages may be accessed at: +http://sourceforge.net/projects/sitedocs + +=head1 SECURITY + +For security-related information for this application, please review +the documentation provided for the adocman utility. + +=head1 EXAMPLES + +The following are examples for using this program to export project +data via the XML data export facility on SourceForge.net. It is presumed +that you have a valid SourceForge.net user account, which is listed as +a project administrator on the project in question. This tool will +only work for project administrators. The group ID for the project +may be derived from the URL for the Admin page for the project, or by +viewing the Project Admin page for the project (look for the text +"Your Group ID is: xxxxxx"). + +To login to the SourceForge.net site via the command-line: + + adocman --username=myusername --password=mypassword --login \ + --groupid=8675309 + +To login to the SourceForge.net site, and be prompted to enter your +password interactively: + + adocman --username=myusername --interactive --login --groupid=8675309 + +To perform an export (after logging-in): + + xml_export --groupid=8675309 > output.xml + +To logout of SourceForge.net: + + adocman --logout + +Additional capabilities (including the use of configuration files to +specify information that would otherwise be provided interactively +or on the command-line) are detailed in the documentation provided for +the adocman utility. + +To obtain output for debugging a problem, perform the same command +as originally tested, but first add the --verbose flag, and determine +whether you are able to solve the issue on your own. If the problem +persists, see the "SUPPORT AND BUGS" section, below. + +=head1 SUPPORT AND BUGS + +This program was written by a member of the SourceForge.net staff +team. This software has been released under an Open Source license, +for the greater benefit of the SourceForge.net developer community. + +The SourceForge.net Site Documentation project is the caretaker of +this software. Issues related to the use of this program, or bugs +found in using this program, may be reported to the SourceForge.net +Site Documentation project using their Support Request Tracker at: +https://sourceforge.net/tracker/?func=add&group_id=52614&atid=467457 + +Any support that is provided for this program is provided as to +further enhance the stability and functionality of this program +for SourceForge.net users. The SourceForge.net Site Documentation +project makes use of this software for its own internal purposes, +in managing the Site Documentation collection for the SourceForge.net +site. + +=head1 AUTHOR + +Nathan Oostendorp <oostendo@sourceforge.net> and +Jacob Moorman <moorman@sourceforge.net> + +=head1 PREREQUISITES + +C<LWP::UserAgent>, C<HTML::TokeParser>, C<Crypt::SSLeay>, C<Digest::MD5>, +C<Term::ReadKey> + +These prerequisites may be installed in an interactive, but automated +fashion through the use of perl's CPAN module, invoked as: + + perl -MCPAN -e shell; + +=head1 LICENSE + +Copyright (c) 2002 Open Source Development Network, Inc. ("OSDN") + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +1. The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +2. Neither the names of VA Software Corporation, OSDN, SourceForge.net, +the SourceForge.net Site Documentation project, nor the names of its +contributors may be used to endorse or promote products derived from +the Software without specific prior written permission of OSDN. + +3. The name and trademarks of copyright holders may NOT be used in +advertising or publicity pertaining to the Software without specific, +written prior permission. Title to copyright in the Software and +any associated documentation will at all times remain with copyright +holders. + +4. If any files are modified, you must cause the modified files to carry +prominent notices stating that you changed the files and the date of +any change. We recommend that you provide URLs to the location from which +the code is derived. + +5. Altered versions of the Software must be plainly marked as such, and +must not be misrepresented as being the original Software. + +6. The origin of the Software must not be misrepresented; you must not +claim that you wrote the original Software. If you use the Software in a +product, an acknowledgment in the product documentation would be +appreciated but is not required. + +7. The data files supplied as input to, or produced as output from, +the programs of the Software do not automatically fall under the +copyright of the Software, but belong to whomever generated them, and may +be sold commercially, and may be aggregated with the Software. + +8. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE OR DOCUMENTATION. + +This Software consists of contributions made by OSDN and many individuals +on behalf of OSDN. Specific attributions are listed in the accompanying +credits file. + +=head1 HISTORY + +B<2002-12-03> Completed version 0.10 - move to classes, added POD + +=cut diff --git a/bin/xml_export-LICENSE b/bin/xml_export-LICENSE new file mode 100644 index 0000000..3f06fb7 --- /dev/null +++ b/bin/xml_export-LICENSE @@ -0,0 +1,52 @@ +Copyright (c) 2002 Open Source Development Network, Inc. ("OSDN") + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +1. The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +2. Neither the names of VA Software Corporation, OSDN, SourceForge.net, +the SourceForge.net Site Documentation project, nor the names of its +contributors may be used to endorse or promote products derived from +the Software without specific prior written permission of OSDN. + +3. The name and trademarks of copyright holders may NOT be used in +advertising or publicity pertaining to the Software without specific, +written prior permission. Title to copyright in the Software and +any associated documentation will at all times remain with copyright +holders. + +4. If any files are modified, you must cause the modified files to carry +prominent notices stating that you changed the files and the date of +any change. We recommend that you provide URLs to the location from which +the code is derived. + +5. Altered versions of the Software must be plainly marked as such, and +must not be misrepresented as being the original Software. + +6. The origin of the Software must not be misrepresented; you must not +claim that you wrote the original Software. If you use the Software in a +product, an acknowledgment in the product documentation would be +appreciated but is not required. + +7. The data files supplied as input to, or produced as output from, +the programs of the Software do not automatically fall under the +copyright of the Software, but belong to whomever generated them, and may +be sold commercially, and may be aggregated with the Software. + +8. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE OR DOCUMENTATION. + +This Software consists of contributions made by OSDN and many individuals +on behalf of OSDN. Specific attributions are listed in the accompanying +credits file. diff --git a/bin/xml_export-README b/bin/xml_export-README new file mode 100644 index 0000000..82e3524 --- /dev/null +++ b/bin/xml_export-README @@ -0,0 +1,56 @@ +This copy of xml_export was snarfed from adocman-0.10 from SourceForge. +We're checking in a copy as a convenience for any future SCons project +administrator who may need to download exported XML data. The original, +unmodified contents of the README file for that release of adocman are +as follows: + + +adocman - Automation tool for SourceForge.net DocManager handling +Copyright (C) 2002 Open Source Development Network, Inc. ("OSDN") + + +File manifest: + +Alexandria perl-based API for performing operations against the + SourceForge.net site, currently including basic Client + operations (i.e. login/logout) and DocManager operations + +adocman The adocman program, providing the means to perform + DocManager operations from the command-line or scripts + (by project developers or admins listed as DocManager Editors) + +xml_export The xml_export program, providing the means to automate + downloads of data from the XML data export facility + on SourceForge.net (by project administrators) + +adocman.html Manual for adocman, including background information, + command-line options detail, etc. + +xml_export.html Manual for xml_export, including basic info about + command-line options. See adocman.html for additional + information. + +LICENSE License terms for adocman + +README This file + +TODO List of ongoing work in improving adocman. NOTE: + Please contact the maintainer before starting any effort + to improve this code. We have significantly modified + the structure and design of this program for the next + release; structure and command-line interface are subject + to change without notice. + +A list of the prerequisites required to execute 'adocman' may be found +at in the PREREQUISITES section of the adocman manual (adocman.html). +Though not listed, a recent installation of 'perl' is also a prerequisite. + +Support for this program may be obtained as per the SUPPORT AND BUGS +section of the adocman.html manual. Any questions or concerns regarding +this software should be escalated as per the SUPPORT AND BUGS section +of the provided manual. + +The authoritative source of this software is: + https://sourceforge.net/projects/sitedocs + + diff --git a/bin/xmlagenda.py b/bin/xmlagenda.py new file mode 100755 index 0000000..b3cd520 --- /dev/null +++ b/bin/xmlagenda.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python + +# Query the scons.tigris.org database for the issues of interest. +# The typical triage query is found on http://www.scons.org/wiki/BugParty + +# Download the issues from Issuezilla as XML; this creates a file +# named 'issues.xml'. Run this script in the dir containing +# issues.xml (or pass full path as arg to this script) to translate +# 'issues.xml' into a CSV file named 'editlist.csv'. Upload the CSV +# into a Google spreadsheet. + +# In the spreadsheet: +# Select all the columns and pick "align-->top" +# Select the ID and votes columns and pick "align-->right" +# Select the priority column and pick "align-->center" +# Select the first row and click on the "bold" button +# Grab the lines between the column headers to adjust the column widths +# Grab the sort bar on the far left (just above the "1" for row one) +# and move it down one row. (Row one becomes a floating header) +# Voila! + +# The team members +# FIXME: These names really should be external to this script +team = sorted('Steven Gary Greg Ken Jim Bill Jason Dirk Anatoly'.split()) + +# The elements to be picked out of the issue +PickList = [ + # sort key -- these are used to sort the entry + 'target_milestone', 'priority', 'votes_desc', 'creation_ts', + # payload -- these are displayed + 'issue_id', 'votes', 'issue_type', 'target_milestone', + 'priority', 'assigned_to', 'short_desc', + ] + +# Conbert a leaf element into its value as a text string +# We assume it's "short enough" that there's only one substring +def Value(element): + v = element.firstChild + if v is None: return '' + return v.nodeValue + +# Parse the XML issues file and produce a DOM for it +import sys +if len(sys.argv) > 1: xml = sys.argv[1] +else: xml = 'issues.xml' +from xml.dom.minidom import parse +xml = parse(xml) + +# Go through the issues in the DOM, pick out the elements we want, +# and put them in our list of issues. +issues = [] +for issuezilla in xml.childNodes: + # The Issuezilla element contains the issues + if issuezilla.nodeType != issuezilla.ELEMENT_NODE: continue + for issue in issuezilla.childNodes: + # The issue elements contain the info for an issue + if issue.nodeType != issue.ELEMENT_NODE: continue + # Accumulate the pieces we want to include + d = {} + for element in issue.childNodes: + if element.nodeName in PickList: + d[element.nodeName] = Value(element) + # convert 'votes' to numeric, ascending and descending + try: + v = int('0' + d['votes']) + except KeyError: + pass + else: + d['votes_desc'] = -v + d['votes'] = v + # Marshal the elements and add them to the list + issues.append([ d[ix] for ix in PickList ]) +issues.sort() + +# Transcribe the issues into comma-separated values. +# FIXME: parameterize the output file name +import csv +writer = csv.writer(open('editlist.csv', 'w')) +# header +writer.writerow(['ID', 'Votes', 'Type/Member', 'Milestone', + 'Pri', 'Owner', 'Summary/Comments']) +for issue in issues: + row = issue[4:] # strip off sort key + #row[0] = """=hyperlink("http://scons.tigris.org/issues/show_bug.cgi?id=%s","%s")""" % (row[0],row[0]) + if row[3] == '-unspecified-': row[3] = 'triage' + writer.writerow(['','','','','','','']) + writer.writerow(row) + writer.writerow(['','','consensus','','','','']) + writer.writerow(['','','','','','','']) + for member in team: writer.writerow(['','',member,'','','','']) + +print "Exported %d issues to editlist.csv. Ready to upload to Google."%len(issues) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: |