1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/fr/examples/run-example	Wed Dec 02 18:33:34 2009 +0100
     1.3 @@ -0,0 +1,465 @@
     1.4 +#!/usr/bin/env python
     1.5 +#
     1.6 +# This program takes something that resembles a shell script and runs
     1.7 +# it, spitting input (commands from the script) and output into text
     1.8 +# files, for use in examples.
     1.9 +
    1.10 +import cStringIO
    1.11 +import errno
    1.12 +import getopt
    1.13 +import glob
    1.14 +import os
    1.15 +import pty
    1.16 +import re
    1.17 +import select
    1.18 +import shutil
    1.19 +import signal
    1.20 +import stat
    1.21 +import sys
    1.22 +import tempfile
    1.23 +import time
    1.24 +
    1.25 +xml_subs = {
    1.26 +    '<': '&lt;',
    1.27 +    '>': '&gt;',
    1.28 +    '&': '&amp;',
    1.29 +    }
    1.30 +
    1.31 +def gensubs(s):
    1.32 +    start = 0
    1.33 +    for i, c in enumerate(s):
    1.34 +        sub = xml_subs.get(c)
    1.35 +        if sub:
    1.36 +            yield s[start:i]
    1.37 +            start = i + 1
    1.38 +            yield sub
    1.39 +    yield s[start:]
    1.40 +
    1.41 +def xml_escape(s):
    1.42 +    return ''.join(gensubs(s))
    1.43 +        
    1.44 +def maybe_unlink(name):
    1.45 +    try:
    1.46 +        os.unlink(name)
    1.47 +        return True
    1.48 +    except OSError, err:
    1.49 +        if err.errno != errno.ENOENT:
    1.50 +            raise
    1.51 +    return False
    1.52 +
    1.53 +def find_path_to(program):
    1.54 +    for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
    1.55 +        name = os.path.join(p, program)
    1.56 +        if os.access(name, os.X_OK):
    1.57 +            return p
    1.58 +    return None
    1.59 +        
    1.60 +def result_name(name):
    1.61 +    return os.path.normpath(os.path.join('results', name.replace(os.sep, '-')))
    1.62 +
    1.63 +def wopen(name):
    1.64 +    path = os.path.dirname(name)
    1.65 +    if path:
    1.66 +        try:
    1.67 +            os.makedirs(path)
    1.68 +        except OSError, err:
    1.69 +            if err.errno != errno.EEXIST:
    1.70 +                raise
    1.71 +    return open(name, 'w')
    1.72 +
    1.73 +class example:
    1.74 +    entities = dict.fromkeys(l.rstrip() for l in open('auto-snippets.xml'))
    1.75 +
    1.76 +    def __init__(self, name, verbose, keep_change):
    1.77 +        self.name = os.path.normpath(name)
    1.78 +        self.verbose = verbose
    1.79 +        self.keep_change = keep_change
    1.80 +        
    1.81 +    def status(self, s):
    1.82 +        sys.stdout.write(s)
    1.83 +        if not s.endswith('\n'):
    1.84 +            sys.stdout.flush()
    1.85 +
    1.86 +    def rename_output(self, base, ignore=[]):
    1.87 +        mangle_re = re.compile('(?:' + '|'.join(ignore) + ')')
    1.88 +        def mangle(s):
    1.89 +            return mangle_re.sub('', s)
    1.90 +        def matchfp(fp1, fp2):
    1.91 +            while True:
    1.92 +                s1 = mangle(fp1.readline())
    1.93 +                s2 = mangle(fp2.readline())
    1.94 +                if cmp(s1, s2):
    1.95 +                    break
    1.96 +                if not s1:
    1.97 +                    return True
    1.98 +            return False
    1.99 +
   1.100 +        oldname = result_name(base + '.out')
   1.101 +        tmpname = result_name(base + '.tmp')
   1.102 +        errname = result_name(base + '.err')
   1.103 +        errfp = open(errname, 'w+')
   1.104 +        for line in open(tmpname):
   1.105 +            errfp.write(mangle_re.sub('', line))
   1.106 +        os.rename(tmpname, result_name(base + '.lxo'))
   1.107 +        errfp.seek(0)
   1.108 +        try:
   1.109 +            oldfp = open(oldname)
   1.110 +        except IOError, err:
   1.111 +            if err.errno != errno.ENOENT:
   1.112 +                raise
   1.113 +            os.rename(errname, oldname)
   1.114 +            return False
   1.115 +        if matchfp(oldfp, errfp):
   1.116 +            os.unlink(errname)
   1.117 +            return False
   1.118 +        else:
   1.119 +            print >> sys.stderr, '\nOutput of %s has changed!' % base
   1.120 +            if self.keep_change:
   1.121 +                os.rename(errname, oldname)
   1.122 +                return False
   1.123 +            else:
   1.124 +                os.system('diff -u %s %s 1>&2' % (oldname, errname))
   1.125 +            return True
   1.126 +
   1.127 +class static_example(example):
   1.128 +    def run(self):
   1.129 +        self.status('running %s\n' % self.name)
   1.130 +        s = open(self.name).read().rstrip()
   1.131 +        s = s.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
   1.132 +        ofp = wopen(result_name(self.name + '.tmp'))
   1.133 +        ofp.write('<!-- BEGIN %s -->\n' % self.name)
   1.134 +        ofp.write('<programlisting>')
   1.135 +        ofp.write(s)
   1.136 +        ofp.write('</programlisting>\n')
   1.137 +        ofp.write('<!-- END %s -->\n' % self.name)
   1.138 +        ofp.close()
   1.139 +        self.rename_output(self.name)
   1.140 +        norm = self.name.replace(os.sep, '-')
   1.141 +        example.entities[
   1.142 +            '<!ENTITY %s SYSTEM "results/%s.lxo">' % (norm, norm)] = 1
   1.143 +
   1.144 +
   1.145 +class shell_example(example):
   1.146 +    shell = '/usr/bin/env bash'
   1.147 +    ps1 = '__run_example_ps1__ '
   1.148 +    ps2 = '__run_example_ps2__ '
   1.149 +    pi_re = re.compile(r'#\$\s*(name|ignore):\s*(.*)$')
   1.150 +    
   1.151 +    timeout = 10
   1.152 +
   1.153 +    def __init__(self, name, verbose, keep_change):
   1.154 +        example.__init__(self, name, verbose, keep_change)
   1.155 +        self.poll = select.poll()
   1.156 +
   1.157 +    def parse(self):
   1.158 +        '''yield each hunk of input from the file.'''
   1.159 +        fp = open(self.name)
   1.160 +        cfp = cStringIO.StringIO()
   1.161 +        for line in fp:
   1.162 +            cfp.write(line)
   1.163 +            if not line.rstrip().endswith('\\'):
   1.164 +                yield cfp.getvalue()
   1.165 +                cfp.seek(0)
   1.166 +                cfp.truncate()
   1.167 +
   1.168 +    def send(self, s):
   1.169 +        if self.verbose:
   1.170 +            print >> sys.stderr, '>', self.debugrepr(s)
   1.171 +        while s:
   1.172 +            count = os.write(self.cfd, s)
   1.173 +            s = s[count:]
   1.174 +
   1.175 +    def debugrepr(self, s):
   1.176 +        rs = repr(s)
   1.177 +        limit = 60
   1.178 +        if len(rs) > limit:
   1.179 +            return ('%s%s ... [%d bytes]' % (rs[:limit], rs[0], len(s)))
   1.180 +        else:
   1.181 +            return rs
   1.182 +            
   1.183 +    timeout = 5
   1.184 +
   1.185 +    def read(self, hint):
   1.186 +        events = self.poll.poll(self.timeout * 1000)
   1.187 +        if not events:
   1.188 +            print >> sys.stderr, ('[%stimed out after %d seconds]' %
   1.189 +                                  (hint, self.timeout))
   1.190 +            os.kill(self.pid, signal.SIGHUP)
   1.191 +            return ''
   1.192 +        return os.read(self.cfd, 1024)
   1.193 +        
   1.194 +    def receive(self, hint):
   1.195 +        out = cStringIO.StringIO()
   1.196 +        while True:
   1.197 +            try:
   1.198 +                if self.verbose:
   1.199 +                    sys.stderr.write('< ')
   1.200 +                s = self.read(hint)
   1.201 +            except OSError, err:
   1.202 +                if err.errno == errno.EIO:
   1.203 +                    return '', ''
   1.204 +                raise
   1.205 +            if self.verbose:
   1.206 +                print >> sys.stderr, self.debugrepr(s)
   1.207 +            out.write(s)
   1.208 +            s = out.getvalue()
   1.209 +            if s.endswith(self.ps1):
   1.210 +                return self.ps1, s.replace('\r\n', '\n')[:-len(self.ps1)]
   1.211 +            if s.endswith(self.ps2):
   1.212 +                return self.ps2, s.replace('\r\n', '\n')[:-len(self.ps2)]
   1.213 +        
   1.214 +    def sendreceive(self, s, hint):
   1.215 +        self.send(s)
   1.216 +        ps, r = self.receive(hint)
   1.217 +        if r.startswith(s):
   1.218 +            r = r[len(s):]
   1.219 +        return ps, r
   1.220 +    
   1.221 +    def run(self):
   1.222 +        ofp = None
   1.223 +        basename = os.path.basename(self.name)
   1.224 +        self.status('running %s ' % basename)
   1.225 +        tmpdir = tempfile.mkdtemp(prefix=basename)
   1.226 +
   1.227 +        # remove the marker file that we tell make to use to see if
   1.228 +        # this run succeeded
   1.229 +        maybe_unlink(self.name + '.run')
   1.230 +
   1.231 +        rcfile = os.path.join(tmpdir, '.hgrc')
   1.232 +        rcfp = wopen(rcfile)
   1.233 +        print >> rcfp, '[ui]'
   1.234 +        print >> rcfp, "username = Bryan O'Sullivan <bos@serpentine.com>"
   1.235 +        
   1.236 +        rcfile = os.path.join(tmpdir, '.bashrc')
   1.237 +        rcfp = wopen(rcfile)
   1.238 +        print >> rcfp, 'PS1="%s"' % self.ps1
   1.239 +        print >> rcfp, 'PS2="%s"' % self.ps2
   1.240 +        print >> rcfp, 'unset HISTFILE'
   1.241 +        path = ['/usr/bin', '/bin']
   1.242 +        hg = find_path_to('hg')
   1.243 +        if hg and hg not in path:
   1.244 +            path.append(hg)
   1.245 +        def re_export(envar):
   1.246 +            v = os.getenv(envar)
   1.247 +            if v is not None:
   1.248 +                print >> rcfp, 'export ' + envar + '=' + v
   1.249 +        print >> rcfp, 'export PATH=' + ':'.join(path)
   1.250 +        re_export('PYTHONPATH')
   1.251 +        print >> rcfp, 'export EXAMPLE_DIR="%s"' % os.getcwd()
   1.252 +        print >> rcfp, 'export HGMERGE=merge'
   1.253 +        print >> rcfp, 'export LANG=C'
   1.254 +        print >> rcfp, 'export LC_ALL=C'
   1.255 +        print >> rcfp, 'export TZ=GMT'
   1.256 +        print >> rcfp, 'export HGRC="%s/.hgrc"' % tmpdir
   1.257 +        print >> rcfp, 'export HGRCPATH=$HGRC'
   1.258 +        print >> rcfp, 'cd %s' % tmpdir
   1.259 +        rcfp.close()
   1.260 +        sys.stdout.flush()
   1.261 +        sys.stderr.flush()
   1.262 +        self.pid, self.cfd = pty.fork()
   1.263 +        if self.pid == 0:
   1.264 +            cmdline = ['/usr/bin/env', '-i', 'bash', '--noediting',
   1.265 +                       '--noprofile', '--norc']
   1.266 +            try:
   1.267 +                os.execv(cmdline[0], cmdline)
   1.268 +            except OSError, err:
   1.269 +                print >> sys.stderr, '%s: %s' % (cmdline[0], err.strerror)
   1.270 +                sys.stderr.flush()
   1.271 +                os._exit(0)
   1.272 +        self.poll.register(self.cfd, select.POLLIN | select.POLLERR |
   1.273 +                           select.POLLHUP)
   1.274 +
   1.275 +        prompts = {
   1.276 +            '': '',
   1.277 +            self.ps1: '$',
   1.278 +            self.ps2: '>',
   1.279 +            }
   1.280 +
   1.281 +        ignore = [
   1.282 +            r'\d+:[0-9a-f]{12}', # changeset number:hash
   1.283 +            r'[0-9a-f]{40}', # long changeset hash
   1.284 +            r'[0-9a-f]{12}', # short changeset hash
   1.285 +            r'^(?:---|\+\+\+) .*', # diff header with dates
   1.286 +            r'^date:.*', # date
   1.287 +            #r'^diff -r.*', # "diff -r" is followed by hash
   1.288 +            r'^# Date \d+ \d+', # hg patch header
   1.289 +            ]
   1.290 +
   1.291 +        err = False
   1.292 +        read_hint = ''
   1.293 +
   1.294 +        try:
   1.295 +            try:
   1.296 +                # eat first prompt string from shell
   1.297 +                self.read(read_hint)
   1.298 +                # setup env and prompt
   1.299 +                ps, output = self.sendreceive('source %s\n' % rcfile,
   1.300 +                                              read_hint)
   1.301 +                for hunk in self.parse():
   1.302 +                    # is this line a processing instruction?
   1.303 +                    m = self.pi_re.match(hunk)
   1.304 +                    if m:
   1.305 +                        pi, rest = m.groups()
   1.306 +                        if pi == 'name':
   1.307 +                            self.status('.')
   1.308 +                            out = rest
   1.309 +                            if out in ('err', 'lxo', 'out', 'run', 'tmp'):
   1.310 +                                print >> sys.stderr, ('%s: illegal section '
   1.311 +                                                      'name %r' %
   1.312 +                                                      (self.name, out))
   1.313 +                                return 1
   1.314 +                            assert os.sep not in out
   1.315 +                            if ofp is not None:
   1.316 +                                ofp.write('</screen>\n')
   1.317 +                                ofp.write('<!-- END %s -->\n' % ofp_basename)
   1.318 +                                ofp.close()
   1.319 +                                err |= self.rename_output(ofp_basename, ignore)
   1.320 +                            if out:
   1.321 +                                ofp_basename = '%s.%s' % (self.name, out)
   1.322 +                                norm = os.path.normpath(ofp_basename)
   1.323 +                                norm = norm.replace(os.sep, '-')
   1.324 +                                example.entities[
   1.325 +                                    '<!ENTITY interaction.%s '
   1.326 +                                    'SYSTEM "results/%s.lxo">'
   1.327 +                                    % (norm, norm)] = 1
   1.328 +                                read_hint = ofp_basename + ' '
   1.329 +                                ofp = wopen(result_name(ofp_basename + '.tmp'))
   1.330 +                                ofp.write('<!-- BEGIN %s -->\n' % ofp_basename)
   1.331 +                                ofp.write('<screen>')
   1.332 +                            else:
   1.333 +                                ofp = None
   1.334 +                        elif pi == 'ignore':
   1.335 +                            ignore.append(rest)
   1.336 +                    elif hunk.strip():
   1.337 +                        # it's something we should execute
   1.338 +                        newps, output = self.sendreceive(hunk, read_hint)
   1.339 +                        if not ofp:
   1.340 +                            continue
   1.341 +                        # first, print the command we ran
   1.342 +                        if not hunk.startswith('#'):
   1.343 +                            nl = hunk.endswith('\n')
   1.344 +                            hunk = ('<prompt>%s</prompt> '
   1.345 +                                    '<userinput>%s</userinput>' %
   1.346 +                                    (prompts[ps],
   1.347 +                                     xml_escape(hunk.rstrip('\n'))))
   1.348 +                            if nl: hunk += '\n'
   1.349 +                        ofp.write(hunk)
   1.350 +                        # then its output
   1.351 +                        ofp.write(xml_escape(output))
   1.352 +                        ps = newps
   1.353 +                self.status('\n')
   1.354 +            except:
   1.355 +                print >> sys.stderr, '(killed)'
   1.356 +                os.kill(self.pid, signal.SIGKILL)
   1.357 +                pid, rc = os.wait()
   1.358 +                raise
   1.359 +            else:
   1.360 +                try:
   1.361 +                    ps, output = self.sendreceive('exit\n', read_hint)
   1.362 +                    if ofp is not None:
   1.363 +                        ofp.write(output)
   1.364 +                        ofp.write('</screen>\n')
   1.365 +                        ofp.write('<!-- END %s -->\n' % ofp_basename)
   1.366 +                        ofp.close()
   1.367 +                        err |= self.rename_output(ofp_basename, ignore)
   1.368 +                    os.close(self.cfd)
   1.369 +                except IOError:
   1.370 +                    pass
   1.371 +                os.kill(self.pid, signal.SIGTERM)
   1.372 +                pid, rc = os.wait()
   1.373 +                err = err or rc
   1.374 +                if err:
   1.375 +                    if os.WIFEXITED(rc):
   1.376 +                        print >> sys.stderr, '(exit %s)' % os.WEXITSTATUS(rc)
   1.377 +                    elif os.WIFSIGNALED(rc):
   1.378 +                        print >> sys.stderr, '(signal %s)' % os.WTERMSIG(rc)
   1.379 +                else:
   1.380 +                    wopen(result_name(self.name + '.run'))
   1.381 +                return err
   1.382 +        finally:
   1.383 +            shutil.rmtree(tmpdir)
   1.384 +
   1.385 +def print_help(exit, msg=None):
   1.386 +    if msg:
   1.387 +        print >> sys.stderr, 'Error:', msg
   1.388 +    print >> sys.stderr, 'Usage: run-example [options] [test...]'
   1.389 +    print >> sys.stderr, 'Options:'
   1.390 +    print >> sys.stderr, '  -a --all       run all examples in this directory'
   1.391 +    print >> sys.stderr, '  -h --help      print this help message'
   1.392 +    print >> sys.stderr, '     --keep      keep new output as desired output'
   1.393 +    print >> sys.stderr, '  -v --verbose   display extra debug output'
   1.394 +    sys.exit(exit)
   1.395 +
   1.396 +def main(path='.'):
   1.397 +    if os.path.realpath(path).split(os.sep)[-1] != 'examples':
   1.398 +        print >> sys.stderr, 'Not being run from the examples directory!'
   1.399 +        sys.exit(1)
   1.400 +
   1.401 +    opts, args = getopt.getopt(sys.argv[1:], '?ahv',
   1.402 +                               ['all', 'help', 'keep', 'verbose'])
   1.403 +    verbose = False
   1.404 +    run_all = False
   1.405 +    keep_change = False
   1.406 +
   1.407 +    for o, a in opts:
   1.408 +        if o in ('-h', '-?', '--help'):
   1.409 +            print_help(0)
   1.410 +        if o in ('-a', '--all'):
   1.411 +            run_all = True
   1.412 +        if o in ('--keep',):
   1.413 +            keep_change = True
   1.414 +        if o in ('-v', '--verbose'):
   1.415 +            verbose = True
   1.416 +    errs = 0
   1.417 +    if args:
   1.418 +        for a in args:
   1.419 +            try:
   1.420 +                st = os.lstat(a)
   1.421 +            except OSError, err:
   1.422 +                print >> sys.stderr, '%s: %s' % (a, err.strerror)
   1.423 +                errs += 1
   1.424 +                continue
   1.425 +            if stat.S_ISREG(st.st_mode):
   1.426 +                if st.st_mode & 0111:
   1.427 +                    if shell_example(a, verbose, keep_change).run():
   1.428 +                        errs += 1
   1.429 +                elif a.endswith('.lst'):
   1.430 +                    static_example(a, verbose, keep_change).run()
   1.431 +            else:
   1.432 +                print >> sys.stderr, '%s: not a file, or not executable' % a
   1.433 +                errs += 1
   1.434 +    elif run_all:
   1.435 +        names = glob.glob("*") + glob.glob("app*/*") + glob.glob("ch*/*")
   1.436 +        names.sort()
   1.437 +        for name in names:
   1.438 +            if name == 'run-example' or name.endswith('~'): continue
   1.439 +            pathname = os.path.join(path, name)
   1.440 +            try:
   1.441 +                st = os.lstat(pathname)
   1.442 +            except OSError, err:
   1.443 +                # could be an output file that was removed while we ran
   1.444 +                if err.errno != errno.ENOENT:
   1.445 +                    raise
   1.446 +                continue
   1.447 +            if stat.S_ISREG(st.st_mode):
   1.448 +                if st.st_mode & 0111:
   1.449 +                    if shell_example(pathname, verbose, keep_change).run():
   1.450 +                        errs += 1
   1.451 +                elif pathname.endswith('.lst'):
   1.452 +                    static_example(pathname, verbose, keep_change).run()
   1.453 +        print >> wopen(os.path.join(path, '.run')), time.asctime()
   1.454 +    else:
   1.455 +        print_help(1, msg='no test names given, and --all not provided')
   1.456 +
   1.457 +    fp = wopen('auto-snippets.xml')
   1.458 +    for key in sorted(example.entities.iterkeys()):
   1.459 +        print >> fp, key
   1.460 +    fp.close()
   1.461 +    return errs
   1.462 +
   1.463 +if __name__ == '__main__':
   1.464 +    try:
   1.465 +        sys.exit(main())
   1.466 +    except KeyboardInterrupt:
   1.467 +        print >> sys.stderr, 'interrupted!'
   1.468 +        sys.exit(1)