hgbook

annotate en/examples/run-example @ 564:0d5935744f87

find changesets by author, revision, files, or words in the commit message
Switch from LaTeX to XML for examples.
author Bryan O'Sullivan <bos@serpentine.com>
date Mon Mar 09 21:39:23 2009 -0700 (2009-03-09)
parents d8913b7869b5
children 8a9c66da6fcb
rev   line source
bos@67 1 #!/usr/bin/env python
bos@4 2 #
bos@4 3 # This program takes something that resembles a shell script and runs
bos@4 4 # it, spitting input (commands from the script) and output into text
bos@4 5 # files, for use in examples.
bos@3 6
bos@3 7 import cStringIO
bos@73 8 import errno
bos@78 9 import getopt
bos@3 10 import os
bos@3 11 import pty
bos@3 12 import re
bos@79 13 import select
bos@4 14 import shutil
bos@6 15 import signal
bos@36 16 import stat
bos@3 17 import sys
bos@4 18 import tempfile
bos@4 19 import time
bos@3 20
bos@564 21 xml_subs = {
bos@564 22 '<': '&lt;',
bos@564 23 '>': '&gt;',
bos@564 24 '&': '&amp;',
bos@77 25 }
bos@77 26
bos@77 27 def gensubs(s):
bos@77 28 start = 0
bos@77 29 for i, c in enumerate(s):
bos@564 30 sub = xml_subs.get(c)
bos@77 31 if sub:
bos@77 32 yield s[start:i]
bos@77 33 start = i + 1
bos@77 34 yield sub
bos@77 35 yield s[start:]
bos@77 36
bos@564 37 def xml_escape(s):
bos@77 38 return ''.join(gensubs(s))
bos@4 39
bos@137 40 def maybe_unlink(name):
bos@137 41 try:
bos@137 42 os.unlink(name)
bos@137 43 return True
bos@137 44 except OSError, err:
bos@137 45 if err.errno != errno.ENOENT:
bos@137 46 raise
bos@137 47 return False
bos@137 48
bos@172 49 def find_path_to(program):
bos@172 50 for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
bos@172 51 name = os.path.join(p, program)
bos@172 52 if os.access(name, os.X_OK):
bos@172 53 return p
bos@172 54 return None
bos@172 55
bos@564 56 def result_name(name):
bos@564 57 dirname, basename = os.path.split(name)
bos@564 58 return os.path.join(dirname, 'results', basename)
bos@564 59
bos@3 60 class example:
bos@70 61 shell = '/usr/bin/env bash'
bos@103 62 ps1 = '__run_example_ps1__ '
bos@103 63 ps2 = '__run_example_ps2__ '
bos@168 64 pi_re = re.compile(r'#\$\s*(name|ignore):\s*(.*)$')
bos@4 65
bos@192 66 timeout = 10
bos@79 67
bos@545 68 def __init__(self, name, verbose, keep_change):
bos@3 69 self.name = name
bos@78 70 self.verbose = verbose
bos@545 71 self.keep_change = keep_change
bos@79 72 self.poll = select.poll()
bos@3 73
bos@3 74 def parse(self):
bos@4 75 '''yield each hunk of input from the file.'''
bos@3 76 fp = open(self.name)
bos@3 77 cfp = cStringIO.StringIO()
bos@3 78 for line in fp:
bos@3 79 cfp.write(line)
bos@3 80 if not line.rstrip().endswith('\\'):
bos@3 81 yield cfp.getvalue()
bos@3 82 cfp.seek(0)
bos@3 83 cfp.truncate()
bos@3 84
bos@3 85 def status(self, s):
bos@3 86 sys.stdout.write(s)
bos@3 87 if not s.endswith('\n'):
bos@3 88 sys.stdout.flush()
bos@3 89
bos@6 90 def send(self, s):
bos@78 91 if self.verbose:
bos@78 92 print >> sys.stderr, '>', self.debugrepr(s)
bos@73 93 while s:
bos@73 94 count = os.write(self.cfd, s)
bos@73 95 s = s[count:]
bos@6 96
bos@78 97 def debugrepr(self, s):
bos@78 98 rs = repr(s)
bos@78 99 limit = 60
bos@78 100 if len(rs) > limit:
bos@78 101 return ('%s%s ... [%d bytes]' % (rs[:limit], rs[0], len(s)))
bos@78 102 else:
bos@78 103 return rs
bos@78 104
bos@79 105 timeout = 5
bos@79 106
bos@173 107 def read(self, hint):
bos@79 108 events = self.poll.poll(self.timeout * 1000)
bos@79 109 if not events:
bos@173 110 print >> sys.stderr, ('[%stimed out after %d seconds]' %
bos@173 111 (hint, self.timeout))
bos@79 112 os.kill(self.pid, signal.SIGHUP)
bos@79 113 return ''
bos@79 114 return os.read(self.cfd, 1024)
bos@79 115
bos@173 116 def receive(self, hint):
bos@6 117 out = cStringIO.StringIO()
bos@4 118 while True:
bos@73 119 try:
bos@78 120 if self.verbose:
bos@78 121 sys.stderr.write('< ')
bos@173 122 s = self.read(hint)
bos@73 123 except OSError, err:
bos@73 124 if err.errno == errno.EIO:
bos@103 125 return '', ''
bos@73 126 raise
bos@78 127 if self.verbose:
bos@78 128 print >> sys.stderr, self.debugrepr(s)
bos@6 129 out.write(s)
bos@73 130 s = out.getvalue()
bos@103 131 if s.endswith(self.ps1):
bos@103 132 return self.ps1, s.replace('\r\n', '\n')[:-len(self.ps1)]
bos@103 133 if s.endswith(self.ps2):
bos@103 134 return self.ps2, s.replace('\r\n', '\n')[:-len(self.ps2)]
bos@4 135
bos@173 136 def sendreceive(self, s, hint):
bos@6 137 self.send(s)
bos@173 138 ps, r = self.receive(hint)
bos@6 139 if r.startswith(s):
bos@6 140 r = r[len(s):]
bos@103 141 return ps, r
bos@6 142
bos@3 143 def run(self):
bos@3 144 ofp = None
bos@4 145 basename = os.path.basename(self.name)
bos@4 146 self.status('running %s ' % basename)
bos@4 147 tmpdir = tempfile.mkdtemp(prefix=basename)
bos@78 148
bos@136 149 # remove the marker file that we tell make to use to see if
bos@136 150 # this run succeeded
bos@137 151 maybe_unlink(self.name + '.run')
bos@136 152
bos@78 153 rcfile = os.path.join(tmpdir, '.hgrc')
bos@78 154 rcfp = open(rcfile, 'w')
bos@78 155 print >> rcfp, '[ui]'
bos@78 156 print >> rcfp, "username = Bryan O'Sullivan <bos@serpentine.com>"
bos@78 157
bos@6 158 rcfile = os.path.join(tmpdir, '.bashrc')
bos@6 159 rcfp = open(rcfile, 'w')
bos@103 160 print >> rcfp, 'PS1="%s"' % self.ps1
bos@103 161 print >> rcfp, 'PS2="%s"' % self.ps2
bos@6 162 print >> rcfp, 'unset HISTFILE'
bos@172 163 path = ['/usr/bin', '/bin']
bos@172 164 hg = find_path_to('hg')
bos@172 165 if hg and hg not in path:
bos@172 166 path.append(hg)
bos@172 167 def re_export(envar):
bos@172 168 v = os.getenv(envar)
bos@172 169 if v is not None:
bos@172 170 print >> rcfp, 'export ' + envar + '=' + v
bos@172 171 print >> rcfp, 'export PATH=' + ':'.join(path)
bos@172 172 re_export('PYTHONPATH')
bos@19 173 print >> rcfp, 'export EXAMPLE_DIR="%s"' % os.getcwd()
bos@124 174 print >> rcfp, 'export HGMERGE=merge'
bos@6 175 print >> rcfp, 'export LANG=C'
bos@6 176 print >> rcfp, 'export LC_ALL=C'
bos@6 177 print >> rcfp, 'export TZ=GMT'
bos@6 178 print >> rcfp, 'export HGRC="%s/.hgrc"' % tmpdir
bos@6 179 print >> rcfp, 'export HGRCPATH=$HGRC'
bos@6 180 print >> rcfp, 'cd %s' % tmpdir
bos@6 181 rcfp.close()
bos@68 182 sys.stdout.flush()
bos@68 183 sys.stderr.flush()
bos@79 184 self.pid, self.cfd = pty.fork()
bos@79 185 if self.pid == 0:
bos@172 186 cmdline = ['/usr/bin/env', '-i', 'bash', '--noediting',
bos@172 187 '--noprofile', '--norc']
bos@68 188 try:
bos@68 189 os.execv(cmdline[0], cmdline)
bos@68 190 except OSError, err:
bos@68 191 print >> sys.stderr, '%s: %s' % (cmdline[0], err.strerror)
bos@68 192 sys.stderr.flush()
bos@68 193 os._exit(0)
bos@79 194 self.poll.register(self.cfd, select.POLLIN | select.POLLERR |
bos@79 195 select.POLLHUP)
bos@103 196
bos@103 197 prompts = {
bos@103 198 '': '',
bos@103 199 self.ps1: '$',
bos@103 200 self.ps2: '>',
bos@103 201 }
bos@103 202
bos@137 203 ignore = [
bos@137 204 r'\d+:[0-9a-f]{12}', # changeset number:hash
bos@141 205 r'[0-9a-f]{40}', # long changeset hash
bos@141 206 r'[0-9a-f]{12}', # short changeset hash
bos@137 207 r'^(?:---|\+\+\+) .*', # diff header with dates
bos@137 208 r'^date:.*', # date
bos@141 209 #r'^diff -r.*', # "diff -r" is followed by hash
bos@138 210 r'^# Date \d+ \d+', # hg patch header
bos@137 211 ]
bos@137 212
bos@138 213 err = False
bos@173 214 read_hint = ''
bos@138 215
bos@4 216 try:
bos@71 217 try:
bos@73 218 # eat first prompt string from shell
bos@173 219 self.read(read_hint)
bos@71 220 # setup env and prompt
bos@173 221 ps, output = self.sendreceive('source %s\n' % rcfile,
bos@173 222 read_hint)
bos@71 223 for hunk in self.parse():
bos@71 224 # is this line a processing instruction?
bos@71 225 m = self.pi_re.match(hunk)
bos@71 226 if m:
bos@71 227 pi, rest = m.groups()
bos@71 228 if pi == 'name':
bos@71 229 self.status('.')
bos@71 230 out = rest
bos@155 231 if out in ('err', 'lxo', 'out', 'run', 'tmp'):
bos@155 232 print >> sys.stderr, ('%s: illegal section '
bos@155 233 'name %r' %
bos@155 234 (self.name, out))
bos@155 235 return 1
bos@71 236 assert os.sep not in out
bos@137 237 if ofp is not None:
bos@564 238 ofp.write('</screen>\n')
bos@137 239 ofp.close()
bos@160 240 err |= self.rename_output(ofp_basename, ignore)
bos@71 241 if out:
bos@137 242 ofp_basename = '%s.%s' % (self.name, out)
bos@173 243 read_hint = ofp_basename + ' '
bos@564 244 ofp = open(result_name(ofp_basename + '.tmp'),
bos@564 245 'w')
bos@564 246 ofp.write('<screen>')
bos@71 247 else:
bos@71 248 ofp = None
bos@137 249 elif pi == 'ignore':
bos@137 250 ignore.append(rest)
bos@71 251 elif hunk.strip():
bos@71 252 # it's something we should execute
bos@173 253 newps, output = self.sendreceive(hunk, read_hint)
bos@168 254 if not ofp:
bos@71 255 continue
bos@71 256 # first, print the command we ran
bos@71 257 if not hunk.startswith('#'):
bos@71 258 nl = hunk.endswith('\n')
bos@564 259 hunk = ('<prompt>%s</prompt> <userinput>%s</userinput>' %
bos@103 260 (prompts[ps],
bos@564 261 xml_escape(hunk.rstrip('\n'))))
bos@71 262 if nl: hunk += '\n'
bos@71 263 ofp.write(hunk)
bos@71 264 # then its output
bos@564 265 ofp.write(xml_escape(output))
bos@103 266 ps = newps
bos@71 267 self.status('\n')
bos@71 268 except:
bos@72 269 print >> sys.stderr, '(killed)'
bos@79 270 os.kill(self.pid, signal.SIGKILL)
bos@72 271 pid, rc = os.wait()
bos@71 272 raise
bos@72 273 else:
bos@71 274 try:
bos@173 275 ps, output = self.sendreceive('exit\n', read_hint)
bos@138 276 if ofp is not None:
bos@71 277 ofp.write(output)
bos@564 278 ofp.write('</screen>\n')
bos@138 279 ofp.close()
bos@160 280 err |= self.rename_output(ofp_basename, ignore)
bos@73 281 os.close(self.cfd)
bos@71 282 except IOError:
bos@71 283 pass
bos@79 284 os.kill(self.pid, signal.SIGTERM)
bos@72 285 pid, rc = os.wait()
bos@160 286 err = err or rc
bos@160 287 if err:
bos@72 288 if os.WIFEXITED(rc):
bos@72 289 print >> sys.stderr, '(exit %s)' % os.WEXITSTATUS(rc)
bos@72 290 elif os.WIFSIGNALED(rc):
bos@72 291 print >> sys.stderr, '(signal %s)' % os.WTERMSIG(rc)
bos@136 292 else:
bos@564 293 open(result_name(self.name + '.run'), 'w')
bos@160 294 return err
bos@72 295 finally:
bos@4 296 shutil.rmtree(tmpdir)
bos@3 297
bos@137 298 def rename_output(self, base, ignore):
bos@137 299 mangle_re = re.compile('(?:' + '|'.join(ignore) + ')')
bos@137 300 def mangle(s):
bos@137 301 return mangle_re.sub('', s)
bos@137 302 def matchfp(fp1, fp2):
bos@137 303 while True:
bos@137 304 s1 = mangle(fp1.readline())
bos@137 305 s2 = mangle(fp2.readline())
bos@137 306 if cmp(s1, s2):
bos@137 307 break
bos@137 308 if not s1:
bos@137 309 return True
bos@137 310 return False
bos@137 311
bos@564 312 oldname = result_name(base + '.out')
bos@564 313 tmpname = result_name(base + '.tmp')
bos@564 314 errname = result_name(base + '.err')
bos@137 315 errfp = open(errname, 'w+')
bos@137 316 for line in open(tmpname):
bos@137 317 errfp.write(mangle_re.sub('', line))
bos@564 318 os.rename(tmpname, result_name(base + '.lxo'))
bos@137 319 errfp.seek(0)
bos@137 320 try:
bos@137 321 oldfp = open(oldname)
bos@137 322 except IOError, err:
bos@137 323 if err.errno != errno.ENOENT:
bos@137 324 raise
bos@137 325 os.rename(errname, oldname)
bos@160 326 return False
bos@137 327 if matchfp(oldfp, errfp):
bos@137 328 os.unlink(errname)
bos@160 329 return False
bos@137 330 else:
bos@137 331 print >> sys.stderr, '\nOutput of %s has changed!' % base
bos@545 332 if self.keep_change:
bos@545 333 os.rename(errname, oldname)
bos@545 334 return False
bos@545 335 else:
bos@545 336 os.system('diff -u %s %s 1>&2' % (oldname, errname))
bos@138 337 return True
bos@137 338
bos@258 339 def print_help(exit, msg=None):
bos@258 340 if msg:
bos@258 341 print >> sys.stderr, 'Error:', msg
bos@258 342 print >> sys.stderr, 'Usage: run-example [options] [test...]'
bos@258 343 print >> sys.stderr, 'Options:'
bos@258 344 print >> sys.stderr, ' -a --all run all tests in this directory'
bos@258 345 print >> sys.stderr, ' -h --help print this help message'
bos@545 346 print >> sys.stderr, ' --help keep new output as desired output'
bos@258 347 print >> sys.stderr, ' -v --verbose display extra debug output'
bos@258 348 sys.exit(exit)
bos@258 349
bos@3 350 def main(path='.'):
bos@258 351 opts, args = getopt.getopt(sys.argv[1:], '?ahv',
bos@545 352 ['all', 'help', 'keep', 'verbose'])
bos@78 353 verbose = False
bos@258 354 run_all = False
bos@545 355 keep_change = False
bos@78 356 for o, a in opts:
bos@258 357 if o in ('-h', '-?', '--help'):
bos@258 358 print_help(0)
bos@258 359 if o in ('-a', '--all'):
bos@258 360 run_all = True
bos@545 361 if o in ('--keep',):
bos@545 362 keep_change = True
bos@78 363 if o in ('-v', '--verbose'):
bos@78 364 verbose = True
bos@71 365 errs = 0
bos@3 366 if args:
bos@3 367 for a in args:
bos@75 368 try:
bos@75 369 st = os.lstat(a)
bos@75 370 except OSError, err:
bos@75 371 print >> sys.stderr, '%s: %s' % (a, err.strerror)
bos@75 372 errs += 1
bos@75 373 continue
bos@75 374 if stat.S_ISREG(st.st_mode) and st.st_mode & 0111:
bos@545 375 if example(a, verbose, keep_change).run():
bos@75 376 errs += 1
bos@75 377 else:
bos@75 378 print >> sys.stderr, '%s: not a file, or not executable' % a
bos@71 379 errs += 1
bos@258 380 elif run_all:
bos@258 381 names = os.listdir(path)
bos@258 382 names.sort()
bos@258 383 for name in names:
bos@258 384 if name == 'run-example' or name.startswith('.'): continue
bos@564 385 if name.endswith('~'): continue
bos@258 386 pathname = os.path.join(path, name)
bos@258 387 try:
bos@258 388 st = os.lstat(pathname)
bos@258 389 except OSError, err:
bos@258 390 # could be an output file that was removed while we ran
bos@258 391 if err.errno != errno.ENOENT:
bos@258 392 raise
bos@258 393 continue
bos@258 394 if stat.S_ISREG(st.st_mode) and st.st_mode & 0111:
bos@545 395 if example(pathname, verbose, keep_change).run():
bos@258 396 errs += 1
bos@258 397 print >> open(os.path.join(path, '.run'), 'w'), time.asctime()
bos@258 398 else:
bos@258 399 print_help(1, msg='no test names given, and --all not provided')
bos@71 400 return errs
bos@3 401
bos@3 402 if __name__ == '__main__':
bos@258 403 try:
bos@258 404 sys.exit(main())
bos@258 405 except KeyboardInterrupt:
bos@258 406 print >> sys.stderr, 'interrupted!'
bos@258 407 sys.exit(1)