hgbook

annotate en/examples/run-example @ 732:1c31e38a9720

find changesets by author, revision, files, or words in the commit message
Complete revision of Ch.2.
author Giulio@puck
date Sun Jun 28 17:18:56 2009 +0200 (2009-06-28)
parents c82ff69f0935
children
rev   line source
bos@67 1 #!/usr/bin/env python
bos@4 2 #
bos@4 3 # This program takes something that resembles a shell script and runs
bos@4 4 # it, spitting input (commands from the script) and output into text
bos@4 5 # files, for use in examples.
bos@3 6
bos@3 7 import cStringIO
bos@73 8 import errno
bos@78 9 import getopt
bos@580 10 import glob
bos@3 11 import os
bos@3 12 import pty
bos@3 13 import re
bos@79 14 import select
bos@4 15 import shutil
bos@6 16 import signal
bos@36 17 import stat
bos@3 18 import sys
bos@4 19 import tempfile
bos@4 20 import time
bos@3 21
bos@564 22 xml_subs = {
bos@564 23 '<': '&lt;',
bos@564 24 '>': '&gt;',
bos@564 25 '&': '&amp;',
bos@77 26 }
bos@77 27
bos@77 28 def gensubs(s):
bos@77 29 start = 0
bos@77 30 for i, c in enumerate(s):
bos@564 31 sub = xml_subs.get(c)
bos@77 32 if sub:
bos@77 33 yield s[start:i]
bos@77 34 start = i + 1
bos@77 35 yield sub
bos@77 36 yield s[start:]
bos@77 37
bos@564 38 def xml_escape(s):
bos@77 39 return ''.join(gensubs(s))
bos@4 40
bos@137 41 def maybe_unlink(name):
bos@137 42 try:
bos@137 43 os.unlink(name)
bos@137 44 return True
bos@137 45 except OSError, err:
bos@137 46 if err.errno != errno.ENOENT:
bos@137 47 raise
bos@137 48 return False
bos@137 49
bos@172 50 def find_path_to(program):
bos@172 51 for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
bos@172 52 name = os.path.join(p, program)
bos@172 53 if os.access(name, os.X_OK):
bos@172 54 return p
bos@172 55 return None
bos@172 56
bos@564 57 def result_name(name):
bos@580 58 return os.path.normpath(os.path.join('results', name.replace(os.sep, '-')))
bos@580 59
bos@580 60 def wopen(name):
bos@580 61 path = os.path.dirname(name)
bos@580 62 if path:
bos@580 63 try:
bos@580 64 os.makedirs(path)
bos@580 65 except OSError, err:
bos@580 66 if err.errno != errno.EEXIST:
bos@580 67 raise
bos@580 68 return open(name, 'w')
bos@564 69
bos@3 70 class example:
bos@579 71 entities = dict.fromkeys(l.rstrip() for l in open('auto-snippets.xml'))
bos@579 72
bos@579 73 def __init__(self, name, verbose, keep_change):
bos@580 74 self.name = os.path.normpath(name)
bos@579 75 self.verbose = verbose
bos@579 76 self.keep_change = keep_change
bos@580 77
bos@580 78 def status(self, s):
bos@580 79 sys.stdout.write(s)
bos@580 80 if not s.endswith('\n'):
bos@580 81 sys.stdout.flush()
bos@579 82
bos@579 83 def rename_output(self, base, ignore=[]):
bos@579 84 mangle_re = re.compile('(?:' + '|'.join(ignore) + ')')
bos@579 85 def mangle(s):
bos@579 86 return mangle_re.sub('', s)
bos@579 87 def matchfp(fp1, fp2):
bos@579 88 while True:
bos@579 89 s1 = mangle(fp1.readline())
bos@579 90 s2 = mangle(fp2.readline())
bos@579 91 if cmp(s1, s2):
bos@579 92 break
bos@579 93 if not s1:
bos@579 94 return True
bos@579 95 return False
bos@579 96
bos@579 97 oldname = result_name(base + '.out')
bos@579 98 tmpname = result_name(base + '.tmp')
bos@579 99 errname = result_name(base + '.err')
bos@579 100 errfp = open(errname, 'w+')
bos@579 101 for line in open(tmpname):
bos@579 102 errfp.write(mangle_re.sub('', line))
bos@579 103 os.rename(tmpname, result_name(base + '.lxo'))
bos@579 104 errfp.seek(0)
bos@579 105 try:
bos@579 106 oldfp = open(oldname)
bos@579 107 except IOError, err:
bos@579 108 if err.errno != errno.ENOENT:
bos@579 109 raise
bos@579 110 os.rename(errname, oldname)
bos@579 111 return False
bos@579 112 if matchfp(oldfp, errfp):
bos@579 113 os.unlink(errname)
bos@579 114 return False
bos@579 115 else:
bos@580 116 print >> sys.stderr, '\nOutput of %s has changed!' % base
bos@579 117 if self.keep_change:
bos@579 118 os.rename(errname, oldname)
bos@579 119 return False
bos@579 120 else:
bos@579 121 os.system('diff -u %s %s 1>&2' % (oldname, errname))
bos@579 122 return True
bos@579 123
bos@579 124 class static_example(example):
bos@579 125 def run(self):
bos@580 126 self.status('running %s\n' % self.name)
bos@579 127 s = open(self.name).read().rstrip()
bos@579 128 s = s.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
bos@579 129 ofp = wopen(result_name(self.name + '.tmp'))
bos@610 130 ofp.write('<!-- BEGIN %s -->\n' % self.name)
bos@579 131 ofp.write('<programlisting>')
bos@579 132 ofp.write(s)
bos@579 133 ofp.write('</programlisting>\n')
bos@610 134 ofp.write('<!-- END %s -->\n' % self.name)
bos@579 135 ofp.close()
bos@579 136 self.rename_output(self.name)
bos@579 137 norm = self.name.replace(os.sep, '-')
bos@579 138 example.entities[
bos@590 139 '<!ENTITY %s SYSTEM "results/%s.lxo">' % (norm, norm)] = 1
bos@579 140
bos@579 141
bos@579 142 class shell_example(example):
bos@70 143 shell = '/usr/bin/env bash'
bos@103 144 ps1 = '__run_example_ps1__ '
bos@103 145 ps2 = '__run_example_ps2__ '
bos@168 146 pi_re = re.compile(r'#\$\s*(name|ignore):\s*(.*)$')
bos@4 147
bos@192 148 timeout = 10
bos@79 149
bos@545 150 def __init__(self, name, verbose, keep_change):
bos@579 151 example.__init__(self, name, verbose, keep_change)
bos@79 152 self.poll = select.poll()
bos@3 153
bos@3 154 def parse(self):
bos@4 155 '''yield each hunk of input from the file.'''
bos@3 156 fp = open(self.name)
bos@3 157 cfp = cStringIO.StringIO()
bos@3 158 for line in fp:
bos@3 159 cfp.write(line)
bos@3 160 if not line.rstrip().endswith('\\'):
bos@3 161 yield cfp.getvalue()
bos@3 162 cfp.seek(0)
bos@3 163 cfp.truncate()
bos@3 164
bos@6 165 def send(self, s):
bos@78 166 if self.verbose:
bos@78 167 print >> sys.stderr, '>', self.debugrepr(s)
bos@73 168 while s:
bos@73 169 count = os.write(self.cfd, s)
bos@73 170 s = s[count:]
bos@6 171
bos@78 172 def debugrepr(self, s):
bos@78 173 rs = repr(s)
bos@78 174 limit = 60
bos@78 175 if len(rs) > limit:
bos@78 176 return ('%s%s ... [%d bytes]' % (rs[:limit], rs[0], len(s)))
bos@78 177 else:
bos@78 178 return rs
bos@78 179
bos@79 180 timeout = 5
bos@79 181
bos@173 182 def read(self, hint):
bos@79 183 events = self.poll.poll(self.timeout * 1000)
bos@79 184 if not events:
bos@173 185 print >> sys.stderr, ('[%stimed out after %d seconds]' %
bos@173 186 (hint, self.timeout))
bos@79 187 os.kill(self.pid, signal.SIGHUP)
bos@79 188 return ''
bos@79 189 return os.read(self.cfd, 1024)
bos@79 190
bos@173 191 def receive(self, hint):
bos@6 192 out = cStringIO.StringIO()
bos@4 193 while True:
bos@73 194 try:
bos@78 195 if self.verbose:
bos@78 196 sys.stderr.write('< ')
bos@173 197 s = self.read(hint)
bos@73 198 except OSError, err:
bos@73 199 if err.errno == errno.EIO:
bos@103 200 return '', ''
bos@73 201 raise
bos@78 202 if self.verbose:
bos@78 203 print >> sys.stderr, self.debugrepr(s)
bos@6 204 out.write(s)
bos@73 205 s = out.getvalue()
bos@103 206 if s.endswith(self.ps1):
bos@103 207 return self.ps1, s.replace('\r\n', '\n')[:-len(self.ps1)]
bos@103 208 if s.endswith(self.ps2):
bos@103 209 return self.ps2, s.replace('\r\n', '\n')[:-len(self.ps2)]
bos@4 210
bos@173 211 def sendreceive(self, s, hint):
bos@6 212 self.send(s)
bos@173 213 ps, r = self.receive(hint)
bos@6 214 if r.startswith(s):
bos@6 215 r = r[len(s):]
bos@103 216 return ps, r
bos@6 217
bos@3 218 def run(self):
bos@3 219 ofp = None
bos@4 220 basename = os.path.basename(self.name)
bos@4 221 self.status('running %s ' % basename)
bos@4 222 tmpdir = tempfile.mkdtemp(prefix=basename)
bos@78 223
bos@136 224 # remove the marker file that we tell make to use to see if
bos@136 225 # this run succeeded
bos@137 226 maybe_unlink(self.name + '.run')
bos@136 227
bos@78 228 rcfile = os.path.join(tmpdir, '.hgrc')
bos@579 229 rcfp = wopen(rcfile)
bos@78 230 print >> rcfp, '[ui]'
bos@78 231 print >> rcfp, "username = Bryan O'Sullivan <bos@serpentine.com>"
bos@78 232
bos@6 233 rcfile = os.path.join(tmpdir, '.bashrc')
bos@579 234 rcfp = wopen(rcfile)
bos@103 235 print >> rcfp, 'PS1="%s"' % self.ps1
bos@103 236 print >> rcfp, 'PS2="%s"' % self.ps2
bos@6 237 print >> rcfp, 'unset HISTFILE'
bos@172 238 path = ['/usr/bin', '/bin']
bos@172 239 hg = find_path_to('hg')
bos@172 240 if hg and hg not in path:
bos@172 241 path.append(hg)
bos@172 242 def re_export(envar):
bos@172 243 v = os.getenv(envar)
bos@172 244 if v is not None:
bos@172 245 print >> rcfp, 'export ' + envar + '=' + v
bos@172 246 print >> rcfp, 'export PATH=' + ':'.join(path)
bos@172 247 re_export('PYTHONPATH')
bos@19 248 print >> rcfp, 'export EXAMPLE_DIR="%s"' % os.getcwd()
bos@124 249 print >> rcfp, 'export HGMERGE=merge'
bos@6 250 print >> rcfp, 'export LANG=C'
bos@6 251 print >> rcfp, 'export LC_ALL=C'
bos@6 252 print >> rcfp, 'export TZ=GMT'
bos@6 253 print >> rcfp, 'export HGRC="%s/.hgrc"' % tmpdir
bos@6 254 print >> rcfp, 'export HGRCPATH=$HGRC'
bos@6 255 print >> rcfp, 'cd %s' % tmpdir
bos@6 256 rcfp.close()
bos@68 257 sys.stdout.flush()
bos@68 258 sys.stderr.flush()
bos@79 259 self.pid, self.cfd = pty.fork()
bos@79 260 if self.pid == 0:
bos@172 261 cmdline = ['/usr/bin/env', '-i', 'bash', '--noediting',
bos@172 262 '--noprofile', '--norc']
bos@68 263 try:
bos@68 264 os.execv(cmdline[0], cmdline)
bos@68 265 except OSError, err:
bos@68 266 print >> sys.stderr, '%s: %s' % (cmdline[0], err.strerror)
bos@68 267 sys.stderr.flush()
bos@68 268 os._exit(0)
bos@79 269 self.poll.register(self.cfd, select.POLLIN | select.POLLERR |
bos@79 270 select.POLLHUP)
bos@103 271
bos@103 272 prompts = {
bos@103 273 '': '',
bos@103 274 self.ps1: '$',
bos@103 275 self.ps2: '>',
bos@103 276 }
bos@103 277
bos@137 278 ignore = [
bos@137 279 r'\d+:[0-9a-f]{12}', # changeset number:hash
bos@141 280 r'[0-9a-f]{40}', # long changeset hash
bos@141 281 r'[0-9a-f]{12}', # short changeset hash
bos@137 282 r'^(?:---|\+\+\+) .*', # diff header with dates
bos@137 283 r'^date:.*', # date
bos@141 284 #r'^diff -r.*', # "diff -r" is followed by hash
bos@138 285 r'^# Date \d+ \d+', # hg patch header
bos@137 286 ]
bos@137 287
bos@138 288 err = False
bos@173 289 read_hint = ''
bos@138 290
bos@4 291 try:
bos@71 292 try:
bos@73 293 # eat first prompt string from shell
bos@173 294 self.read(read_hint)
bos@71 295 # setup env and prompt
bos@173 296 ps, output = self.sendreceive('source %s\n' % rcfile,
bos@173 297 read_hint)
bos@71 298 for hunk in self.parse():
bos@71 299 # is this line a processing instruction?
bos@71 300 m = self.pi_re.match(hunk)
bos@71 301 if m:
bos@71 302 pi, rest = m.groups()
bos@71 303 if pi == 'name':
bos@71 304 self.status('.')
bos@71 305 out = rest
bos@155 306 if out in ('err', 'lxo', 'out', 'run', 'tmp'):
bos@155 307 print >> sys.stderr, ('%s: illegal section '
bos@155 308 'name %r' %
bos@155 309 (self.name, out))
bos@155 310 return 1
bos@71 311 assert os.sep not in out
bos@137 312 if ofp is not None:
bos@564 313 ofp.write('</screen>\n')
bos@610 314 ofp.write('<!-- END %s -->\n' % ofp_basename)
bos@137 315 ofp.close()
bos@160 316 err |= self.rename_output(ofp_basename, ignore)
bos@71 317 if out:
bos@137 318 ofp_basename = '%s.%s' % (self.name, out)
bos@566 319 norm = os.path.normpath(ofp_basename)
bos@674 320 norm = norm.replace(os.sep, '-')
bos@566 321 example.entities[
bos@569 322 '<!ENTITY interaction.%s '
bos@590 323 'SYSTEM "results/%s.lxo">'
bos@566 324 % (norm, norm)] = 1
bos@173 325 read_hint = ofp_basename + ' '
bos@579 326 ofp = wopen(result_name(ofp_basename + '.tmp'))
bos@610 327 ofp.write('<!-- BEGIN %s -->\n' % ofp_basename)
bos@564 328 ofp.write('<screen>')
bos@71 329 else:
bos@71 330 ofp = None
bos@137 331 elif pi == 'ignore':
bos@137 332 ignore.append(rest)
bos@71 333 elif hunk.strip():
bos@71 334 # it's something we should execute
bos@173 335 newps, output = self.sendreceive(hunk, read_hint)
bos@168 336 if not ofp:
bos@71 337 continue
bos@71 338 # first, print the command we ran
bos@71 339 if not hunk.startswith('#'):
bos@71 340 nl = hunk.endswith('\n')
bos@610 341 hunk = ('<prompt>%s</prompt> '
bos@610 342 '<userinput>%s</userinput>' %
bos@103 343 (prompts[ps],
bos@564 344 xml_escape(hunk.rstrip('\n'))))
bos@71 345 if nl: hunk += '\n'
bos@71 346 ofp.write(hunk)
bos@71 347 # then its output
bos@564 348 ofp.write(xml_escape(output))
bos@674 349 ps = newps
bos@71 350 self.status('\n')
bos@71 351 except:
bos@72 352 print >> sys.stderr, '(killed)'
bos@79 353 os.kill(self.pid, signal.SIGKILL)
bos@72 354 pid, rc = os.wait()
bos@71 355 raise
bos@72 356 else:
bos@71 357 try:
bos@173 358 ps, output = self.sendreceive('exit\n', read_hint)
bos@138 359 if ofp is not None:
bos@71 360 ofp.write(output)
bos@564 361 ofp.write('</screen>\n')
bos@610 362 ofp.write('<!-- END %s -->\n' % ofp_basename)
bos@138 363 ofp.close()
bos@160 364 err |= self.rename_output(ofp_basename, ignore)
bos@73 365 os.close(self.cfd)
bos@71 366 except IOError:
bos@71 367 pass
bos@79 368 os.kill(self.pid, signal.SIGTERM)
bos@72 369 pid, rc = os.wait()
bos@160 370 err = err or rc
bos@160 371 if err:
bos@72 372 if os.WIFEXITED(rc):
bos@72 373 print >> sys.stderr, '(exit %s)' % os.WEXITSTATUS(rc)
bos@72 374 elif os.WIFSIGNALED(rc):
bos@72 375 print >> sys.stderr, '(signal %s)' % os.WTERMSIG(rc)
bos@136 376 else:
bos@579 377 wopen(result_name(self.name + '.run'))
bos@160 378 return err
bos@72 379 finally:
bos@4 380 shutil.rmtree(tmpdir)
bos@3 381
bos@258 382 def print_help(exit, msg=None):
bos@258 383 if msg:
bos@258 384 print >> sys.stderr, 'Error:', msg
bos@258 385 print >> sys.stderr, 'Usage: run-example [options] [test...]'
bos@258 386 print >> sys.stderr, 'Options:'
bos@565 387 print >> sys.stderr, ' -a --all run all examples in this directory'
bos@258 388 print >> sys.stderr, ' -h --help print this help message'
bos@610 389 print >> sys.stderr, ' --keep keep new output as desired output'
bos@258 390 print >> sys.stderr, ' -v --verbose display extra debug output'
bos@258 391 sys.exit(exit)
bos@258 392
bos@3 393 def main(path='.'):
bos@566 394 if os.path.realpath(path).split(os.sep)[-1] != 'examples':
bos@566 395 print >> sys.stderr, 'Not being run from the examples directory!'
bos@566 396 sys.exit(1)
bos@566 397
bos@258 398 opts, args = getopt.getopt(sys.argv[1:], '?ahv',
bos@545 399 ['all', 'help', 'keep', 'verbose'])
bos@78 400 verbose = False
bos@258 401 run_all = False
bos@545 402 keep_change = False
bos@566 403
bos@78 404 for o, a in opts:
bos@258 405 if o in ('-h', '-?', '--help'):
bos@258 406 print_help(0)
bos@258 407 if o in ('-a', '--all'):
bos@258 408 run_all = True
bos@545 409 if o in ('--keep',):
bos@545 410 keep_change = True
bos@78 411 if o in ('-v', '--verbose'):
bos@78 412 verbose = True
bos@71 413 errs = 0
bos@3 414 if args:
bos@3 415 for a in args:
bos@75 416 try:
bos@75 417 st = os.lstat(a)
bos@75 418 except OSError, err:
bos@75 419 print >> sys.stderr, '%s: %s' % (a, err.strerror)
bos@75 420 errs += 1
bos@75 421 continue
bos@579 422 if stat.S_ISREG(st.st_mode):
bos@579 423 if st.st_mode & 0111:
bos@579 424 if shell_example(a, verbose, keep_change).run():
bos@579 425 errs += 1
bos@579 426 elif a.endswith('.lst'):
bos@579 427 static_example(a, verbose, keep_change).run()
bos@75 428 else:
bos@75 429 print >> sys.stderr, '%s: not a file, or not executable' % a
bos@71 430 errs += 1
bos@258 431 elif run_all:
bos@579 432 names = glob.glob("*") + glob.glob("app*/*") + glob.glob("ch*/*")
bos@258 433 names.sort()
bos@258 434 for name in names:
bos@579 435 if name == 'run-example' or name.endswith('~'): continue
bos@258 436 pathname = os.path.join(path, name)
bos@258 437 try:
bos@258 438 st = os.lstat(pathname)
bos@258 439 except OSError, err:
bos@258 440 # could be an output file that was removed while we ran
bos@258 441 if err.errno != errno.ENOENT:
bos@258 442 raise
bos@258 443 continue
bos@579 444 if stat.S_ISREG(st.st_mode):
bos@579 445 if st.st_mode & 0111:
bos@579 446 if shell_example(pathname, verbose, keep_change).run():
bos@579 447 errs += 1
bos@579 448 elif pathname.endswith('.lst'):
bos@579 449 static_example(pathname, verbose, keep_change).run()
bos@579 450 print >> wopen(os.path.join(path, '.run')), time.asctime()
bos@258 451 else:
bos@258 452 print_help(1, msg='no test names given, and --all not provided')
bos@566 453
bos@579 454 fp = wopen('auto-snippets.xml')
bos@566 455 for key in sorted(example.entities.iterkeys()):
bos@566 456 print >> fp, key
bos@566 457 fp.close()
bos@71 458 return errs
bos@3 459
bos@3 460 if __name__ == '__main__':
bos@258 461 try:
bos@258 462 sys.exit(main())
bos@258 463 except KeyboardInterrupt:
bos@258 464 print >> sys.stderr, 'interrupted!'
bos@258 465 sys.exit(1)