hgbook
view en/examples/run-example @ 609:c44d5854620b
Fix up chapter 1.
| author | Bryan O'Sullivan <bos@serpentine.com> |
|---|---|
| date | Tue Mar 31 22:38:30 2009 -0700 (2009-03-31) |
| parents | 8366882f67f2 |
| children | c82ff69f0935 |
line source
1 #!/usr/bin/env python
2 #
3 # This program takes something that resembles a shell script and runs
4 # it, spitting input (commands from the script) and output into text
5 # files, for use in examples.
7 import cStringIO
8 import errno
9 import getopt
10 import glob
11 import os
12 import pty
13 import re
14 import select
15 import shutil
16 import signal
17 import stat
18 import sys
19 import tempfile
20 import time
22 xml_subs = {
23 '<': '<',
24 '>': '>',
25 '&': '&',
26 }
28 def gensubs(s):
29 start = 0
30 for i, c in enumerate(s):
31 sub = xml_subs.get(c)
32 if sub:
33 yield s[start:i]
34 start = i + 1
35 yield sub
36 yield s[start:]
38 def xml_escape(s):
39 return ''.join(gensubs(s))
41 def maybe_unlink(name):
42 try:
43 os.unlink(name)
44 return True
45 except OSError, err:
46 if err.errno != errno.ENOENT:
47 raise
48 return False
50 def find_path_to(program):
51 for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
52 name = os.path.join(p, program)
53 if os.access(name, os.X_OK):
54 return p
55 return None
57 def result_name(name):
58 return os.path.normpath(os.path.join('results', name.replace(os.sep, '-')))
60 def wopen(name):
61 path = os.path.dirname(name)
62 if path:
63 try:
64 os.makedirs(path)
65 except OSError, err:
66 if err.errno != errno.EEXIST:
67 raise
68 return open(name, 'w')
70 class example:
71 entities = dict.fromkeys(l.rstrip() for l in open('auto-snippets.xml'))
73 def __init__(self, name, verbose, keep_change):
74 self.name = os.path.normpath(name)
75 self.verbose = verbose
76 self.keep_change = keep_change
78 def status(self, s):
79 sys.stdout.write(s)
80 if not s.endswith('\n'):
81 sys.stdout.flush()
83 def rename_output(self, base, ignore=[]):
84 mangle_re = re.compile('(?:' + '|'.join(ignore) + ')')
85 def mangle(s):
86 return mangle_re.sub('', s)
87 def matchfp(fp1, fp2):
88 while True:
89 s1 = mangle(fp1.readline())
90 s2 = mangle(fp2.readline())
91 if cmp(s1, s2):
92 break
93 if not s1:
94 return True
95 return False
97 oldname = result_name(base + '.out')
98 tmpname = result_name(base + '.tmp')
99 errname = result_name(base + '.err')
100 errfp = open(errname, 'w+')
101 for line in open(tmpname):
102 errfp.write(mangle_re.sub('', line))
103 os.rename(tmpname, result_name(base + '.lxo'))
104 errfp.seek(0)
105 try:
106 oldfp = open(oldname)
107 except IOError, err:
108 if err.errno != errno.ENOENT:
109 raise
110 os.rename(errname, oldname)
111 return False
112 if matchfp(oldfp, errfp):
113 os.unlink(errname)
114 return False
115 else:
116 print >> sys.stderr, '\nOutput of %s has changed!' % base
117 if self.keep_change:
118 os.rename(errname, oldname)
119 return False
120 else:
121 os.system('diff -u %s %s 1>&2' % (oldname, errname))
122 return True
124 class static_example(example):
125 def run(self):
126 self.status('running %s\n' % self.name)
127 s = open(self.name).read().rstrip()
128 s = s.replace('&', '&').replace('<', '<').replace('>', '>')
129 ofp = wopen(result_name(self.name + '.tmp'))
130 ofp.write('<programlisting>')
131 ofp.write(s)
132 ofp.write('</programlisting>\n')
133 ofp.close()
134 self.rename_output(self.name)
135 norm = self.name.replace(os.sep, '-')
136 example.entities[
137 '<!ENTITY %s SYSTEM "results/%s.lxo">' % (norm, norm)] = 1
140 class shell_example(example):
141 shell = '/usr/bin/env bash'
142 ps1 = '__run_example_ps1__ '
143 ps2 = '__run_example_ps2__ '
144 pi_re = re.compile(r'#\$\s*(name|ignore):\s*(.*)$')
146 timeout = 10
148 def __init__(self, name, verbose, keep_change):
149 example.__init__(self, name, verbose, keep_change)
150 self.poll = select.poll()
152 def parse(self):
153 '''yield each hunk of input from the file.'''
154 fp = open(self.name)
155 cfp = cStringIO.StringIO()
156 for line in fp:
157 cfp.write(line)
158 if not line.rstrip().endswith('\\'):
159 yield cfp.getvalue()
160 cfp.seek(0)
161 cfp.truncate()
163 def send(self, s):
164 if self.verbose:
165 print >> sys.stderr, '>', self.debugrepr(s)
166 while s:
167 count = os.write(self.cfd, s)
168 s = s[count:]
170 def debugrepr(self, s):
171 rs = repr(s)
172 limit = 60
173 if len(rs) > limit:
174 return ('%s%s ... [%d bytes]' % (rs[:limit], rs[0], len(s)))
175 else:
176 return rs
178 timeout = 5
180 def read(self, hint):
181 events = self.poll.poll(self.timeout * 1000)
182 if not events:
183 print >> sys.stderr, ('[%stimed out after %d seconds]' %
184 (hint, self.timeout))
185 os.kill(self.pid, signal.SIGHUP)
186 return ''
187 return os.read(self.cfd, 1024)
189 def receive(self, hint):
190 out = cStringIO.StringIO()
191 while True:
192 try:
193 if self.verbose:
194 sys.stderr.write('< ')
195 s = self.read(hint)
196 except OSError, err:
197 if err.errno == errno.EIO:
198 return '', ''
199 raise
200 if self.verbose:
201 print >> sys.stderr, self.debugrepr(s)
202 out.write(s)
203 s = out.getvalue()
204 if s.endswith(self.ps1):
205 return self.ps1, s.replace('\r\n', '\n')[:-len(self.ps1)]
206 if s.endswith(self.ps2):
207 return self.ps2, s.replace('\r\n', '\n')[:-len(self.ps2)]
209 def sendreceive(self, s, hint):
210 self.send(s)
211 ps, r = self.receive(hint)
212 if r.startswith(s):
213 r = r[len(s):]
214 return ps, r
216 def run(self):
217 ofp = None
218 basename = os.path.basename(self.name)
219 self.status('running %s ' % basename)
220 tmpdir = tempfile.mkdtemp(prefix=basename)
222 # remove the marker file that we tell make to use to see if
223 # this run succeeded
224 maybe_unlink(self.name + '.run')
226 rcfile = os.path.join(tmpdir, '.hgrc')
227 rcfp = wopen(rcfile)
228 print >> rcfp, '[ui]'
229 print >> rcfp, "username = Bryan O'Sullivan <bos@serpentine.com>"
231 rcfile = os.path.join(tmpdir, '.bashrc')
232 rcfp = wopen(rcfile)
233 print >> rcfp, 'PS1="%s"' % self.ps1
234 print >> rcfp, 'PS2="%s"' % self.ps2
235 print >> rcfp, 'unset HISTFILE'
236 path = ['/usr/bin', '/bin']
237 hg = find_path_to('hg')
238 if hg and hg not in path:
239 path.append(hg)
240 def re_export(envar):
241 v = os.getenv(envar)
242 if v is not None:
243 print >> rcfp, 'export ' + envar + '=' + v
244 print >> rcfp, 'export PATH=' + ':'.join(path)
245 re_export('PYTHONPATH')
246 print >> rcfp, 'export EXAMPLE_DIR="%s"' % os.getcwd()
247 print >> rcfp, 'export HGMERGE=merge'
248 print >> rcfp, 'export LANG=C'
249 print >> rcfp, 'export LC_ALL=C'
250 print >> rcfp, 'export TZ=GMT'
251 print >> rcfp, 'export HGRC="%s/.hgrc"' % tmpdir
252 print >> rcfp, 'export HGRCPATH=$HGRC'
253 print >> rcfp, 'cd %s' % tmpdir
254 rcfp.close()
255 sys.stdout.flush()
256 sys.stderr.flush()
257 self.pid, self.cfd = pty.fork()
258 if self.pid == 0:
259 cmdline = ['/usr/bin/env', '-i', 'bash', '--noediting',
260 '--noprofile', '--norc']
261 try:
262 os.execv(cmdline[0], cmdline)
263 except OSError, err:
264 print >> sys.stderr, '%s: %s' % (cmdline[0], err.strerror)
265 sys.stderr.flush()
266 os._exit(0)
267 self.poll.register(self.cfd, select.POLLIN | select.POLLERR |
268 select.POLLHUP)
270 prompts = {
271 '': '',
272 self.ps1: '$',
273 self.ps2: '>',
274 }
276 ignore = [
277 r'\d+:[0-9a-f]{12}', # changeset number:hash
278 r'[0-9a-f]{40}', # long changeset hash
279 r'[0-9a-f]{12}', # short changeset hash
280 r'^(?:---|\+\+\+) .*', # diff header with dates
281 r'^date:.*', # date
282 #r'^diff -r.*', # "diff -r" is followed by hash
283 r'^# Date \d+ \d+', # hg patch header
284 ]
286 err = False
287 read_hint = ''
289 try:
290 try:
291 # eat first prompt string from shell
292 self.read(read_hint)
293 # setup env and prompt
294 ps, output = self.sendreceive('source %s\n' % rcfile,
295 read_hint)
296 for hunk in self.parse():
297 # is this line a processing instruction?
298 m = self.pi_re.match(hunk)
299 if m:
300 pi, rest = m.groups()
301 if pi == 'name':
302 self.status('.')
303 out = rest
304 if out in ('err', 'lxo', 'out', 'run', 'tmp'):
305 print >> sys.stderr, ('%s: illegal section '
306 'name %r' %
307 (self.name, out))
308 return 1
309 assert os.sep not in out
310 if ofp is not None:
311 ofp.write('</screen>\n')
312 ofp.close()
313 err |= self.rename_output(ofp_basename, ignore)
314 if out:
315 ofp_basename = '%s.%s' % (self.name, out)
316 norm = os.path.normpath(ofp_basename)
317 example.entities[
318 '<!ENTITY interaction.%s '
319 'SYSTEM "results/%s.lxo">'
320 % (norm, norm)] = 1
321 read_hint = ofp_basename + ' '
322 ofp = wopen(result_name(ofp_basename + '.tmp'))
323 ofp.write('<screen>')
324 else:
325 ofp = None
326 elif pi == 'ignore':
327 ignore.append(rest)
328 elif hunk.strip():
329 # it's something we should execute
330 newps, output = self.sendreceive(hunk, read_hint)
331 if not ofp:
332 continue
333 # first, print the command we ran
334 if not hunk.startswith('#'):
335 nl = hunk.endswith('\n')
336 hunk = ('<prompt>%s</prompt> <userinput>%s</userinput>' %
337 (prompts[ps],
338 xml_escape(hunk.rstrip('\n'))))
339 if nl: hunk += '\n'
340 ofp.write(hunk)
341 # then its output
342 ofp.write(xml_escape(output))
343 ps = newps
344 self.status('\n')
345 except:
346 print >> sys.stderr, '(killed)'
347 os.kill(self.pid, signal.SIGKILL)
348 pid, rc = os.wait()
349 raise
350 else:
351 try:
352 ps, output = self.sendreceive('exit\n', read_hint)
353 if ofp is not None:
354 ofp.write(output)
355 ofp.write('</screen>\n')
356 ofp.close()
357 err |= self.rename_output(ofp_basename, ignore)
358 os.close(self.cfd)
359 except IOError:
360 pass
361 os.kill(self.pid, signal.SIGTERM)
362 pid, rc = os.wait()
363 err = err or rc
364 if err:
365 if os.WIFEXITED(rc):
366 print >> sys.stderr, '(exit %s)' % os.WEXITSTATUS(rc)
367 elif os.WIFSIGNALED(rc):
368 print >> sys.stderr, '(signal %s)' % os.WTERMSIG(rc)
369 else:
370 wopen(result_name(self.name + '.run'))
371 return err
372 finally:
373 shutil.rmtree(tmpdir)
375 def print_help(exit, msg=None):
376 if msg:
377 print >> sys.stderr, 'Error:', msg
378 print >> sys.stderr, 'Usage: run-example [options] [test...]'
379 print >> sys.stderr, 'Options:'
380 print >> sys.stderr, ' -a --all run all examples in this directory'
381 print >> sys.stderr, ' -h --help print this help message'
382 print >> sys.stderr, ' --help keep new output as desired output'
383 print >> sys.stderr, ' -v --verbose display extra debug output'
384 sys.exit(exit)
386 def main(path='.'):
387 if os.path.realpath(path).split(os.sep)[-1] != 'examples':
388 print >> sys.stderr, 'Not being run from the examples directory!'
389 sys.exit(1)
391 opts, args = getopt.getopt(sys.argv[1:], '?ahv',
392 ['all', 'help', 'keep', 'verbose'])
393 verbose = False
394 run_all = False
395 keep_change = False
397 for o, a in opts:
398 if o in ('-h', '-?', '--help'):
399 print_help(0)
400 if o in ('-a', '--all'):
401 run_all = True
402 if o in ('--keep',):
403 keep_change = True
404 if o in ('-v', '--verbose'):
405 verbose = True
406 errs = 0
407 if args:
408 for a in args:
409 try:
410 st = os.lstat(a)
411 except OSError, err:
412 print >> sys.stderr, '%s: %s' % (a, err.strerror)
413 errs += 1
414 continue
415 if stat.S_ISREG(st.st_mode):
416 if st.st_mode & 0111:
417 if shell_example(a, verbose, keep_change).run():
418 errs += 1
419 elif a.endswith('.lst'):
420 static_example(a, verbose, keep_change).run()
421 else:
422 print >> sys.stderr, '%s: not a file, or not executable' % a
423 errs += 1
424 elif run_all:
425 names = glob.glob("*") + glob.glob("app*/*") + glob.glob("ch*/*")
426 names.sort()
427 for name in names:
428 if name == 'run-example' or name.endswith('~'): continue
429 pathname = os.path.join(path, name)
430 try:
431 st = os.lstat(pathname)
432 except OSError, err:
433 # could be an output file that was removed while we ran
434 if err.errno != errno.ENOENT:
435 raise
436 continue
437 if stat.S_ISREG(st.st_mode):
438 if st.st_mode & 0111:
439 if shell_example(pathname, verbose, keep_change).run():
440 errs += 1
441 elif pathname.endswith('.lst'):
442 static_example(pathname, verbose, keep_change).run()
443 print >> wopen(os.path.join(path, '.run')), time.asctime()
444 else:
445 print_help(1, msg='no test names given, and --all not provided')
447 fp = wopen('auto-snippets.xml')
448 for key in sorted(example.entities.iterkeys()):
449 print >> fp, key
450 fp.close()
451 return errs
453 if __name__ == '__main__':
454 try:
455 sys.exit(main())
456 except KeyboardInterrupt:
457 print >> sys.stderr, 'interrupted!'
458 sys.exit(1)