hgbook
view en/examples/run-example @ 579:80928ea6e7ae
Add the ability to include text files and have them XML-mangled.
| author | Bryan O'Sullivan <bos@serpentine.com> |
|---|---|
| date | Tue Mar 17 21:47:12 2009 -0700 (2009-03-17) |
| parents | 60ee738fdc0e |
| children | 8366882f67f2 |
line source
1 #!/usr/bin/env python
2 #
3 # This program takes something that resembles a shell script and runs
4 # it, spitting input (commands from the script) and output into text
5 # files, for use in examples.
7 import cStringIO
8 import errno
9 import getopt
10 import os
11 import pty
12 import re
13 import select
14 import shutil
15 import signal
16 import stat
17 import sys
18 import tempfile
19 import time
21 xml_subs = {
22 '<': '<',
23 '>': '>',
24 '&': '&',
25 }
27 def gensubs(s):
28 start = 0
29 for i, c in enumerate(s):
30 sub = xml_subs.get(c)
31 if sub:
32 yield s[start:i]
33 start = i + 1
34 yield sub
35 yield s[start:]
37 def xml_escape(s):
38 return ''.join(gensubs(s))
40 def maybe_unlink(name):
41 try:
42 os.unlink(name)
43 return True
44 except OSError, err:
45 if err.errno != errno.ENOENT:
46 raise
47 return False
49 def find_path_to(program):
50 for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
51 name = os.path.join(p, program)
52 if os.access(name, os.X_OK):
53 return p
54 return None
56 def result_name(name):
57 return os.path.join('results', name.replace(os.sep, '-'))
59 class example:
60 entities = dict.fromkeys(l.rstrip() for l in open('auto-snippets.xml'))
62 def __init__(self, name, verbose, keep_change):
63 self.name = name
64 self.verbose = verbose
65 self.keep_change = keep_change
67 def rename_output(self, base, ignore=[]):
68 mangle_re = re.compile('(?:' + '|'.join(ignore) + ')')
69 def mangle(s):
70 return mangle_re.sub('', s)
71 def matchfp(fp1, fp2):
72 while True:
73 s1 = mangle(fp1.readline())
74 s2 = mangle(fp2.readline())
75 if cmp(s1, s2):
76 break
77 if not s1:
78 return True
79 return False
81 oldname = result_name(base + '.out')
82 tmpname = result_name(base + '.tmp')
83 errname = result_name(base + '.err')
84 errfp = open(errname, 'w+')
85 for line in open(tmpname):
86 errfp.write(mangle_re.sub('', line))
87 os.rename(tmpname, result_name(base + '.lxo'))
88 errfp.seek(0)
89 try:
90 oldfp = open(oldname)
91 except IOError, err:
92 if err.errno != errno.ENOENT:
93 raise
94 os.rename(errname, oldname)
95 return False
96 if matchfp(oldfp, errfp):
97 os.unlink(errname)
98 return False
99 else:
100 print >> sys.stderr, '\nOutput of %s has changed!' % baseq
101 if self.keep_change:
102 os.rename(errname, oldname)
103 return False
104 else:
105 os.system('diff -u %s %s 1>&2' % (oldname, errname))
106 return True
108 def wopen(name):
109 path = os.path.dirname(name)
110 if path:
111 try:
112 os.makedirs(path)
113 except OSError, err:
114 if err.errno != errno.EEXIST:
115 raise
116 return open(name, 'w')
118 class static_example(example):
119 def run(self):
120 s = open(self.name).read().rstrip()
121 s = s.replace('&', '&').replace('<', '<').replace('>', '>')
122 ofp = wopen(result_name(self.name + '.tmp'))
123 ofp.write('<programlisting>')
124 ofp.write(s)
125 ofp.write('</programlisting>\n')
126 ofp.close()
127 self.rename_output(self.name)
128 norm = self.name.replace(os.sep, '-')
129 example.entities[
130 '<!ENTITY %s SYSTEM "results/%s.out">' % (norm, norm)] = 1
133 class shell_example(example):
134 shell = '/usr/bin/env bash'
135 ps1 = '__run_example_ps1__ '
136 ps2 = '__run_example_ps2__ '
137 pi_re = re.compile(r'#\$\s*(name|ignore):\s*(.*)$')
139 timeout = 10
141 def __init__(self, name, verbose, keep_change):
142 example.__init__(self, name, verbose, keep_change)
143 self.poll = select.poll()
145 def parse(self):
146 '''yield each hunk of input from the file.'''
147 fp = open(self.name)
148 cfp = cStringIO.StringIO()
149 for line in fp:
150 cfp.write(line)
151 if not line.rstrip().endswith('\\'):
152 yield cfp.getvalue()
153 cfp.seek(0)
154 cfp.truncate()
156 def status(self, s):
157 sys.stdout.write(s)
158 if not s.endswith('\n'):
159 sys.stdout.flush()
161 def send(self, s):
162 if self.verbose:
163 print >> sys.stderr, '>', self.debugrepr(s)
164 while s:
165 count = os.write(self.cfd, s)
166 s = s[count:]
168 def debugrepr(self, s):
169 rs = repr(s)
170 limit = 60
171 if len(rs) > limit:
172 return ('%s%s ... [%d bytes]' % (rs[:limit], rs[0], len(s)))
173 else:
174 return rs
176 timeout = 5
178 def read(self, hint):
179 events = self.poll.poll(self.timeout * 1000)
180 if not events:
181 print >> sys.stderr, ('[%stimed out after %d seconds]' %
182 (hint, self.timeout))
183 os.kill(self.pid, signal.SIGHUP)
184 return ''
185 return os.read(self.cfd, 1024)
187 def receive(self, hint):
188 out = cStringIO.StringIO()
189 while True:
190 try:
191 if self.verbose:
192 sys.stderr.write('< ')
193 s = self.read(hint)
194 except OSError, err:
195 if err.errno == errno.EIO:
196 return '', ''
197 raise
198 if self.verbose:
199 print >> sys.stderr, self.debugrepr(s)
200 out.write(s)
201 s = out.getvalue()
202 if s.endswith(self.ps1):
203 return self.ps1, s.replace('\r\n', '\n')[:-len(self.ps1)]
204 if s.endswith(self.ps2):
205 return self.ps2, s.replace('\r\n', '\n')[:-len(self.ps2)]
207 def sendreceive(self, s, hint):
208 self.send(s)
209 ps, r = self.receive(hint)
210 if r.startswith(s):
211 r = r[len(s):]
212 return ps, r
214 def run(self):
215 ofp = None
216 basename = os.path.basename(self.name)
217 self.status('running %s ' % basename)
218 tmpdir = tempfile.mkdtemp(prefix=basename)
220 # remove the marker file that we tell make to use to see if
221 # this run succeeded
222 maybe_unlink(self.name + '.run')
224 rcfile = os.path.join(tmpdir, '.hgrc')
225 rcfp = wopen(rcfile)
226 print >> rcfp, '[ui]'
227 print >> rcfp, "username = Bryan O'Sullivan <bos@serpentine.com>"
229 rcfile = os.path.join(tmpdir, '.bashrc')
230 rcfp = wopen(rcfile)
231 print >> rcfp, 'PS1="%s"' % self.ps1
232 print >> rcfp, 'PS2="%s"' % self.ps2
233 print >> rcfp, 'unset HISTFILE'
234 path = ['/usr/bin', '/bin']
235 hg = find_path_to('hg')
236 if hg and hg not in path:
237 path.append(hg)
238 def re_export(envar):
239 v = os.getenv(envar)
240 if v is not None:
241 print >> rcfp, 'export ' + envar + '=' + v
242 print >> rcfp, 'export PATH=' + ':'.join(path)
243 re_export('PYTHONPATH')
244 print >> rcfp, 'export EXAMPLE_DIR="%s"' % os.getcwd()
245 print >> rcfp, 'export HGMERGE=merge'
246 print >> rcfp, 'export LANG=C'
247 print >> rcfp, 'export LC_ALL=C'
248 print >> rcfp, 'export TZ=GMT'
249 print >> rcfp, 'export HGRC="%s/.hgrc"' % tmpdir
250 print >> rcfp, 'export HGRCPATH=$HGRC'
251 print >> rcfp, 'cd %s' % tmpdir
252 rcfp.close()
253 sys.stdout.flush()
254 sys.stderr.flush()
255 self.pid, self.cfd = pty.fork()
256 if self.pid == 0:
257 cmdline = ['/usr/bin/env', '-i', 'bash', '--noediting',
258 '--noprofile', '--norc']
259 try:
260 os.execv(cmdline[0], cmdline)
261 except OSError, err:
262 print >> sys.stderr, '%s: %s' % (cmdline[0], err.strerror)
263 sys.stderr.flush()
264 os._exit(0)
265 self.poll.register(self.cfd, select.POLLIN | select.POLLERR |
266 select.POLLHUP)
268 prompts = {
269 '': '',
270 self.ps1: '$',
271 self.ps2: '>',
272 }
274 ignore = [
275 r'\d+:[0-9a-f]{12}', # changeset number:hash
276 r'[0-9a-f]{40}', # long changeset hash
277 r'[0-9a-f]{12}', # short changeset hash
278 r'^(?:---|\+\+\+) .*', # diff header with dates
279 r'^date:.*', # date
280 #r'^diff -r.*', # "diff -r" is followed by hash
281 r'^# Date \d+ \d+', # hg patch header
282 ]
284 err = False
285 read_hint = ''
287 try:
288 try:
289 # eat first prompt string from shell
290 self.read(read_hint)
291 # setup env and prompt
292 ps, output = self.sendreceive('source %s\n' % rcfile,
293 read_hint)
294 for hunk in self.parse():
295 # is this line a processing instruction?
296 m = self.pi_re.match(hunk)
297 if m:
298 pi, rest = m.groups()
299 if pi == 'name':
300 self.status('.')
301 out = rest
302 if out in ('err', 'lxo', 'out', 'run', 'tmp'):
303 print >> sys.stderr, ('%s: illegal section '
304 'name %r' %
305 (self.name, out))
306 return 1
307 assert os.sep not in out
308 if ofp is not None:
309 ofp.write('</screen>\n')
310 ofp.close()
311 err |= self.rename_output(ofp_basename, ignore)
312 if out:
313 ofp_basename = '%s.%s' % (self.name, out)
314 norm = os.path.normpath(ofp_basename)
315 example.entities[
316 '<!ENTITY interaction.%s '
317 'SYSTEM "results/%s.out">'
318 % (norm, norm)] = 1
319 read_hint = ofp_basename + ' '
320 ofp = wopen(result_name(ofp_basename + '.tmp'))
321 ofp.write('<screen>')
322 else:
323 ofp = None
324 elif pi == 'ignore':
325 ignore.append(rest)
326 elif hunk.strip():
327 # it's something we should execute
328 newps, output = self.sendreceive(hunk, read_hint)
329 if not ofp:
330 continue
331 # first, print the command we ran
332 if not hunk.startswith('#'):
333 nl = hunk.endswith('\n')
334 hunk = ('<prompt>%s</prompt> <userinput>%s</userinput>' %
335 (prompts[ps],
336 xml_escape(hunk.rstrip('\n'))))
337 if nl: hunk += '\n'
338 ofp.write(hunk)
339 # then its output
340 ofp.write(xml_escape(output))
341 ps = newps
342 self.status('\n')
343 except:
344 print >> sys.stderr, '(killed)'
345 os.kill(self.pid, signal.SIGKILL)
346 pid, rc = os.wait()
347 raise
348 else:
349 try:
350 ps, output = self.sendreceive('exit\n', read_hint)
351 if ofp is not None:
352 ofp.write(output)
353 ofp.write('</screen>\n')
354 ofp.close()
355 err |= self.rename_output(ofp_basename, ignore)
356 os.close(self.cfd)
357 except IOError:
358 pass
359 os.kill(self.pid, signal.SIGTERM)
360 pid, rc = os.wait()
361 err = err or rc
362 if err:
363 if os.WIFEXITED(rc):
364 print >> sys.stderr, '(exit %s)' % os.WEXITSTATUS(rc)
365 elif os.WIFSIGNALED(rc):
366 print >> sys.stderr, '(signal %s)' % os.WTERMSIG(rc)
367 else:
368 wopen(result_name(self.name + '.run'))
369 return err
370 finally:
371 shutil.rmtree(tmpdir)
373 def print_help(exit, msg=None):
374 if msg:
375 print >> sys.stderr, 'Error:', msg
376 print >> sys.stderr, 'Usage: run-example [options] [test...]'
377 print >> sys.stderr, 'Options:'
378 print >> sys.stderr, ' -a --all run all examples in this directory'
379 print >> sys.stderr, ' -h --help print this help message'
380 print >> sys.stderr, ' --help keep new output as desired output'
381 print >> sys.stderr, ' -v --verbose display extra debug output'
382 sys.exit(exit)
384 def main(path='.'):
385 if os.path.realpath(path).split(os.sep)[-1] != 'examples':
386 print >> sys.stderr, 'Not being run from the examples directory!'
387 sys.exit(1)
389 opts, args = getopt.getopt(sys.argv[1:], '?ahv',
390 ['all', 'help', 'keep', 'verbose'])
391 verbose = False
392 run_all = False
393 keep_change = False
395 for o, a in opts:
396 if o in ('-h', '-?', '--help'):
397 print_help(0)
398 if o in ('-a', '--all'):
399 run_all = True
400 if o in ('--keep',):
401 keep_change = True
402 if o in ('-v', '--verbose'):
403 verbose = True
404 errs = 0
405 if args:
406 for a in args:
407 try:
408 st = os.lstat(a)
409 except OSError, err:
410 print >> sys.stderr, '%s: %s' % (a, err.strerror)
411 errs += 1
412 continue
413 if stat.S_ISREG(st.st_mode):
414 if st.st_mode & 0111:
415 if shell_example(a, verbose, keep_change).run():
416 errs += 1
417 elif a.endswith('.lst'):
418 static_example(a, verbose, keep_change).run()
419 else:
420 print >> sys.stderr, '%s: not a file, or not executable' % a
421 errs += 1
422 elif run_all:
423 names = glob.glob("*") + glob.glob("app*/*") + glob.glob("ch*/*")
424 names.sort()
425 for name in names:
426 if name == 'run-example' or name.endswith('~'): continue
427 pathname = os.path.join(path, name)
428 try:
429 st = os.lstat(pathname)
430 except OSError, err:
431 # could be an output file that was removed while we ran
432 if err.errno != errno.ENOENT:
433 raise
434 continue
435 if stat.S_ISREG(st.st_mode):
436 if st.st_mode & 0111:
437 if shell_example(pathname, verbose, keep_change).run():
438 errs += 1
439 elif pathname.endswith('.lst'):
440 static_example(pathname, verbose, keep_change).run()
441 print >> wopen(os.path.join(path, '.run')), time.asctime()
442 else:
443 print_help(1, msg='no test names given, and --all not provided')
445 fp = wopen('auto-snippets.xml')
446 for key in sorted(example.entities.iterkeys()):
447 print >> fp, key
448 fp.close()
449 return errs
451 if __name__ == '__main__':
452 try:
453 sys.exit(main())
454 except KeyboardInterrupt:
455 print >> sys.stderr, 'interrupted!'
456 sys.exit(1)