hgbook
view en/examples/run-example @ 569:60ee738fdc0e
Fix quoting of entity declarations.
| author | Bryan O'Sullivan <bos@serpentine.com> |
|---|---|
| date | Mon Mar 09 23:32:15 2009 -0700 (2009-03-09) |
| parents | 27043f385f3f |
| children | 80928ea6e7ae |
line source
1 #!/usr/bin/env python
2 #
3 # This program takes something that resembles a shell script and runs
4 # it, spitting input (commands from the script) and output into text
5 # files, for use in examples.
7 import cStringIO
8 import errno
9 import getopt
10 import os
11 import pty
12 import re
13 import select
14 import shutil
15 import signal
16 import stat
17 import sys
18 import tempfile
19 import time
21 xml_subs = {
22 '<': '<',
23 '>': '>',
24 '&': '&',
25 }
27 def gensubs(s):
28 start = 0
29 for i, c in enumerate(s):
30 sub = xml_subs.get(c)
31 if sub:
32 yield s[start:i]
33 start = i + 1
34 yield sub
35 yield s[start:]
37 def xml_escape(s):
38 return ''.join(gensubs(s))
40 def maybe_unlink(name):
41 try:
42 os.unlink(name)
43 return True
44 except OSError, err:
45 if err.errno != errno.ENOENT:
46 raise
47 return False
49 def find_path_to(program):
50 for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
51 name = os.path.join(p, program)
52 if os.access(name, os.X_OK):
53 return p
54 return None
56 def result_name(name):
57 dirname, basename = os.path.split(name)
58 return os.path.join(dirname, 'results', basename)
60 class example:
61 shell = '/usr/bin/env bash'
62 ps1 = '__run_example_ps1__ '
63 ps2 = '__run_example_ps2__ '
64 pi_re = re.compile(r'#\$\s*(name|ignore):\s*(.*)$')
66 timeout = 10
68 entities = dict.fromkeys(l.rstrip() for l in open('auto-snippets.xml'))
70 def __init__(self, name, verbose, keep_change):
71 self.name = name
72 self.verbose = verbose
73 self.keep_change = keep_change
74 self.poll = select.poll()
76 def parse(self):
77 '''yield each hunk of input from the file.'''
78 fp = open(self.name)
79 cfp = cStringIO.StringIO()
80 for line in fp:
81 cfp.write(line)
82 if not line.rstrip().endswith('\\'):
83 yield cfp.getvalue()
84 cfp.seek(0)
85 cfp.truncate()
87 def status(self, s):
88 sys.stdout.write(s)
89 if not s.endswith('\n'):
90 sys.stdout.flush()
92 def send(self, s):
93 if self.verbose:
94 print >> sys.stderr, '>', self.debugrepr(s)
95 while s:
96 count = os.write(self.cfd, s)
97 s = s[count:]
99 def debugrepr(self, s):
100 rs = repr(s)
101 limit = 60
102 if len(rs) > limit:
103 return ('%s%s ... [%d bytes]' % (rs[:limit], rs[0], len(s)))
104 else:
105 return rs
107 timeout = 5
109 def read(self, hint):
110 events = self.poll.poll(self.timeout * 1000)
111 if not events:
112 print >> sys.stderr, ('[%stimed out after %d seconds]' %
113 (hint, self.timeout))
114 os.kill(self.pid, signal.SIGHUP)
115 return ''
116 return os.read(self.cfd, 1024)
118 def receive(self, hint):
119 out = cStringIO.StringIO()
120 while True:
121 try:
122 if self.verbose:
123 sys.stderr.write('< ')
124 s = self.read(hint)
125 except OSError, err:
126 if err.errno == errno.EIO:
127 return '', ''
128 raise
129 if self.verbose:
130 print >> sys.stderr, self.debugrepr(s)
131 out.write(s)
132 s = out.getvalue()
133 if s.endswith(self.ps1):
134 return self.ps1, s.replace('\r\n', '\n')[:-len(self.ps1)]
135 if s.endswith(self.ps2):
136 return self.ps2, s.replace('\r\n', '\n')[:-len(self.ps2)]
138 def sendreceive(self, s, hint):
139 self.send(s)
140 ps, r = self.receive(hint)
141 if r.startswith(s):
142 r = r[len(s):]
143 return ps, r
145 def run(self):
146 ofp = None
147 basename = os.path.basename(self.name)
148 self.status('running %s ' % basename)
149 tmpdir = tempfile.mkdtemp(prefix=basename)
151 # remove the marker file that we tell make to use to see if
152 # this run succeeded
153 maybe_unlink(self.name + '.run')
155 rcfile = os.path.join(tmpdir, '.hgrc')
156 rcfp = open(rcfile, 'w')
157 print >> rcfp, '[ui]'
158 print >> rcfp, "username = Bryan O'Sullivan <bos@serpentine.com>"
160 rcfile = os.path.join(tmpdir, '.bashrc')
161 rcfp = open(rcfile, 'w')
162 print >> rcfp, 'PS1="%s"' % self.ps1
163 print >> rcfp, 'PS2="%s"' % self.ps2
164 print >> rcfp, 'unset HISTFILE'
165 path = ['/usr/bin', '/bin']
166 hg = find_path_to('hg')
167 if hg and hg not in path:
168 path.append(hg)
169 def re_export(envar):
170 v = os.getenv(envar)
171 if v is not None:
172 print >> rcfp, 'export ' + envar + '=' + v
173 print >> rcfp, 'export PATH=' + ':'.join(path)
174 re_export('PYTHONPATH')
175 print >> rcfp, 'export EXAMPLE_DIR="%s"' % os.getcwd()
176 print >> rcfp, 'export HGMERGE=merge'
177 print >> rcfp, 'export LANG=C'
178 print >> rcfp, 'export LC_ALL=C'
179 print >> rcfp, 'export TZ=GMT'
180 print >> rcfp, 'export HGRC="%s/.hgrc"' % tmpdir
181 print >> rcfp, 'export HGRCPATH=$HGRC'
182 print >> rcfp, 'cd %s' % tmpdir
183 rcfp.close()
184 sys.stdout.flush()
185 sys.stderr.flush()
186 self.pid, self.cfd = pty.fork()
187 if self.pid == 0:
188 cmdline = ['/usr/bin/env', '-i', 'bash', '--noediting',
189 '--noprofile', '--norc']
190 try:
191 os.execv(cmdline[0], cmdline)
192 except OSError, err:
193 print >> sys.stderr, '%s: %s' % (cmdline[0], err.strerror)
194 sys.stderr.flush()
195 os._exit(0)
196 self.poll.register(self.cfd, select.POLLIN | select.POLLERR |
197 select.POLLHUP)
199 prompts = {
200 '': '',
201 self.ps1: '$',
202 self.ps2: '>',
203 }
205 ignore = [
206 r'\d+:[0-9a-f]{12}', # changeset number:hash
207 r'[0-9a-f]{40}', # long changeset hash
208 r'[0-9a-f]{12}', # short changeset hash
209 r'^(?:---|\+\+\+) .*', # diff header with dates
210 r'^date:.*', # date
211 #r'^diff -r.*', # "diff -r" is followed by hash
212 r'^# Date \d+ \d+', # hg patch header
213 ]
215 err = False
216 read_hint = ''
218 try:
219 try:
220 # eat first prompt string from shell
221 self.read(read_hint)
222 # setup env and prompt
223 ps, output = self.sendreceive('source %s\n' % rcfile,
224 read_hint)
225 for hunk in self.parse():
226 # is this line a processing instruction?
227 m = self.pi_re.match(hunk)
228 if m:
229 pi, rest = m.groups()
230 if pi == 'name':
231 self.status('.')
232 out = rest
233 if out in ('err', 'lxo', 'out', 'run', 'tmp'):
234 print >> sys.stderr, ('%s: illegal section '
235 'name %r' %
236 (self.name, out))
237 return 1
238 assert os.sep not in out
239 if ofp is not None:
240 ofp.write('</screen>\n')
241 ofp.close()
242 err |= self.rename_output(ofp_basename, ignore)
243 if out:
244 ofp_basename = '%s.%s' % (self.name, out)
245 norm = os.path.normpath(ofp_basename)
246 example.entities[
247 '<!ENTITY interaction.%s '
248 'SYSTEM "results/%s.out">'
249 % (norm, norm)] = 1
250 read_hint = ofp_basename + ' '
251 ofp = open(result_name(ofp_basename + '.tmp'),
252 'w')
253 ofp.write('<screen>')
254 else:
255 ofp = None
256 elif pi == 'ignore':
257 ignore.append(rest)
258 elif hunk.strip():
259 # it's something we should execute
260 newps, output = self.sendreceive(hunk, read_hint)
261 if not ofp:
262 continue
263 # first, print the command we ran
264 if not hunk.startswith('#'):
265 nl = hunk.endswith('\n')
266 hunk = ('<prompt>%s</prompt> <userinput>%s</userinput>' %
267 (prompts[ps],
268 xml_escape(hunk.rstrip('\n'))))
269 if nl: hunk += '\n'
270 ofp.write(hunk)
271 # then its output
272 ofp.write(xml_escape(output))
273 ps = newps
274 self.status('\n')
275 except:
276 print >> sys.stderr, '(killed)'
277 os.kill(self.pid, signal.SIGKILL)
278 pid, rc = os.wait()
279 raise
280 else:
281 try:
282 ps, output = self.sendreceive('exit\n', read_hint)
283 if ofp is not None:
284 ofp.write(output)
285 ofp.write('</screen>\n')
286 ofp.close()
287 err |= self.rename_output(ofp_basename, ignore)
288 os.close(self.cfd)
289 except IOError:
290 pass
291 os.kill(self.pid, signal.SIGTERM)
292 pid, rc = os.wait()
293 err = err or rc
294 if err:
295 if os.WIFEXITED(rc):
296 print >> sys.stderr, '(exit %s)' % os.WEXITSTATUS(rc)
297 elif os.WIFSIGNALED(rc):
298 print >> sys.stderr, '(signal %s)' % os.WTERMSIG(rc)
299 else:
300 open(result_name(self.name + '.run'), 'w')
301 return err
302 finally:
303 shutil.rmtree(tmpdir)
305 def rename_output(self, base, ignore):
306 mangle_re = re.compile('(?:' + '|'.join(ignore) + ')')
307 def mangle(s):
308 return mangle_re.sub('', s)
309 def matchfp(fp1, fp2):
310 while True:
311 s1 = mangle(fp1.readline())
312 s2 = mangle(fp2.readline())
313 if cmp(s1, s2):
314 break
315 if not s1:
316 return True
317 return False
319 oldname = result_name(base + '.out')
320 tmpname = result_name(base + '.tmp')
321 errname = result_name(base + '.err')
322 errfp = open(errname, 'w+')
323 for line in open(tmpname):
324 errfp.write(mangle_re.sub('', line))
325 os.rename(tmpname, result_name(base + '.lxo'))
326 errfp.seek(0)
327 try:
328 oldfp = open(oldname)
329 except IOError, err:
330 if err.errno != errno.ENOENT:
331 raise
332 os.rename(errname, oldname)
333 return False
334 if matchfp(oldfp, errfp):
335 os.unlink(errname)
336 return False
337 else:
338 print >> sys.stderr, '\nOutput of %s has changed!' % base
339 if self.keep_change:
340 os.rename(errname, oldname)
341 return False
342 else:
343 os.system('diff -u %s %s 1>&2' % (oldname, errname))
344 return True
346 def print_help(exit, msg=None):
347 if msg:
348 print >> sys.stderr, 'Error:', msg
349 print >> sys.stderr, 'Usage: run-example [options] [test...]'
350 print >> sys.stderr, 'Options:'
351 print >> sys.stderr, ' -a --all run all examples in this directory'
352 print >> sys.stderr, ' -h --help print this help message'
353 print >> sys.stderr, ' --help keep new output as desired output'
354 print >> sys.stderr, ' -v --verbose display extra debug output'
355 sys.exit(exit)
357 def main(path='.'):
358 if os.path.realpath(path).split(os.sep)[-1] != 'examples':
359 print >> sys.stderr, 'Not being run from the examples directory!'
360 sys.exit(1)
362 opts, args = getopt.getopt(sys.argv[1:], '?ahv',
363 ['all', 'help', 'keep', 'verbose'])
364 verbose = False
365 run_all = False
366 keep_change = False
368 for o, a in opts:
369 if o in ('-h', '-?', '--help'):
370 print_help(0)
371 if o in ('-a', '--all'):
372 run_all = True
373 if o in ('--keep',):
374 keep_change = True
375 if o in ('-v', '--verbose'):
376 verbose = True
377 errs = 0
378 if args:
379 for a in args:
380 try:
381 st = os.lstat(a)
382 except OSError, err:
383 print >> sys.stderr, '%s: %s' % (a, err.strerror)
384 errs += 1
385 continue
386 if stat.S_ISREG(st.st_mode) and st.st_mode & 0111:
387 if example(a, verbose, keep_change).run():
388 errs += 1
389 else:
390 print >> sys.stderr, '%s: not a file, or not executable' % a
391 errs += 1
392 elif run_all:
393 names = os.listdir(path)
394 names.sort()
395 for name in names:
396 if name == 'run-example' or name.startswith('.'): continue
397 if name.endswith('~'): continue
398 pathname = os.path.join(path, name)
399 try:
400 st = os.lstat(pathname)
401 except OSError, err:
402 # could be an output file that was removed while we ran
403 if err.errno != errno.ENOENT:
404 raise
405 continue
406 if stat.S_ISREG(st.st_mode) and st.st_mode & 0111:
407 if example(pathname, verbose, keep_change).run():
408 errs += 1
409 print >> open(os.path.join(path, '.run'), 'w'), time.asctime()
410 else:
411 print_help(1, msg='no test names given, and --all not provided')
413 fp = open('auto-snippets.xml', 'w')
414 for key in sorted(example.entities.iterkeys()):
415 print >> fp, key
416 fp.close()
417 return errs
419 if __name__ == '__main__':
420 try:
421 sys.exit(main())
422 except KeyboardInterrupt:
423 print >> sys.stderr, 'interrupted!'
424 sys.exit(1)