rev |
line source |
bos@67
|
1 #!/usr/bin/env python
|
bos@4
|
2 #
|
bos@4
|
3 # This program takes something that resembles a shell script and runs
|
bos@4
|
4 # it, spitting input (commands from the script) and output into text
|
bos@4
|
5 # files, for use in examples.
|
bos@3
|
6
|
bos@3
|
7 import cStringIO
|
bos@73
|
8 import errno
|
bos@78
|
9 import getopt
|
bos@3
|
10 import os
|
bos@3
|
11 import pty
|
bos@3
|
12 import re
|
bos@79
|
13 import select
|
bos@4
|
14 import shutil
|
bos@6
|
15 import signal
|
bos@36
|
16 import stat
|
bos@3
|
17 import sys
|
bos@4
|
18 import tempfile
|
bos@4
|
19 import time
|
bos@3
|
20
|
bos@564
|
21 xml_subs = {
|
bos@564
|
22 '<': '<',
|
bos@564
|
23 '>': '>',
|
bos@564
|
24 '&': '&',
|
bos@77
|
25 }
|
bos@77
|
26
|
bos@77
|
27 def gensubs(s):
|
bos@77
|
28 start = 0
|
bos@77
|
29 for i, c in enumerate(s):
|
bos@564
|
30 sub = xml_subs.get(c)
|
bos@77
|
31 if sub:
|
bos@77
|
32 yield s[start:i]
|
bos@77
|
33 start = i + 1
|
bos@77
|
34 yield sub
|
bos@77
|
35 yield s[start:]
|
bos@77
|
36
|
bos@564
|
37 def xml_escape(s):
|
bos@77
|
38 return ''.join(gensubs(s))
|
bos@4
|
39
|
bos@137
|
40 def maybe_unlink(name):
|
bos@137
|
41 try:
|
bos@137
|
42 os.unlink(name)
|
bos@137
|
43 return True
|
bos@137
|
44 except OSError, err:
|
bos@137
|
45 if err.errno != errno.ENOENT:
|
bos@137
|
46 raise
|
bos@137
|
47 return False
|
bos@137
|
48
|
bos@172
|
49 def find_path_to(program):
|
bos@172
|
50 for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
|
bos@172
|
51 name = os.path.join(p, program)
|
bos@172
|
52 if os.access(name, os.X_OK):
|
bos@172
|
53 return p
|
bos@172
|
54 return None
|
bos@172
|
55
|
bos@564
|
56 def result_name(name):
|
bos@564
|
57 dirname, basename = os.path.split(name)
|
bos@564
|
58 return os.path.join(dirname, 'results', basename)
|
bos@564
|
59
|
bos@3
|
60 class example:
|
bos@70
|
61 shell = '/usr/bin/env bash'
|
bos@103
|
62 ps1 = '__run_example_ps1__ '
|
bos@103
|
63 ps2 = '__run_example_ps2__ '
|
bos@168
|
64 pi_re = re.compile(r'#\$\s*(name|ignore):\s*(.*)$')
|
bos@4
|
65
|
bos@192
|
66 timeout = 10
|
bos@79
|
67
|
bos@566
|
68 entities = dict.fromkeys(l.rstrip() for l in open('auto-snippets.xml'))
|
bos@566
|
69
|
bos@545
|
70 def __init__(self, name, verbose, keep_change):
|
bos@3
|
71 self.name = name
|
bos@78
|
72 self.verbose = verbose
|
bos@545
|
73 self.keep_change = keep_change
|
bos@79
|
74 self.poll = select.poll()
|
bos@3
|
75
|
bos@3
|
76 def parse(self):
|
bos@4
|
77 '''yield each hunk of input from the file.'''
|
bos@3
|
78 fp = open(self.name)
|
bos@3
|
79 cfp = cStringIO.StringIO()
|
bos@3
|
80 for line in fp:
|
bos@3
|
81 cfp.write(line)
|
bos@3
|
82 if not line.rstrip().endswith('\\'):
|
bos@3
|
83 yield cfp.getvalue()
|
bos@3
|
84 cfp.seek(0)
|
bos@3
|
85 cfp.truncate()
|
bos@3
|
86
|
bos@3
|
87 def status(self, s):
|
bos@3
|
88 sys.stdout.write(s)
|
bos@3
|
89 if not s.endswith('\n'):
|
bos@3
|
90 sys.stdout.flush()
|
bos@3
|
91
|
bos@6
|
92 def send(self, s):
|
bos@78
|
93 if self.verbose:
|
bos@78
|
94 print >> sys.stderr, '>', self.debugrepr(s)
|
bos@73
|
95 while s:
|
bos@73
|
96 count = os.write(self.cfd, s)
|
bos@73
|
97 s = s[count:]
|
bos@6
|
98
|
bos@78
|
99 def debugrepr(self, s):
|
bos@78
|
100 rs = repr(s)
|
bos@78
|
101 limit = 60
|
bos@78
|
102 if len(rs) > limit:
|
bos@78
|
103 return ('%s%s ... [%d bytes]' % (rs[:limit], rs[0], len(s)))
|
bos@78
|
104 else:
|
bos@78
|
105 return rs
|
bos@78
|
106
|
bos@79
|
107 timeout = 5
|
bos@79
|
108
|
bos@173
|
109 def read(self, hint):
|
bos@79
|
110 events = self.poll.poll(self.timeout * 1000)
|
bos@79
|
111 if not events:
|
bos@173
|
112 print >> sys.stderr, ('[%stimed out after %d seconds]' %
|
bos@173
|
113 (hint, self.timeout))
|
bos@79
|
114 os.kill(self.pid, signal.SIGHUP)
|
bos@79
|
115 return ''
|
bos@79
|
116 return os.read(self.cfd, 1024)
|
bos@79
|
117
|
bos@173
|
118 def receive(self, hint):
|
bos@6
|
119 out = cStringIO.StringIO()
|
bos@4
|
120 while True:
|
bos@73
|
121 try:
|
bos@78
|
122 if self.verbose:
|
bos@78
|
123 sys.stderr.write('< ')
|
bos@173
|
124 s = self.read(hint)
|
bos@73
|
125 except OSError, err:
|
bos@73
|
126 if err.errno == errno.EIO:
|
bos@103
|
127 return '', ''
|
bos@73
|
128 raise
|
bos@78
|
129 if self.verbose:
|
bos@78
|
130 print >> sys.stderr, self.debugrepr(s)
|
bos@6
|
131 out.write(s)
|
bos@73
|
132 s = out.getvalue()
|
bos@103
|
133 if s.endswith(self.ps1):
|
bos@103
|
134 return self.ps1, s.replace('\r\n', '\n')[:-len(self.ps1)]
|
bos@103
|
135 if s.endswith(self.ps2):
|
bos@103
|
136 return self.ps2, s.replace('\r\n', '\n')[:-len(self.ps2)]
|
bos@4
|
137
|
bos@173
|
138 def sendreceive(self, s, hint):
|
bos@6
|
139 self.send(s)
|
bos@173
|
140 ps, r = self.receive(hint)
|
bos@6
|
141 if r.startswith(s):
|
bos@6
|
142 r = r[len(s):]
|
bos@103
|
143 return ps, r
|
bos@6
|
144
|
bos@3
|
145 def run(self):
|
bos@3
|
146 ofp = None
|
bos@4
|
147 basename = os.path.basename(self.name)
|
bos@4
|
148 self.status('running %s ' % basename)
|
bos@4
|
149 tmpdir = tempfile.mkdtemp(prefix=basename)
|
bos@78
|
150
|
bos@136
|
151 # remove the marker file that we tell make to use to see if
|
bos@136
|
152 # this run succeeded
|
bos@137
|
153 maybe_unlink(self.name + '.run')
|
bos@136
|
154
|
bos@78
|
155 rcfile = os.path.join(tmpdir, '.hgrc')
|
bos@78
|
156 rcfp = open(rcfile, 'w')
|
bos@78
|
157 print >> rcfp, '[ui]'
|
bos@78
|
158 print >> rcfp, "username = Bryan O'Sullivan <bos@serpentine.com>"
|
bos@78
|
159
|
bos@6
|
160 rcfile = os.path.join(tmpdir, '.bashrc')
|
bos@6
|
161 rcfp = open(rcfile, 'w')
|
bos@103
|
162 print >> rcfp, 'PS1="%s"' % self.ps1
|
bos@103
|
163 print >> rcfp, 'PS2="%s"' % self.ps2
|
bos@6
|
164 print >> rcfp, 'unset HISTFILE'
|
bos@172
|
165 path = ['/usr/bin', '/bin']
|
bos@172
|
166 hg = find_path_to('hg')
|
bos@172
|
167 if hg and hg not in path:
|
bos@172
|
168 path.append(hg)
|
bos@172
|
169 def re_export(envar):
|
bos@172
|
170 v = os.getenv(envar)
|
bos@172
|
171 if v is not None:
|
bos@172
|
172 print >> rcfp, 'export ' + envar + '=' + v
|
bos@172
|
173 print >> rcfp, 'export PATH=' + ':'.join(path)
|
bos@172
|
174 re_export('PYTHONPATH')
|
bos@19
|
175 print >> rcfp, 'export EXAMPLE_DIR="%s"' % os.getcwd()
|
bos@124
|
176 print >> rcfp, 'export HGMERGE=merge'
|
bos@6
|
177 print >> rcfp, 'export LANG=C'
|
bos@6
|
178 print >> rcfp, 'export LC_ALL=C'
|
bos@6
|
179 print >> rcfp, 'export TZ=GMT'
|
bos@6
|
180 print >> rcfp, 'export HGRC="%s/.hgrc"' % tmpdir
|
bos@6
|
181 print >> rcfp, 'export HGRCPATH=$HGRC'
|
bos@6
|
182 print >> rcfp, 'cd %s' % tmpdir
|
bos@6
|
183 rcfp.close()
|
bos@68
|
184 sys.stdout.flush()
|
bos@68
|
185 sys.stderr.flush()
|
bos@79
|
186 self.pid, self.cfd = pty.fork()
|
bos@79
|
187 if self.pid == 0:
|
bos@172
|
188 cmdline = ['/usr/bin/env', '-i', 'bash', '--noediting',
|
bos@172
|
189 '--noprofile', '--norc']
|
bos@68
|
190 try:
|
bos@68
|
191 os.execv(cmdline[0], cmdline)
|
bos@68
|
192 except OSError, err:
|
bos@68
|
193 print >> sys.stderr, '%s: %s' % (cmdline[0], err.strerror)
|
bos@68
|
194 sys.stderr.flush()
|
bos@68
|
195 os._exit(0)
|
bos@79
|
196 self.poll.register(self.cfd, select.POLLIN | select.POLLERR |
|
bos@79
|
197 select.POLLHUP)
|
bos@103
|
198
|
bos@103
|
199 prompts = {
|
bos@103
|
200 '': '',
|
bos@103
|
201 self.ps1: '$',
|
bos@103
|
202 self.ps2: '>',
|
bos@103
|
203 }
|
bos@103
|
204
|
bos@137
|
205 ignore = [
|
bos@137
|
206 r'\d+:[0-9a-f]{12}', # changeset number:hash
|
bos@141
|
207 r'[0-9a-f]{40}', # long changeset hash
|
bos@141
|
208 r'[0-9a-f]{12}', # short changeset hash
|
bos@137
|
209 r'^(?:---|\+\+\+) .*', # diff header with dates
|
bos@137
|
210 r'^date:.*', # date
|
bos@141
|
211 #r'^diff -r.*', # "diff -r" is followed by hash
|
bos@138
|
212 r'^# Date \d+ \d+', # hg patch header
|
bos@137
|
213 ]
|
bos@137
|
214
|
bos@138
|
215 err = False
|
bos@173
|
216 read_hint = ''
|
bos@138
|
217
|
bos@4
|
218 try:
|
bos@71
|
219 try:
|
bos@73
|
220 # eat first prompt string from shell
|
bos@173
|
221 self.read(read_hint)
|
bos@71
|
222 # setup env and prompt
|
bos@173
|
223 ps, output = self.sendreceive('source %s\n' % rcfile,
|
bos@173
|
224 read_hint)
|
bos@71
|
225 for hunk in self.parse():
|
bos@71
|
226 # is this line a processing instruction?
|
bos@71
|
227 m = self.pi_re.match(hunk)
|
bos@71
|
228 if m:
|
bos@71
|
229 pi, rest = m.groups()
|
bos@71
|
230 if pi == 'name':
|
bos@71
|
231 self.status('.')
|
bos@71
|
232 out = rest
|
bos@155
|
233 if out in ('err', 'lxo', 'out', 'run', 'tmp'):
|
bos@155
|
234 print >> sys.stderr, ('%s: illegal section '
|
bos@155
|
235 'name %r' %
|
bos@155
|
236 (self.name, out))
|
bos@155
|
237 return 1
|
bos@71
|
238 assert os.sep not in out
|
bos@137
|
239 if ofp is not None:
|
bos@564
|
240 ofp.write('</screen>\n')
|
bos@137
|
241 ofp.close()
|
bos@160
|
242 err |= self.rename_output(ofp_basename, ignore)
|
bos@71
|
243 if out:
|
bos@137
|
244 ofp_basename = '%s.%s' % (self.name, out)
|
bos@566
|
245 norm = os.path.normpath(ofp_basename)
|
bos@566
|
246 example.entities[
|
bos@566
|
247 '<!ENTITY "interaction.%s" '
|
bos@566
|
248 'SYSTEM "results/%s.out">'
|
bos@566
|
249 % (norm, norm)] = 1
|
bos@173
|
250 read_hint = ofp_basename + ' '
|
bos@564
|
251 ofp = open(result_name(ofp_basename + '.tmp'),
|
bos@564
|
252 'w')
|
bos@564
|
253 ofp.write('<screen>')
|
bos@71
|
254 else:
|
bos@71
|
255 ofp = None
|
bos@137
|
256 elif pi == 'ignore':
|
bos@137
|
257 ignore.append(rest)
|
bos@71
|
258 elif hunk.strip():
|
bos@71
|
259 # it's something we should execute
|
bos@173
|
260 newps, output = self.sendreceive(hunk, read_hint)
|
bos@168
|
261 if not ofp:
|
bos@71
|
262 continue
|
bos@71
|
263 # first, print the command we ran
|
bos@71
|
264 if not hunk.startswith('#'):
|
bos@71
|
265 nl = hunk.endswith('\n')
|
bos@564
|
266 hunk = ('<prompt>%s</prompt> <userinput>%s</userinput>' %
|
bos@103
|
267 (prompts[ps],
|
bos@564
|
268 xml_escape(hunk.rstrip('\n'))))
|
bos@71
|
269 if nl: hunk += '\n'
|
bos@71
|
270 ofp.write(hunk)
|
bos@71
|
271 # then its output
|
bos@564
|
272 ofp.write(xml_escape(output))
|
bos@103
|
273 ps = newps
|
bos@71
|
274 self.status('\n')
|
bos@71
|
275 except:
|
bos@72
|
276 print >> sys.stderr, '(killed)'
|
bos@79
|
277 os.kill(self.pid, signal.SIGKILL)
|
bos@72
|
278 pid, rc = os.wait()
|
bos@71
|
279 raise
|
bos@72
|
280 else:
|
bos@71
|
281 try:
|
bos@173
|
282 ps, output = self.sendreceive('exit\n', read_hint)
|
bos@138
|
283 if ofp is not None:
|
bos@71
|
284 ofp.write(output)
|
bos@564
|
285 ofp.write('</screen>\n')
|
bos@138
|
286 ofp.close()
|
bos@160
|
287 err |= self.rename_output(ofp_basename, ignore)
|
bos@73
|
288 os.close(self.cfd)
|
bos@71
|
289 except IOError:
|
bos@71
|
290 pass
|
bos@79
|
291 os.kill(self.pid, signal.SIGTERM)
|
bos@72
|
292 pid, rc = os.wait()
|
bos@160
|
293 err = err or rc
|
bos@160
|
294 if err:
|
bos@72
|
295 if os.WIFEXITED(rc):
|
bos@72
|
296 print >> sys.stderr, '(exit %s)' % os.WEXITSTATUS(rc)
|
bos@72
|
297 elif os.WIFSIGNALED(rc):
|
bos@72
|
298 print >> sys.stderr, '(signal %s)' % os.WTERMSIG(rc)
|
bos@136
|
299 else:
|
bos@564
|
300 open(result_name(self.name + '.run'), 'w')
|
bos@160
|
301 return err
|
bos@72
|
302 finally:
|
bos@4
|
303 shutil.rmtree(tmpdir)
|
bos@3
|
304
|
bos@137
|
305 def rename_output(self, base, ignore):
|
bos@137
|
306 mangle_re = re.compile('(?:' + '|'.join(ignore) + ')')
|
bos@137
|
307 def mangle(s):
|
bos@137
|
308 return mangle_re.sub('', s)
|
bos@137
|
309 def matchfp(fp1, fp2):
|
bos@137
|
310 while True:
|
bos@137
|
311 s1 = mangle(fp1.readline())
|
bos@137
|
312 s2 = mangle(fp2.readline())
|
bos@137
|
313 if cmp(s1, s2):
|
bos@137
|
314 break
|
bos@137
|
315 if not s1:
|
bos@137
|
316 return True
|
bos@137
|
317 return False
|
bos@137
|
318
|
bos@564
|
319 oldname = result_name(base + '.out')
|
bos@564
|
320 tmpname = result_name(base + '.tmp')
|
bos@564
|
321 errname = result_name(base + '.err')
|
bos@137
|
322 errfp = open(errname, 'w+')
|
bos@137
|
323 for line in open(tmpname):
|
bos@137
|
324 errfp.write(mangle_re.sub('', line))
|
bos@564
|
325 os.rename(tmpname, result_name(base + '.lxo'))
|
bos@137
|
326 errfp.seek(0)
|
bos@137
|
327 try:
|
bos@137
|
328 oldfp = open(oldname)
|
bos@137
|
329 except IOError, err:
|
bos@137
|
330 if err.errno != errno.ENOENT:
|
bos@137
|
331 raise
|
bos@137
|
332 os.rename(errname, oldname)
|
bos@160
|
333 return False
|
bos@137
|
334 if matchfp(oldfp, errfp):
|
bos@137
|
335 os.unlink(errname)
|
bos@160
|
336 return False
|
bos@137
|
337 else:
|
bos@137
|
338 print >> sys.stderr, '\nOutput of %s has changed!' % base
|
bos@545
|
339 if self.keep_change:
|
bos@545
|
340 os.rename(errname, oldname)
|
bos@545
|
341 return False
|
bos@545
|
342 else:
|
bos@545
|
343 os.system('diff -u %s %s 1>&2' % (oldname, errname))
|
bos@138
|
344 return True
|
bos@137
|
345
|
bos@258
|
346 def print_help(exit, msg=None):
|
bos@258
|
347 if msg:
|
bos@258
|
348 print >> sys.stderr, 'Error:', msg
|
bos@258
|
349 print >> sys.stderr, 'Usage: run-example [options] [test...]'
|
bos@258
|
350 print >> sys.stderr, 'Options:'
|
bos@565
|
351 print >> sys.stderr, ' -a --all run all examples in this directory'
|
bos@258
|
352 print >> sys.stderr, ' -h --help print this help message'
|
bos@545
|
353 print >> sys.stderr, ' --help keep new output as desired output'
|
bos@258
|
354 print >> sys.stderr, ' -v --verbose display extra debug output'
|
bos@258
|
355 sys.exit(exit)
|
bos@258
|
356
|
bos@3
|
357 def main(path='.'):
|
bos@566
|
358 if os.path.realpath(path).split(os.sep)[-1] != 'examples':
|
bos@566
|
359 print >> sys.stderr, 'Not being run from the examples directory!'
|
bos@566
|
360 sys.exit(1)
|
bos@566
|
361
|
bos@258
|
362 opts, args = getopt.getopt(sys.argv[1:], '?ahv',
|
bos@545
|
363 ['all', 'help', 'keep', 'verbose'])
|
bos@78
|
364 verbose = False
|
bos@258
|
365 run_all = False
|
bos@545
|
366 keep_change = False
|
bos@566
|
367
|
bos@78
|
368 for o, a in opts:
|
bos@258
|
369 if o in ('-h', '-?', '--help'):
|
bos@258
|
370 print_help(0)
|
bos@258
|
371 if o in ('-a', '--all'):
|
bos@258
|
372 run_all = True
|
bos@545
|
373 if o in ('--keep',):
|
bos@545
|
374 keep_change = True
|
bos@78
|
375 if o in ('-v', '--verbose'):
|
bos@78
|
376 verbose = True
|
bos@71
|
377 errs = 0
|
bos@3
|
378 if args:
|
bos@3
|
379 for a in args:
|
bos@75
|
380 try:
|
bos@75
|
381 st = os.lstat(a)
|
bos@75
|
382 except OSError, err:
|
bos@75
|
383 print >> sys.stderr, '%s: %s' % (a, err.strerror)
|
bos@75
|
384 errs += 1
|
bos@75
|
385 continue
|
bos@75
|
386 if stat.S_ISREG(st.st_mode) and st.st_mode & 0111:
|
bos@545
|
387 if example(a, verbose, keep_change).run():
|
bos@75
|
388 errs += 1
|
bos@75
|
389 else:
|
bos@75
|
390 print >> sys.stderr, '%s: not a file, or not executable' % a
|
bos@71
|
391 errs += 1
|
bos@258
|
392 elif run_all:
|
bos@258
|
393 names = os.listdir(path)
|
bos@258
|
394 names.sort()
|
bos@258
|
395 for name in names:
|
bos@258
|
396 if name == 'run-example' or name.startswith('.'): continue
|
bos@564
|
397 if name.endswith('~'): continue
|
bos@258
|
398 pathname = os.path.join(path, name)
|
bos@258
|
399 try:
|
bos@258
|
400 st = os.lstat(pathname)
|
bos@258
|
401 except OSError, err:
|
bos@258
|
402 # could be an output file that was removed while we ran
|
bos@258
|
403 if err.errno != errno.ENOENT:
|
bos@258
|
404 raise
|
bos@258
|
405 continue
|
bos@258
|
406 if stat.S_ISREG(st.st_mode) and st.st_mode & 0111:
|
bos@545
|
407 if example(pathname, verbose, keep_change).run():
|
bos@258
|
408 errs += 1
|
bos@258
|
409 print >> open(os.path.join(path, '.run'), 'w'), time.asctime()
|
bos@258
|
410 else:
|
bos@258
|
411 print_help(1, msg='no test names given, and --all not provided')
|
bos@566
|
412
|
bos@566
|
413 fp = open('auto-snippets.xml', 'w')
|
bos@566
|
414 for key in sorted(example.entities.iterkeys()):
|
bos@566
|
415 print >> fp, key
|
bos@566
|
416 fp.close()
|
bos@71
|
417 return errs
|
bos@3
|
418
|
bos@3
|
419 if __name__ == '__main__':
|
bos@258
|
420 try:
|
bos@258
|
421 sys.exit(main())
|
bos@258
|
422 except KeyboardInterrupt:
|
bos@258
|
423 print >> sys.stderr, 'interrupted!'
|
bos@258
|
424 sys.exit(1)
|