hgbook
view en/examples/run-example @ 172:5f305adeb584
Try to tighten up the run environment to make things more reproducible.
| author | Bryan O'Sullivan <bos@serpentine.com> |
|---|---|
| date | Tue Mar 27 15:04:47 2007 -0700 (2007-03-27) |
| parents | f8b5b782e150 |
| children | 754312dc23d5 |
line source
1 #!/usr/bin/env python
2 #
3 # This program takes something that resembles a shell script and runs
4 # it, spitting input (commands from the script) and output into text
5 # files, for use in examples.
7 import cStringIO
8 import errno
9 import getopt
10 import os
11 import pty
12 import re
13 import select
14 import shutil
15 import signal
16 import stat
17 import sys
18 import tempfile
19 import time
21 tex_subs = {
22 '\\': '\\textbackslash{}',
23 '{': '\\{',
24 '}': '\\}',
25 }
27 def gensubs(s):
28 start = 0
29 for i, c in enumerate(s):
30 sub = tex_subs.get(c)
31 if sub:
32 yield s[start:i]
33 start = i + 1
34 yield sub
35 yield s[start:]
37 def tex_escape(s):
38 return ''.join(gensubs(s))
40 def maybe_unlink(name):
41 try:
42 os.unlink(name)
43 return True
44 except OSError, err:
45 if err.errno != errno.ENOENT:
46 raise
47 return False
49 def find_path_to(program):
50 for p in os.environ.get('PATH', os.defpath).split(os.pathsep):
51 name = os.path.join(p, program)
52 if os.access(name, os.X_OK):
53 return p
54 return None
56 class example:
57 shell = '/usr/bin/env bash'
58 ps1 = '__run_example_ps1__ '
59 ps2 = '__run_example_ps2__ '
60 pi_re = re.compile(r'#\$\s*(name|ignore):\s*(.*)$')
62 timeout = 5
64 def __init__(self, name, verbose):
65 self.name = name
66 self.verbose = verbose
67 self.poll = select.poll()
69 def parse(self):
70 '''yield each hunk of input from the file.'''
71 fp = open(self.name)
72 cfp = cStringIO.StringIO()
73 for line in fp:
74 cfp.write(line)
75 if not line.rstrip().endswith('\\'):
76 yield cfp.getvalue()
77 cfp.seek(0)
78 cfp.truncate()
80 def status(self, s):
81 sys.stdout.write(s)
82 if not s.endswith('\n'):
83 sys.stdout.flush()
85 def send(self, s):
86 if self.verbose:
87 print >> sys.stderr, '>', self.debugrepr(s)
88 while s:
89 count = os.write(self.cfd, s)
90 s = s[count:]
92 def debugrepr(self, s):
93 rs = repr(s)
94 limit = 60
95 if len(rs) > limit:
96 return ('%s%s ... [%d bytes]' % (rs[:limit], rs[0], len(s)))
97 else:
98 return rs
100 timeout = 5
102 def read(self):
103 events = self.poll.poll(self.timeout * 1000)
104 if not events:
105 print >> sys.stderr, '[timed out after %d seconds]' % self.timeout
106 os.kill(self.pid, signal.SIGHUP)
107 return ''
108 return os.read(self.cfd, 1024)
110 def receive(self):
111 out = cStringIO.StringIO()
112 while True:
113 try:
114 if self.verbose:
115 sys.stderr.write('< ')
116 s = self.read()
117 except OSError, err:
118 if err.errno == errno.EIO:
119 return '', ''
120 raise
121 if self.verbose:
122 print >> sys.stderr, self.debugrepr(s)
123 out.write(s)
124 s = out.getvalue()
125 if s.endswith(self.ps1):
126 return self.ps1, s.replace('\r\n', '\n')[:-len(self.ps1)]
127 if s.endswith(self.ps2):
128 return self.ps2, s.replace('\r\n', '\n')[:-len(self.ps2)]
130 def sendreceive(self, s):
131 self.send(s)
132 ps, r = self.receive()
133 if r.startswith(s):
134 r = r[len(s):]
135 return ps, r
137 def run(self):
138 ofp = None
139 basename = os.path.basename(self.name)
140 self.status('running %s ' % basename)
141 tmpdir = tempfile.mkdtemp(prefix=basename)
143 # remove the marker file that we tell make to use to see if
144 # this run succeeded
145 maybe_unlink(self.name + '.run')
147 rcfile = os.path.join(tmpdir, '.hgrc')
148 rcfp = open(rcfile, 'w')
149 print >> rcfp, '[ui]'
150 print >> rcfp, "username = Bryan O'Sullivan <bos@serpentine.com>"
152 rcfile = os.path.join(tmpdir, '.bashrc')
153 rcfp = open(rcfile, 'w')
154 print >> rcfp, 'PS1="%s"' % self.ps1
155 print >> rcfp, 'PS2="%s"' % self.ps2
156 print >> rcfp, 'unset HISTFILE'
157 path = ['/usr/bin', '/bin']
158 hg = find_path_to('hg')
159 if hg and hg not in path:
160 path.append(hg)
161 def re_export(envar):
162 v = os.getenv(envar)
163 if v is not None:
164 print >> rcfp, 'export ' + envar + '=' + v
165 print >> rcfp, 'export PATH=' + ':'.join(path)
166 re_export('PYTHONPATH')
167 print >> rcfp, 'export EXAMPLE_DIR="%s"' % os.getcwd()
168 print >> rcfp, 'export HGMERGE=merge'
169 print >> rcfp, 'export LANG=C'
170 print >> rcfp, 'export LC_ALL=C'
171 print >> rcfp, 'export TZ=GMT'
172 print >> rcfp, 'export HGRC="%s/.hgrc"' % tmpdir
173 print >> rcfp, 'export HGRCPATH=$HGRC'
174 print >> rcfp, 'cd %s' % tmpdir
175 rcfp.close()
176 sys.stdout.flush()
177 sys.stderr.flush()
178 self.pid, self.cfd = pty.fork()
179 if self.pid == 0:
180 cmdline = ['/usr/bin/env', '-i', 'bash', '--noediting',
181 '--noprofile', '--norc']
182 try:
183 os.execv(cmdline[0], cmdline)
184 except OSError, err:
185 print >> sys.stderr, '%s: %s' % (cmdline[0], err.strerror)
186 sys.stderr.flush()
187 os._exit(0)
188 self.poll.register(self.cfd, select.POLLIN | select.POLLERR |
189 select.POLLHUP)
191 prompts = {
192 '': '',
193 self.ps1: '$',
194 self.ps2: '>',
195 }
197 ignore = [
198 r'\d+:[0-9a-f]{12}', # changeset number:hash
199 r'[0-9a-f]{40}', # long changeset hash
200 r'[0-9a-f]{12}', # short changeset hash
201 r'^(?:---|\+\+\+) .*', # diff header with dates
202 r'^date:.*', # date
203 #r'^diff -r.*', # "diff -r" is followed by hash
204 r'^# Date \d+ \d+', # hg patch header
205 ]
207 err = False
209 try:
210 try:
211 # eat first prompt string from shell
212 self.read()
213 # setup env and prompt
214 ps, output = self.sendreceive('source %s\n' % rcfile)
215 for hunk in self.parse():
216 # is this line a processing instruction?
217 m = self.pi_re.match(hunk)
218 if m:
219 pi, rest = m.groups()
220 if pi == 'name':
221 self.status('.')
222 out = rest
223 if out in ('err', 'lxo', 'out', 'run', 'tmp'):
224 print >> sys.stderr, ('%s: illegal section '
225 'name %r' %
226 (self.name, out))
227 return 1
228 assert os.sep not in out
229 if ofp is not None:
230 ofp.close()
231 err |= self.rename_output(ofp_basename, ignore)
232 if out:
233 ofp_basename = '%s.%s' % (self.name, out)
234 ofp = open(ofp_basename + '.tmp', 'w')
235 else:
236 ofp = None
237 elif pi == 'ignore':
238 ignore.append(rest)
239 elif hunk.strip():
240 # it's something we should execute
241 newps, output = self.sendreceive(hunk)
242 if not ofp:
243 continue
244 # first, print the command we ran
245 if not hunk.startswith('#'):
246 nl = hunk.endswith('\n')
247 hunk = ('%s \\textbf{%s}' %
248 (prompts[ps],
249 tex_escape(hunk.rstrip('\n'))))
250 if nl: hunk += '\n'
251 ofp.write(hunk)
252 # then its output
253 ofp.write(tex_escape(output))
254 ps = newps
255 self.status('\n')
256 except:
257 print >> sys.stderr, '(killed)'
258 os.kill(self.pid, signal.SIGKILL)
259 pid, rc = os.wait()
260 raise
261 else:
262 try:
263 ps, output = self.sendreceive('exit\n')
264 if ofp is not None:
265 ofp.write(output)
266 ofp.close()
267 err |= self.rename_output(ofp_basename, ignore)
268 os.close(self.cfd)
269 except IOError:
270 pass
271 os.kill(self.pid, signal.SIGTERM)
272 pid, rc = os.wait()
273 err = err or rc
274 if err:
275 if os.WIFEXITED(rc):
276 print >> sys.stderr, '(exit %s)' % os.WEXITSTATUS(rc)
277 elif os.WIFSIGNALED(rc):
278 print >> sys.stderr, '(signal %s)' % os.WTERMSIG(rc)
279 else:
280 open(self.name + '.run', 'w')
281 return err
282 finally:
283 shutil.rmtree(tmpdir)
285 def rename_output(self, base, ignore):
286 mangle_re = re.compile('(?:' + '|'.join(ignore) + ')')
287 def mangle(s):
288 return mangle_re.sub('', s)
289 def matchfp(fp1, fp2):
290 while True:
291 s1 = mangle(fp1.readline())
292 s2 = mangle(fp2.readline())
293 if cmp(s1, s2):
294 break
295 if not s1:
296 return True
297 return False
299 oldname = base + '.out'
300 tmpname = base + '.tmp'
301 errname = base + '.err'
302 errfp = open(errname, 'w+')
303 for line in open(tmpname):
304 errfp.write(mangle_re.sub('', line))
305 os.rename(tmpname, base + '.lxo')
306 errfp.seek(0)
307 try:
308 oldfp = open(oldname)
309 except IOError, err:
310 if err.errno != errno.ENOENT:
311 raise
312 os.rename(errname, oldname)
313 return False
314 if matchfp(oldfp, errfp):
315 os.unlink(errname)
316 return False
317 else:
318 print >> sys.stderr, '\nOutput of %s has changed!' % base
319 os.system('diff -u %s %s 1>&2' % (oldname, errname))
320 return True
322 def main(path='.'):
323 opts, args = getopt.getopt(sys.argv[1:], 'v', ['verbose'])
324 verbose = False
325 for o, a in opts:
326 if o in ('-v', '--verbose'):
327 verbose = True
328 errs = 0
329 if args:
330 for a in args:
331 try:
332 st = os.lstat(a)
333 except OSError, err:
334 print >> sys.stderr, '%s: %s' % (a, err.strerror)
335 errs += 1
336 continue
337 if stat.S_ISREG(st.st_mode) and st.st_mode & 0111:
338 if example(a, verbose).run():
339 errs += 1
340 else:
341 print >> sys.stderr, '%s: not a file, or not executable' % a
342 errs += 1
343 return errs
344 names = os.listdir(path)
345 names.sort()
346 for name in names:
347 if name == 'run-example' or name.startswith('.'): continue
348 if name.endswith('.out') or name.endswith('~'): continue
349 if name.endswith('.run'): continue
350 pathname = os.path.join(path, name)
351 try:
352 st = os.lstat(pathname)
353 except OSError, err:
354 # could be an output file that was removed while we ran
355 if err.errno != errno.ENOENT:
356 raise
357 continue
358 if stat.S_ISREG(st.st_mode) and st.st_mode & 0111:
359 if example(pathname, verbose).run():
360 errs += 1
361 print >> open(os.path.join(path, '.run'), 'w'), time.asctime()
362 return errs
364 if __name__ == '__main__':
365 sys.exit(main())