rev |
line source |
bos@67
|
1 #!/usr/bin/env python
|
bos@4
|
2 #
|
bos@4
|
3 # This program takes something that resembles a shell script and runs
|
bos@4
|
4 # it, spitting input (commands from the script) and output into text
|
bos@4
|
5 # files, for use in examples.
|
bos@3
|
6
|
bos@3
|
7 import cStringIO
|
bos@73
|
8 import errno
|
bos@78
|
9 import getopt
|
bos@3
|
10 import os
|
bos@3
|
11 import pty
|
bos@3
|
12 import re
|
bos@79
|
13 import select
|
bos@4
|
14 import shutil
|
bos@6
|
15 import signal
|
bos@36
|
16 import stat
|
bos@3
|
17 import sys
|
bos@4
|
18 import tempfile
|
bos@4
|
19 import time
|
bos@3
|
20
|
bos@77
|
21 tex_subs = {
|
bos@77
|
22 '\\': '\\textbackslash{}',
|
bos@77
|
23 '{': '\\{',
|
bos@77
|
24 '}': '\\}',
|
bos@77
|
25 }
|
bos@77
|
26
|
bos@77
|
27 def gensubs(s):
|
bos@77
|
28 start = 0
|
bos@77
|
29 for i, c in enumerate(s):
|
bos@77
|
30 sub = tex_subs.get(c)
|
bos@77
|
31 if sub:
|
bos@77
|
32 yield s[start:i]
|
bos@77
|
33 start = i + 1
|
bos@77
|
34 yield sub
|
bos@77
|
35 yield s[start:]
|
bos@77
|
36
|
bos@4
|
37 def tex_escape(s):
|
bos@77
|
38 return ''.join(gensubs(s))
|
bos@4
|
39
|
bos@137
|
40 def maybe_unlink(name):
|
bos@137
|
41 try:
|
bos@137
|
42 os.unlink(name)
|
bos@137
|
43 return True
|
bos@137
|
44 except OSError, err:
|
bos@137
|
45 if err.errno != errno.ENOENT:
|
bos@137
|
46 raise
|
bos@137
|
47 return False
|
bos@137
|
48
|
bos@3
|
49 class example:
|
bos@70
|
50 shell = '/usr/bin/env bash'
|
bos@103
|
51 ps1 = '__run_example_ps1__ '
|
bos@103
|
52 ps2 = '__run_example_ps2__ '
|
bos@137
|
53 pi_re = re.compile(r'#\$\s*(name|ignore):\s*(.*)$')
|
bos@4
|
54
|
bos@79
|
55 timeout = 5
|
bos@79
|
56
|
bos@78
|
57 def __init__(self, name, verbose):
|
bos@3
|
58 self.name = name
|
bos@78
|
59 self.verbose = verbose
|
bos@79
|
60 self.poll = select.poll()
|
bos@3
|
61
|
bos@3
|
62 def parse(self):
|
bos@4
|
63 '''yield each hunk of input from the file.'''
|
bos@3
|
64 fp = open(self.name)
|
bos@3
|
65 cfp = cStringIO.StringIO()
|
bos@3
|
66 for line in fp:
|
bos@3
|
67 cfp.write(line)
|
bos@3
|
68 if not line.rstrip().endswith('\\'):
|
bos@3
|
69 yield cfp.getvalue()
|
bos@3
|
70 cfp.seek(0)
|
bos@3
|
71 cfp.truncate()
|
bos@3
|
72
|
bos@3
|
73 def status(self, s):
|
bos@3
|
74 sys.stdout.write(s)
|
bos@3
|
75 if not s.endswith('\n'):
|
bos@3
|
76 sys.stdout.flush()
|
bos@3
|
77
|
bos@6
|
78 def send(self, s):
|
bos@78
|
79 if self.verbose:
|
bos@78
|
80 print >> sys.stderr, '>', self.debugrepr(s)
|
bos@73
|
81 while s:
|
bos@73
|
82 count = os.write(self.cfd, s)
|
bos@73
|
83 s = s[count:]
|
bos@6
|
84
|
bos@78
|
85 def debugrepr(self, s):
|
bos@78
|
86 rs = repr(s)
|
bos@78
|
87 limit = 60
|
bos@78
|
88 if len(rs) > limit:
|
bos@78
|
89 return ('%s%s ... [%d bytes]' % (rs[:limit], rs[0], len(s)))
|
bos@78
|
90 else:
|
bos@78
|
91 return rs
|
bos@78
|
92
|
bos@79
|
93 timeout = 5
|
bos@79
|
94
|
bos@79
|
95 def read(self):
|
bos@79
|
96 events = self.poll.poll(self.timeout * 1000)
|
bos@79
|
97 if not events:
|
bos@79
|
98 print >> sys.stderr, '[timed out after %d seconds]' % self.timeout
|
bos@79
|
99 os.kill(self.pid, signal.SIGHUP)
|
bos@79
|
100 return ''
|
bos@79
|
101 return os.read(self.cfd, 1024)
|
bos@79
|
102
|
bos@6
|
103 def receive(self):
|
bos@6
|
104 out = cStringIO.StringIO()
|
bos@4
|
105 while True:
|
bos@73
|
106 try:
|
bos@78
|
107 if self.verbose:
|
bos@78
|
108 sys.stderr.write('< ')
|
bos@79
|
109 s = self.read()
|
bos@73
|
110 except OSError, err:
|
bos@73
|
111 if err.errno == errno.EIO:
|
bos@103
|
112 return '', ''
|
bos@73
|
113 raise
|
bos@78
|
114 if self.verbose:
|
bos@78
|
115 print >> sys.stderr, self.debugrepr(s)
|
bos@6
|
116 out.write(s)
|
bos@73
|
117 s = out.getvalue()
|
bos@103
|
118 if s.endswith(self.ps1):
|
bos@103
|
119 return self.ps1, s.replace('\r\n', '\n')[:-len(self.ps1)]
|
bos@103
|
120 if s.endswith(self.ps2):
|
bos@103
|
121 return self.ps2, s.replace('\r\n', '\n')[:-len(self.ps2)]
|
bos@4
|
122
|
bos@6
|
123 def sendreceive(self, s):
|
bos@6
|
124 self.send(s)
|
bos@103
|
125 ps, r = self.receive()
|
bos@6
|
126 if r.startswith(s):
|
bos@6
|
127 r = r[len(s):]
|
bos@103
|
128 return ps, r
|
bos@6
|
129
|
bos@3
|
130 def run(self):
|
bos@3
|
131 ofp = None
|
bos@4
|
132 basename = os.path.basename(self.name)
|
bos@4
|
133 self.status('running %s ' % basename)
|
bos@4
|
134 tmpdir = tempfile.mkdtemp(prefix=basename)
|
bos@78
|
135
|
bos@136
|
136 # remove the marker file that we tell make to use to see if
|
bos@136
|
137 # this run succeeded
|
bos@137
|
138 maybe_unlink(self.name + '.run')
|
bos@136
|
139
|
bos@78
|
140 rcfile = os.path.join(tmpdir, '.hgrc')
|
bos@78
|
141 rcfp = open(rcfile, 'w')
|
bos@78
|
142 print >> rcfp, '[ui]'
|
bos@78
|
143 print >> rcfp, "username = Bryan O'Sullivan <bos@serpentine.com>"
|
bos@78
|
144
|
bos@6
|
145 rcfile = os.path.join(tmpdir, '.bashrc')
|
bos@6
|
146 rcfp = open(rcfile, 'w')
|
bos@103
|
147 print >> rcfp, 'PS1="%s"' % self.ps1
|
bos@103
|
148 print >> rcfp, 'PS2="%s"' % self.ps2
|
bos@6
|
149 print >> rcfp, 'unset HISTFILE'
|
bos@19
|
150 print >> rcfp, 'export EXAMPLE_DIR="%s"' % os.getcwd()
|
bos@124
|
151 print >> rcfp, 'export HGMERGE=merge'
|
bos@6
|
152 print >> rcfp, 'export LANG=C'
|
bos@6
|
153 print >> rcfp, 'export LC_ALL=C'
|
bos@6
|
154 print >> rcfp, 'export TZ=GMT'
|
bos@6
|
155 print >> rcfp, 'export HGRC="%s/.hgrc"' % tmpdir
|
bos@6
|
156 print >> rcfp, 'export HGRCPATH=$HGRC'
|
bos@6
|
157 print >> rcfp, 'cd %s' % tmpdir
|
bos@6
|
158 rcfp.close()
|
bos@68
|
159 sys.stdout.flush()
|
bos@68
|
160 sys.stderr.flush()
|
bos@79
|
161 self.pid, self.cfd = pty.fork()
|
bos@79
|
162 if self.pid == 0:
|
bos@70
|
163 cmdline = ['/usr/bin/env', 'bash', '--noediting', '--noprofile',
|
bos@70
|
164 '--norc']
|
bos@68
|
165 try:
|
bos@68
|
166 os.execv(cmdline[0], cmdline)
|
bos@68
|
167 except OSError, err:
|
bos@68
|
168 print >> sys.stderr, '%s: %s' % (cmdline[0], err.strerror)
|
bos@68
|
169 sys.stderr.flush()
|
bos@68
|
170 os._exit(0)
|
bos@79
|
171 self.poll.register(self.cfd, select.POLLIN | select.POLLERR |
|
bos@79
|
172 select.POLLHUP)
|
bos@103
|
173
|
bos@103
|
174 prompts = {
|
bos@103
|
175 '': '',
|
bos@103
|
176 self.ps1: '$',
|
bos@103
|
177 self.ps2: '>',
|
bos@103
|
178 }
|
bos@103
|
179
|
bos@137
|
180 ignore = [
|
bos@137
|
181 r'\d+:[0-9a-f]{12}', # changeset number:hash
|
bos@141
|
182 r'[0-9a-f]{40}', # long changeset hash
|
bos@141
|
183 r'[0-9a-f]{12}', # short changeset hash
|
bos@137
|
184 r'^(?:---|\+\+\+) .*', # diff header with dates
|
bos@137
|
185 r'^date:.*', # date
|
bos@141
|
186 #r'^diff -r.*', # "diff -r" is followed by hash
|
bos@138
|
187 r'^# Date \d+ \d+', # hg patch header
|
bos@137
|
188 ]
|
bos@137
|
189
|
bos@138
|
190 err = False
|
bos@138
|
191
|
bos@4
|
192 try:
|
bos@71
|
193 try:
|
bos@73
|
194 # eat first prompt string from shell
|
bos@79
|
195 self.read()
|
bos@71
|
196 # setup env and prompt
|
bos@103
|
197 ps, output = self.sendreceive('source %s\n' % rcfile)
|
bos@71
|
198 for hunk in self.parse():
|
bos@71
|
199 # is this line a processing instruction?
|
bos@71
|
200 m = self.pi_re.match(hunk)
|
bos@71
|
201 if m:
|
bos@71
|
202 pi, rest = m.groups()
|
bos@71
|
203 if pi == 'name':
|
bos@71
|
204 self.status('.')
|
bos@71
|
205 out = rest
|
bos@154
|
206 assert out not in ('err', 'lxo', 'out', 'run', 'tmp')
|
bos@71
|
207 assert os.sep not in out
|
bos@137
|
208 if ofp is not None:
|
bos@137
|
209 ofp.close()
|
bos@138
|
210 err = self.rename_output(ofp_basename, ignore)
|
bos@71
|
211 if out:
|
bos@137
|
212 ofp_basename = '%s.%s' % (self.name, out)
|
bos@137
|
213 ofp = open(ofp_basename + '.tmp', 'w')
|
bos@71
|
214 else:
|
bos@71
|
215 ofp = None
|
bos@137
|
216 elif pi == 'ignore':
|
bos@137
|
217 ignore.append(rest)
|
bos@71
|
218 elif hunk.strip():
|
bos@71
|
219 # it's something we should execute
|
bos@103
|
220 newps, output = self.sendreceive(hunk)
|
bos@71
|
221 if not ofp:
|
bos@71
|
222 continue
|
bos@71
|
223 # first, print the command we ran
|
bos@71
|
224 if not hunk.startswith('#'):
|
bos@71
|
225 nl = hunk.endswith('\n')
|
bos@103
|
226 hunk = ('%s \\textbf{%s}' %
|
bos@103
|
227 (prompts[ps],
|
bos@103
|
228 tex_escape(hunk.rstrip('\n'))))
|
bos@71
|
229 if nl: hunk += '\n'
|
bos@71
|
230 ofp.write(hunk)
|
bos@71
|
231 # then its output
|
bos@71
|
232 ofp.write(tex_escape(output))
|
bos@103
|
233 ps = newps
|
bos@71
|
234 self.status('\n')
|
bos@71
|
235 except:
|
bos@72
|
236 print >> sys.stderr, '(killed)'
|
bos@79
|
237 os.kill(self.pid, signal.SIGKILL)
|
bos@72
|
238 pid, rc = os.wait()
|
bos@71
|
239 raise
|
bos@72
|
240 else:
|
bos@71
|
241 try:
|
bos@103
|
242 ps, output = self.sendreceive('exit\n')
|
bos@138
|
243 if ofp is not None:
|
bos@71
|
244 ofp.write(output)
|
bos@138
|
245 ofp.close()
|
bos@138
|
246 err = self.rename_output(ofp_basename, ignore)
|
bos@73
|
247 os.close(self.cfd)
|
bos@71
|
248 except IOError:
|
bos@71
|
249 pass
|
bos@79
|
250 os.kill(self.pid, signal.SIGTERM)
|
bos@72
|
251 pid, rc = os.wait()
|
bos@72
|
252 if rc:
|
bos@72
|
253 if os.WIFEXITED(rc):
|
bos@72
|
254 print >> sys.stderr, '(exit %s)' % os.WEXITSTATUS(rc)
|
bos@72
|
255 elif os.WIFSIGNALED(rc):
|
bos@72
|
256 print >> sys.stderr, '(signal %s)' % os.WTERMSIG(rc)
|
bos@136
|
257 else:
|
bos@136
|
258 open(self.name + '.run', 'w')
|
bos@138
|
259 return rc or err
|
bos@72
|
260 finally:
|
bos@4
|
261 shutil.rmtree(tmpdir)
|
bos@3
|
262
|
bos@137
|
263 def rename_output(self, base, ignore):
|
bos@137
|
264 mangle_re = re.compile('(?:' + '|'.join(ignore) + ')')
|
bos@137
|
265 def mangle(s):
|
bos@137
|
266 return mangle_re.sub('', s)
|
bos@137
|
267 def matchfp(fp1, fp2):
|
bos@137
|
268 while True:
|
bos@137
|
269 s1 = mangle(fp1.readline())
|
bos@137
|
270 s2 = mangle(fp2.readline())
|
bos@137
|
271 if cmp(s1, s2):
|
bos@137
|
272 break
|
bos@137
|
273 if not s1:
|
bos@137
|
274 return True
|
bos@137
|
275 return False
|
bos@137
|
276
|
bos@137
|
277 oldname = base + '.out'
|
bos@137
|
278 tmpname = base + '.tmp'
|
bos@137
|
279 errname = base + '.err'
|
bos@137
|
280 errfp = open(errname, 'w+')
|
bos@137
|
281 for line in open(tmpname):
|
bos@137
|
282 errfp.write(mangle_re.sub('', line))
|
bos@146
|
283 os.rename(tmpname, base + '.lxo')
|
bos@137
|
284 errfp.seek(0)
|
bos@137
|
285 try:
|
bos@137
|
286 oldfp = open(oldname)
|
bos@137
|
287 except IOError, err:
|
bos@137
|
288 if err.errno != errno.ENOENT:
|
bos@137
|
289 raise
|
bos@137
|
290 os.rename(errname, oldname)
|
bos@137
|
291 return
|
bos@137
|
292 if matchfp(oldfp, errfp):
|
bos@137
|
293 os.unlink(errname)
|
bos@137
|
294 else:
|
bos@137
|
295 print >> sys.stderr, '\nOutput of %s has changed!' % base
|
bos@137
|
296 os.system('diff -u %s %s 1>&2' % (oldname, errname))
|
bos@138
|
297 return True
|
bos@137
|
298
|
bos@3
|
299 def main(path='.'):
|
bos@78
|
300 opts, args = getopt.getopt(sys.argv[1:], 'v', ['verbose'])
|
bos@78
|
301 verbose = False
|
bos@78
|
302 for o, a in opts:
|
bos@78
|
303 if o in ('-v', '--verbose'):
|
bos@78
|
304 verbose = True
|
bos@71
|
305 errs = 0
|
bos@3
|
306 if args:
|
bos@3
|
307 for a in args:
|
bos@75
|
308 try:
|
bos@75
|
309 st = os.lstat(a)
|
bos@75
|
310 except OSError, err:
|
bos@75
|
311 print >> sys.stderr, '%s: %s' % (a, err.strerror)
|
bos@75
|
312 errs += 1
|
bos@75
|
313 continue
|
bos@75
|
314 if stat.S_ISREG(st.st_mode) and st.st_mode & 0111:
|
bos@78
|
315 if example(a, verbose).run():
|
bos@75
|
316 errs += 1
|
bos@75
|
317 else:
|
bos@75
|
318 print >> sys.stderr, '%s: not a file, or not executable' % a
|
bos@71
|
319 errs += 1
|
bos@71
|
320 return errs
|
bos@3
|
321 for name in os.listdir(path):
|
bos@3
|
322 if name == 'run-example' or name.startswith('.'): continue
|
bos@3
|
323 if name.endswith('.out') or name.endswith('~'): continue
|
bos@45
|
324 if name.endswith('.run'): continue
|
bos@19
|
325 pathname = os.path.join(path, name)
|
bos@142
|
326 try:
|
bos@142
|
327 st = os.lstat(pathname)
|
bos@142
|
328 except OSError, err:
|
bos@142
|
329 # could be an output file that was removed while we ran
|
bos@142
|
330 if err.errno != errno.ENOENT:
|
bos@142
|
331 raise
|
bos@142
|
332 continue
|
bos@36
|
333 if stat.S_ISREG(st.st_mode) and st.st_mode & 0111:
|
bos@78
|
334 if example(pathname, verbose).run():
|
bos@71
|
335 errs += 1
|
bos@4
|
336 print >> open(os.path.join(path, '.run'), 'w'), time.asctime()
|
bos@71
|
337 return errs
|
bos@3
|
338
|
bos@3
|
339 if __name__ == '__main__':
|
bos@71
|
340 sys.exit(main())
|