433 lines
15 KiB
Python
433 lines
15 KiB
Python
|
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
|
||
|
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
|
||
|
#!/usr/bin/env python2.4
|
||
|
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
|
||
|
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
|
||
|
|
||
|
"""
|
||
|
These are functions for use when doctest-testing a document.
|
||
|
"""
|
||
|
|
||
|
import subprocess
|
||
|
import doctest
|
||
|
import os
|
||
|
import sys
|
||
|
import shutil
|
||
|
import re
|
||
|
import cgi
|
||
|
import rfc822
|
||
|
from cStringIO import StringIO
|
||
|
from paste.util import PySourceColor
|
||
|
|
||
|
|
||
|
here = os.path.abspath(__file__)
|
||
|
paste_parent = os.path.dirname(
|
||
|
os.path.dirname(os.path.dirname(here)))
|
||
|
|
||
|
def run(command):
|
||
|
data = run_raw(command)
|
||
|
if data:
|
||
|
print(data)
|
||
|
|
||
|
def run_raw(command):
|
||
|
"""
|
||
|
Runs the string command, returns any output.
|
||
|
"""
|
||
|
proc = subprocess.Popen(command, shell=True,
|
||
|
stderr=subprocess.STDOUT,
|
||
|
stdout=subprocess.PIPE, env=_make_env())
|
||
|
data = proc.stdout.read()
|
||
|
proc.wait()
|
||
|
while data.endswith('\n') or data.endswith('\r'):
|
||
|
data = data[:-1]
|
||
|
if data:
|
||
|
data = '\n'.join(
|
||
|
[l for l in data.splitlines() if l])
|
||
|
return data
|
||
|
else:
|
||
|
return ''
|
||
|
|
||
|
def run_command(command, name, and_print=False):
|
||
|
output = run_raw(command)
|
||
|
data = '$ %s\n%s' % (command, output)
|
||
|
show_file('shell-command', name, description='shell transcript',
|
||
|
data=data)
|
||
|
if and_print and output:
|
||
|
print(output)
|
||
|
|
||
|
def _make_env():
|
||
|
env = os.environ.copy()
|
||
|
env['PATH'] = (env.get('PATH', '')
|
||
|
+ ':'
|
||
|
+ os.path.join(paste_parent, 'scripts')
|
||
|
+ ':'
|
||
|
+ os.path.join(paste_parent, 'paste', '3rd-party',
|
||
|
'sqlobject-files', 'scripts'))
|
||
|
env['PYTHONPATH'] = (env.get('PYTHONPATH', '')
|
||
|
+ ':'
|
||
|
+ paste_parent)
|
||
|
return env
|
||
|
|
||
|
def clear_dir(dir):
|
||
|
"""
|
||
|
Clears (deletes) the given directory
|
||
|
"""
|
||
|
shutil.rmtree(dir, True)
|
||
|
|
||
|
def ls(dir=None, recurse=False, indent=0):
|
||
|
"""
|
||
|
Show a directory listing
|
||
|
"""
|
||
|
dir = dir or os.getcwd()
|
||
|
fns = os.listdir(dir)
|
||
|
fns.sort()
|
||
|
for fn in fns:
|
||
|
full = os.path.join(dir, fn)
|
||
|
if os.path.isdir(full):
|
||
|
fn = fn + '/'
|
||
|
print(' '*indent + fn)
|
||
|
if os.path.isdir(full) and recurse:
|
||
|
ls(dir=full, recurse=True, indent=indent+2)
|
||
|
|
||
|
default_app = None
|
||
|
default_url = None
|
||
|
|
||
|
def set_default_app(app, url):
|
||
|
global default_app
|
||
|
global default_url
|
||
|
default_app = app
|
||
|
default_url = url
|
||
|
|
||
|
def resource_filename(fn):
|
||
|
"""
|
||
|
Returns the filename of the resource -- generally in the directory
|
||
|
resources/DocumentName/fn
|
||
|
"""
|
||
|
return os.path.join(
|
||
|
os.path.dirname(sys.testing_document_filename),
|
||
|
'resources',
|
||
|
os.path.splitext(os.path.basename(sys.testing_document_filename))[0],
|
||
|
fn)
|
||
|
|
||
|
def show(path_info, example_name):
|
||
|
fn = resource_filename(example_name + '.html')
|
||
|
out = StringIO()
|
||
|
assert default_app is not None, (
|
||
|
"No default_app set")
|
||
|
url = default_url + path_info
|
||
|
out.write('<span class="doctest-url"><a href="%s">%s</a></span><br>\n'
|
||
|
% (url, url))
|
||
|
out.write('<div class="doctest-example">\n')
|
||
|
proc = subprocess.Popen(
|
||
|
['paster', 'serve' '--server=console', '--no-verbose',
|
||
|
'--url=' + path_info],
|
||
|
stderr=subprocess.PIPE,
|
||
|
stdout=subprocess.PIPE,
|
||
|
env=_make_env())
|
||
|
stdout, errors = proc.communicate()
|
||
|
stdout = StringIO(stdout)
|
||
|
headers = rfc822.Message(stdout)
|
||
|
content = stdout.read()
|
||
|
for header, value in headers.items():
|
||
|
if header.lower() == 'status' and int(value.split()[0]) == 200:
|
||
|
continue
|
||
|
if header.lower() in ('content-type', 'content-length'):
|
||
|
continue
|
||
|
if (header.lower() == 'set-cookie'
|
||
|
and value.startswith('_SID_')):
|
||
|
continue
|
||
|
out.write('<span class="doctest-header">%s: %s</span><br>\n'
|
||
|
% (header, value))
|
||
|
lines = [l for l in content.splitlines() if l.strip()]
|
||
|
for line in lines:
|
||
|
out.write(line + '\n')
|
||
|
if errors:
|
||
|
out.write('<pre class="doctest-errors">%s</pre>'
|
||
|
% errors)
|
||
|
out.write('</div>\n')
|
||
|
result = out.getvalue()
|
||
|
if not os.path.exists(fn):
|
||
|
f = open(fn, 'wb')
|
||
|
f.write(result)
|
||
|
f.close()
|
||
|
else:
|
||
|
f = open(fn, 'rb')
|
||
|
expected = f.read()
|
||
|
f.close()
|
||
|
if not html_matches(expected, result):
|
||
|
print('Pages did not match. Expected from %s:' % fn)
|
||
|
print('-'*60)
|
||
|
print(expected)
|
||
|
print('='*60)
|
||
|
print('Actual output:')
|
||
|
print('-'*60)
|
||
|
print(result)
|
||
|
|
||
|
def html_matches(pattern, text):
|
||
|
regex = re.escape(pattern)
|
||
|
regex = regex.replace(r'\.\.\.', '.*')
|
||
|
regex = re.sub(r'0x[0-9a-f]+', '.*', regex)
|
||
|
regex = '^%s$' % regex
|
||
|
return re.search(regex, text)
|
||
|
|
||
|
def convert_docstring_string(data):
|
||
|
if data.startswith('\n'):
|
||
|
data = data[1:]
|
||
|
lines = data.splitlines()
|
||
|
new_lines = []
|
||
|
for line in lines:
|
||
|
if line.rstrip() == '.':
|
||
|
new_lines.append('')
|
||
|
else:
|
||
|
new_lines.append(line)
|
||
|
data = '\n'.join(new_lines) + '\n'
|
||
|
return data
|
||
|
|
||
|
def create_file(path, version, data):
|
||
|
data = convert_docstring_string(data)
|
||
|
write_data(path, data)
|
||
|
show_file(path, version)
|
||
|
|
||
|
def append_to_file(path, version, data):
|
||
|
data = convert_docstring_string(data)
|
||
|
f = open(path, 'a')
|
||
|
f.write(data)
|
||
|
f.close()
|
||
|
# I think these appends can happen so quickly (in less than a second)
|
||
|
# that the .pyc file doesn't appear to be expired, even though it
|
||
|
# is after we've made this change; so we have to get rid of the .pyc
|
||
|
# file:
|
||
|
if path.endswith('.py'):
|
||
|
pyc_file = path + 'c'
|
||
|
if os.path.exists(pyc_file):
|
||
|
os.unlink(pyc_file)
|
||
|
show_file(path, version, description='added to %s' % path,
|
||
|
data=data)
|
||
|
|
||
|
def show_file(path, version, description=None, data=None):
|
||
|
ext = os.path.splitext(path)[1]
|
||
|
if data is None:
|
||
|
f = open(path, 'rb')
|
||
|
data = f.read()
|
||
|
f.close()
|
||
|
if ext == '.py':
|
||
|
html = ('<div class="source-code">%s</div>'
|
||
|
% PySourceColor.str2html(data, PySourceColor.dark))
|
||
|
else:
|
||
|
html = '<pre class="source-code">%s</pre>' % cgi.escape(data, 1)
|
||
|
html = '<span class="source-filename">%s</span><br>%s' % (
|
||
|
description or path, html)
|
||
|
write_data(resource_filename('%s.%s.gen.html' % (path, version)),
|
||
|
html)
|
||
|
|
||
|
def call_source_highlight(input, format):
|
||
|
proc = subprocess.Popen(['source-highlight', '--out-format=html',
|
||
|
'--no-doc', '--css=none',
|
||
|
'--src-lang=%s' % format], shell=False,
|
||
|
stdout=subprocess.PIPE)
|
||
|
stdout, stderr = proc.communicate(input)
|
||
|
result = stdout
|
||
|
proc.wait()
|
||
|
return result
|
||
|
|
||
|
|
||
|
def write_data(path, data):
|
||
|
dir = os.path.dirname(os.path.abspath(path))
|
||
|
if not os.path.exists(dir):
|
||
|
os.makedirs(dir)
|
||
|
f = open(path, 'wb')
|
||
|
f.write(data)
|
||
|
f.close()
|
||
|
|
||
|
|
||
|
def change_file(path, changes):
|
||
|
f = open(os.path.abspath(path), 'rb')
|
||
|
lines = f.readlines()
|
||
|
f.close()
|
||
|
for change_type, line, text in changes:
|
||
|
if change_type == 'insert':
|
||
|
lines[line:line] = [text]
|
||
|
elif change_type == 'delete':
|
||
|
lines[line:text] = []
|
||
|
else:
|
||
|
assert 0, (
|
||
|
"Unknown change_type: %r" % change_type)
|
||
|
f = open(path, 'wb')
|
||
|
f.write(''.join(lines))
|
||
|
f.close()
|
||
|
|
||
|
class LongFormDocTestParser(doctest.DocTestParser):
|
||
|
|
||
|
"""
|
||
|
This parser recognizes some reST comments as commands, without
|
||
|
prompts or expected output, like:
|
||
|
|
||
|
.. run:
|
||
|
|
||
|
do_this(...
|
||
|
...)
|
||
|
"""
|
||
|
|
||
|
_EXAMPLE_RE = re.compile(r"""
|
||
|
# Source consists of a PS1 line followed by zero or more PS2 lines.
|
||
|
(?: (?P<source>
|
||
|
(?:^(?P<indent> [ ]*) >>> .*) # PS1 line
|
||
|
(?:\n [ ]* \.\.\. .*)*) # PS2 lines
|
||
|
\n?
|
||
|
# Want consists of any non-blank lines that do not start with PS1.
|
||
|
(?P<want> (?:(?![ ]*$) # Not a blank line
|
||
|
(?![ ]*>>>) # Not a line starting with PS1
|
||
|
.*$\n? # But any other line
|
||
|
)*))
|
||
|
|
|
||
|
(?: # This is for longer commands that are prefixed with a reST
|
||
|
# comment like '.. run:' (two colons makes that a directive).
|
||
|
# These commands cannot have any output.
|
||
|
|
||
|
(?:^\.\.[ ]*(?P<run>run):[ ]*\n) # Leading command/command
|
||
|
(?:[ ]*\n)? # Blank line following
|
||
|
(?P<runsource>
|
||
|
(?:(?P<runindent> [ ]+)[^ ].*$)
|
||
|
(?:\n [ ]+ .*)*)
|
||
|
)
|
||
|
|
|
||
|
(?: # This is for shell commands
|
||
|
|
||
|
(?P<shellsource>
|
||
|
(?:^(P<shellindent> [ ]*) [$] .*) # Shell line
|
||
|
(?:\n [ ]* [>] .*)*) # Continuation
|
||
|
\n?
|
||
|
# Want consists of any non-blank lines that do not start with $
|
||
|
(?P<shellwant> (?:(?![ ]*$)
|
||
|
(?![ ]*[$]$)
|
||
|
.*$\n?
|
||
|
)*))
|
||
|
""", re.MULTILINE | re.VERBOSE)
|
||
|
|
||
|
def _parse_example(self, m, name, lineno):
|
||
|
r"""
|
||
|
Given a regular expression match from `_EXAMPLE_RE` (`m`),
|
||
|
return a pair `(source, want)`, where `source` is the matched
|
||
|
example's source code (with prompts and indentation stripped);
|
||
|
and `want` is the example's expected output (with indentation
|
||
|
stripped).
|
||
|
|
||
|
`name` is the string's name, and `lineno` is the line number
|
||
|
where the example starts; both are used for error messages.
|
||
|
|
||
|
>>> def parseit(s):
|
||
|
... p = LongFormDocTestParser()
|
||
|
... return p._parse_example(p._EXAMPLE_RE.search(s), '<string>', 1)
|
||
|
>>> parseit('>>> 1\n1')
|
||
|
('1', {}, '1', None)
|
||
|
>>> parseit('>>> (1\n... +1)\n2')
|
||
|
('(1\n+1)', {}, '2', None)
|
||
|
>>> parseit('.. run:\n\n test1\n test2\n')
|
||
|
('test1\ntest2', {}, '', None)
|
||
|
"""
|
||
|
# Get the example's indentation level.
|
||
|
runner = m.group('run') or ''
|
||
|
indent = len(m.group('%sindent' % runner))
|
||
|
|
||
|
# Divide source into lines; check that they're properly
|
||
|
# indented; and then strip their indentation & prompts.
|
||
|
source_lines = m.group('%ssource' % runner).split('\n')
|
||
|
if runner:
|
||
|
self._check_prefix(source_lines[1:], ' '*indent, name, lineno)
|
||
|
else:
|
||
|
self._check_prompt_blank(source_lines, indent, name, lineno)
|
||
|
self._check_prefix(source_lines[2:], ' '*indent + '.', name, lineno)
|
||
|
if runner:
|
||
|
source = '\n'.join([sl[indent:] for sl in source_lines])
|
||
|
else:
|
||
|
source = '\n'.join([sl[indent+4:] for sl in source_lines])
|
||
|
|
||
|
if runner:
|
||
|
want = ''
|
||
|
exc_msg = None
|
||
|
else:
|
||
|
# Divide want into lines; check that it's properly indented; and
|
||
|
# then strip the indentation. Spaces before the last newline should
|
||
|
# be preserved, so plain rstrip() isn't good enough.
|
||
|
want = m.group('want')
|
||
|
want_lines = want.split('\n')
|
||
|
if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
|
||
|
del want_lines[-1] # forget final newline & spaces after it
|
||
|
self._check_prefix(want_lines, ' '*indent, name,
|
||
|
lineno + len(source_lines))
|
||
|
want = '\n'.join([wl[indent:] for wl in want_lines])
|
||
|
|
||
|
# If `want` contains a traceback message, then extract it.
|
||
|
m = self._EXCEPTION_RE.match(want)
|
||
|
if m:
|
||
|
exc_msg = m.group('msg')
|
||
|
else:
|
||
|
exc_msg = None
|
||
|
|
||
|
# Extract options from the source.
|
||
|
options = self._find_options(source, name, lineno)
|
||
|
|
||
|
return source, options, want, exc_msg
|
||
|
|
||
|
|
||
|
def parse(self, string, name='<string>'):
|
||
|
"""
|
||
|
Divide the given string into examples and intervening text,
|
||
|
and return them as a list of alternating Examples and strings.
|
||
|
Line numbers for the Examples are 0-based. The optional
|
||
|
argument `name` is a name identifying this string, and is only
|
||
|
used for error messages.
|
||
|
"""
|
||
|
string = string.expandtabs()
|
||
|
# If all lines begin with the same indentation, then strip it.
|
||
|
min_indent = self._min_indent(string)
|
||
|
if min_indent > 0:
|
||
|
string = '\n'.join([l[min_indent:] for l in string.split('\n')])
|
||
|
|
||
|
output = []
|
||
|
charno, lineno = 0, 0
|
||
|
# Find all doctest examples in the string:
|
||
|
for m in self._EXAMPLE_RE.finditer(string):
|
||
|
# Add the pre-example text to `output`.
|
||
|
output.append(string[charno:m.start()])
|
||
|
# Update lineno (lines before this example)
|
||
|
lineno += string.count('\n', charno, m.start())
|
||
|
# Extract info from the regexp match.
|
||
|
(source, options, want, exc_msg) = \
|
||
|
self._parse_example(m, name, lineno)
|
||
|
# Create an Example, and add it to the list.
|
||
|
if not self._IS_BLANK_OR_COMMENT(source):
|
||
|
# @@: Erg, this is the only line I need to change...
|
||
|
output.append(doctest.Example(
|
||
|
source, want, exc_msg,
|
||
|
lineno=lineno,
|
||
|
indent=min_indent+len(m.group('indent') or m.group('runindent')),
|
||
|
options=options))
|
||
|
# Update lineno (lines inside this example)
|
||
|
lineno += string.count('\n', m.start(), m.end())
|
||
|
# Update charno.
|
||
|
charno = m.end()
|
||
|
# Add any remaining post-example text to `output`.
|
||
|
output.append(string[charno:])
|
||
|
return output
|
||
|
|
||
|
|
||
|
|
||
|
if __name__ == '__main__':
|
||
|
if sys.argv[1:] and sys.argv[1] == 'doctest':
|
||
|
doctest.testmod()
|
||
|
sys.exit()
|
||
|
if not paste_parent in sys.path:
|
||
|
sys.path.append(paste_parent)
|
||
|
for fn in sys.argv[1:]:
|
||
|
fn = os.path.abspath(fn)
|
||
|
# @@: OK, ick; but this module gets loaded twice
|
||
|
sys.testing_document_filename = fn
|
||
|
doctest.testfile(
|
||
|
fn, module_relative=False,
|
||
|
optionflags=doctest.ELLIPSIS|doctest.REPORT_ONLY_FIRST_FAILURE,
|
||
|
parser=LongFormDocTestParser())
|
||
|
new = os.path.splitext(fn)[0] + '.html'
|
||
|
assert new != fn
|
||
|
os.system('rst2html.py %s > %s' % (fn, new))
|