feat: restructure Makefile with debug/release builds and move scripts to subdirectory
This commit is contained in:
Bob Nystrom
Executable
+232
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
from collections import defaultdict
|
||||
from os import listdir
|
||||
from os.path import abspath, dirname, isdir, isfile, join, realpath, relpath, splitext
|
||||
import re
|
||||
from subprocess import Popen, PIPE
|
||||
import sys
|
||||
|
||||
# Runs the tests.
|
||||
WREN_DIR = dirname(dirname(realpath(__file__)))
|
||||
TEST_DIR = join(WREN_DIR, 'test')
|
||||
WREN_APP = join(WREN_DIR, 'wrend')
|
||||
|
||||
EXPECT_PATTERN = re.compile(r'// expect: (.*)')
|
||||
EXPECT_ERROR_PATTERN = re.compile(r'// expect error')
|
||||
EXPECT_ERROR_LINE_PATTERN = re.compile(r'// expect error line (\d+)')
|
||||
EXPECT_RUNTIME_ERROR_PATTERN = re.compile(r'// expect runtime error: (.+)')
|
||||
ERROR_PATTERN = re.compile(r'\[.* line (\d+)\] Error')
|
||||
STACK_TRACE_PATTERN = re.compile(r'\[.* line (\d+)\] in \(script\)')
|
||||
SKIP_PATTERN = re.compile(r'// skip: (.*)')
|
||||
NONTEST_PATTERN = re.compile(r'// nontest')
|
||||
|
||||
if sys.platform == 'win32':
|
||||
class color:
|
||||
GREEN = ''
|
||||
RED = ''
|
||||
DEFAULT = ''
|
||||
PINK = ''
|
||||
YELLOW = ''
|
||||
else:
|
||||
class color:
|
||||
GREEN = '\033[32m'
|
||||
RED = '\033[31m'
|
||||
DEFAULT = '\033[0m'
|
||||
PINK = '\033[91m'
|
||||
YELLOW = '\033[33m'
|
||||
|
||||
passed = 0
|
||||
failed = 0
|
||||
skipped = defaultdict(int)
|
||||
num_skipped = 0;
|
||||
|
||||
|
||||
def walk(dir, callback):
|
||||
""" Walks [dir], and executes [callback] on each file. """
|
||||
dir = abspath(dir)
|
||||
for file in [file for file in listdir(dir) if not file in [".",".."]]:
|
||||
nfile = join(dir, file)
|
||||
if isdir(nfile):
|
||||
walk(nfile, callback)
|
||||
else:
|
||||
callback(nfile)
|
||||
|
||||
|
||||
def print_line(line=None):
|
||||
# Erase the line.
|
||||
print '\033[2K',
|
||||
# Move the cursor to the beginning.
|
||||
print '\r',
|
||||
if line:
|
||||
print line,
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def run_test(path):
|
||||
global passed
|
||||
global failed
|
||||
global skipped
|
||||
global num_skipped
|
||||
|
||||
if (splitext(path)[1] != '.wren'):
|
||||
return
|
||||
|
||||
# Check if we are just running a subset of the tests.
|
||||
if len(sys.argv) == 2:
|
||||
this_test = relpath(path, join(WREN_DIR, 'test'))
|
||||
if not this_test.startswith(sys.argv[1]):
|
||||
return
|
||||
|
||||
# Make a nice short path relative to the working directory.
|
||||
path = relpath(path)
|
||||
|
||||
# Read the test and parse out the expectations.
|
||||
expect_output = []
|
||||
expect_error = []
|
||||
expect_runtime_error_line = 0
|
||||
expect_runtime_error = None
|
||||
expect_return = 0
|
||||
|
||||
print_line('Passed: ' + color.GREEN + str(passed) + color.DEFAULT +
|
||||
' Failed: ' + color.RED + str(failed) + color.DEFAULT +
|
||||
' Skipped: ' + color.YELLOW + str(num_skipped) + color.DEFAULT)
|
||||
|
||||
line_num = 1
|
||||
with open(path, 'r') as file:
|
||||
for line in file:
|
||||
match = EXPECT_PATTERN.search(line)
|
||||
if match:
|
||||
expect_output.append((match.group(1), line_num))
|
||||
|
||||
match = EXPECT_ERROR_PATTERN.search(line)
|
||||
if match:
|
||||
expect_error.append(line_num)
|
||||
# If we expect compile errors, it should exit with EX_DATAERR.
|
||||
expect_return = 65
|
||||
|
||||
match = EXPECT_ERROR_LINE_PATTERN.search(line)
|
||||
if match:
|
||||
expect_error.append(int(match.group(1)))
|
||||
# If we expect compile errors, it should exit with EX_DATAERR.
|
||||
expect_return = 65
|
||||
|
||||
match = EXPECT_RUNTIME_ERROR_PATTERN.search(line)
|
||||
if match:
|
||||
expect_runtime_error_line = line_num
|
||||
expect_runtime_error = match.group(1)
|
||||
# If we expect a runtime error, it should exit with EX_SOFTWARE.
|
||||
expect_return = 70
|
||||
|
||||
match = SKIP_PATTERN.search(line)
|
||||
if match:
|
||||
num_skipped += 1
|
||||
skipped[match.group(1)] += 1
|
||||
return
|
||||
|
||||
match = NONTEST_PATTERN.search(line)
|
||||
if match:
|
||||
# Not a test file at all, so ignore it.
|
||||
return
|
||||
|
||||
line_num += 1
|
||||
|
||||
# Invoke wren and run the test.
|
||||
proc = Popen([WREN_APP, path], stdout=PIPE, stderr=PIPE)
|
||||
(out, err) = proc.communicate()
|
||||
(out, err) = out.replace('\r\n', '\n'), err.replace('\r\n', '\n')
|
||||
|
||||
fails = []
|
||||
|
||||
# Validate that no unexpected errors occurred.
|
||||
if expect_return != 0 and err != '':
|
||||
lines = err.split('\n')
|
||||
if expect_runtime_error:
|
||||
# Make sure we got the right error.
|
||||
if lines[0] != expect_runtime_error:
|
||||
fails.append('Expected runtime error "' + expect_runtime_error +
|
||||
'" and got:')
|
||||
fails.append(lines[0])
|
||||
|
||||
# Make sure the stack trace has the right line.
|
||||
match = STACK_TRACE_PATTERN.search(lines[1])
|
||||
if not match:
|
||||
fails.append('Expected stack trace and got:')
|
||||
fails.append(lines[1])
|
||||
else:
|
||||
stack_line = int(match.group(1))
|
||||
if stack_line != expect_runtime_error_line:
|
||||
fails.append('Expected runtime error on line ' +
|
||||
str(expect_runtime_error_line) + ' but was on line ' +
|
||||
str(stack_line))
|
||||
else:
|
||||
lines = err.split('\n')
|
||||
while len(lines) > 0:
|
||||
line = lines.pop(0)
|
||||
match = ERROR_PATTERN.search(line)
|
||||
if match:
|
||||
if float(match.group(1)) not in expect_error:
|
||||
fails.append('Unexpected error:')
|
||||
fails.append(line)
|
||||
elif line != '':
|
||||
fails.append('Unexpected output on stderr:')
|
||||
fails.append(line)
|
||||
else:
|
||||
for line in expect_error:
|
||||
fails.append('Expected error on line ' + str(line) + ' and got none.')
|
||||
if expect_runtime_error:
|
||||
fails.append('Expected runtime error "' + expect_runtime_error +
|
||||
'" and got none.')
|
||||
|
||||
# Validate the exit code.
|
||||
if proc.returncode != expect_return:
|
||||
fails.append('Expected return code {0} and got {1}.'
|
||||
.format(expect_return, proc.returncode))
|
||||
else:
|
||||
# Validate the output.
|
||||
expect_index = 0
|
||||
|
||||
# Remove the trailing last empty line.
|
||||
out_lines = out.split('\n')
|
||||
if out_lines[-1] == '':
|
||||
del out_lines[-1]
|
||||
|
||||
for line in out_lines:
|
||||
if expect_index >= len(expect_output):
|
||||
fails.append('Got output "{0}" when none was expected.'.format(line))
|
||||
elif expect_output[expect_index][0] != line:
|
||||
fails.append('Expected output "{0}" on line {1} and got "{2}".'.
|
||||
format(expect_output[expect_index][0],
|
||||
expect_output[expect_index][1], line))
|
||||
expect_index += 1
|
||||
|
||||
while expect_index < len(expect_output):
|
||||
fails.append('Missing expected output "{0}" on line {1}.'.
|
||||
format(expect_output[expect_index][0],
|
||||
expect_output[expect_index][1]))
|
||||
expect_index += 1
|
||||
|
||||
# Display the results.
|
||||
if len(fails) == 0:
|
||||
passed += 1
|
||||
#print color.GREEN + 'PASS' + color.DEFAULT + ': ' + path
|
||||
else:
|
||||
failed += 1
|
||||
print_line(color.RED + 'FAIL' + color.DEFAULT + ': ' + path)
|
||||
print
|
||||
for fail in fails:
|
||||
print ' ', color.PINK + fail + color.DEFAULT
|
||||
print
|
||||
|
||||
walk(TEST_DIR, run_test)
|
||||
|
||||
print_line()
|
||||
if failed == 0:
|
||||
print 'All ' + color.GREEN + str(passed) + color.DEFAULT + ' tests passed.'
|
||||
else:
|
||||
print (color.GREEN + str(passed) + color.DEFAULT + ' tests passed. ' +
|
||||
color.RED + str(failed) + color.DEFAULT + ' tests failed.')
|
||||
|
||||
for key in sorted(skipped.keys()):
|
||||
print ('Skipped ' + color.YELLOW + str(skipped[key]) + color.DEFAULT +
|
||||
' tests: ' + key)
|
||||
Reference in New Issue
Block a user