// retoor <retoor@molodetz.nl>
import "os" for Process, Platform
import "pathlib" for Path
import "regex" for Regex
import "argparse" for ArgumentParser
import "subprocess" for Popen
import "io" for Stdout
class Colors {
static green(text) { Platform.isWindows ? text : "\x1b[32m%(text)\x1b[0m" }
static red(text) { Platform.isWindows ? text : "\x1b[31m%(text)\x1b[0m" }
static pink(text) { Platform.isWindows ? text : "\x1b[91m%(text)\x1b[0m" }
static yellow(text) { Platform.isWindows ? text : "\x1b[33m%(text)\x1b[0m" }
}
class Patterns {
static expectPattern { __expectPattern }
static expectErrorLinePattern { __expectErrorLinePattern }
static expectErrorPattern { __expectErrorPattern }
static expectHandledRuntimeErrorPattern { __expectHandledRuntimeErrorPattern }
static expectRuntimeErrorPattern { __expectRuntimeErrorPattern }
static stdinPattern { __stdinPattern }
static skipPattern { __skipPattern }
static nontestPattern { __nontestPattern }
static splitPattern { __splitPattern }
static errorPattern { __errorPattern }
static stackTracePattern { __stackTracePattern }
static init() {
__expectPattern = Regex.new("// expect: ?(.*)")
__expectErrorLinePattern = Regex.new("// expect error line ([0-9]+)")
__expectErrorPattern = Regex.new("// expect error")
__expectHandledRuntimeErrorPattern = Regex.new("// expect handled runtime error: (.+)")
__expectRuntimeErrorPattern = Regex.new("// expect runtime error: (.+)")
__stdinPattern = Regex.new("// stdin: (.*)")
__skipPattern = Regex.new("// skip: (.*)")
__nontestPattern = Regex.new("// nontest")
__splitPattern = Regex.new("\n|\r\n")
__errorPattern = Regex.new("\\[.* line ([0-9]+)\\] Error")
__stackTracePattern = Regex.new("test/.* line ([0-9]+)\\] in")
}
}
class Test {
construct new(path) {
_path = path
_output = []
_compileErrors = []
_runtimeErrorLine = 0
_runtimeErrorMessage = null
_exitCode = 0
_inputBytes = null
_failures = []
}
path { _path }
failures { _failures }
parse() {
var inputLines = []
var lineNum = 1
var content = Path.new(_path).readText()
var lines = Patterns.splitPattern.split(content)
for (line in lines) {
if (line.count == 0) {
lineNum = lineNum + 1
continue
}
var match = Patterns.expectPattern.match(line)
if (match) {
_output.add([match.groups[1], lineNum])
TestRunner.expectations = TestRunner.expectations + 1
}
match = Patterns.expectErrorLinePattern.match(line)
if (match) {
var errLine = Num.fromString(match.groups[1])
if (!listContains_(_compileErrors, errLine)) {
_compileErrors.add(errLine)
}
_exitCode = 65
TestRunner.expectations = TestRunner.expectations + 1
} else {
match = Patterns.expectErrorPattern.match(line)
if (match) {
if (!listContains_(_compileErrors, lineNum)) {
_compileErrors.add(lineNum)
}
_exitCode = 65
TestRunner.expectations = TestRunner.expectations + 1
}
}
match = Patterns.expectHandledRuntimeErrorPattern.match(line)
if (match) {
_runtimeErrorLine = lineNum
_runtimeErrorMessage = match.groups[1]
TestRunner.expectations = TestRunner.expectations + 1
} else {
match = Patterns.expectRuntimeErrorPattern.match(line)
if (match) {
_runtimeErrorLine = lineNum
_runtimeErrorMessage = match.groups[1]
_exitCode = 70
TestRunner.expectations = TestRunner.expectations + 1
}
}
match = Patterns.stdinPattern.match(line)
if (match) {
inputLines.add(match.groups[1])
}
match = Patterns.skipPattern.match(line)
if (match) {
TestRunner.numSkipped = TestRunner.numSkipped + 1
var reason = match.groups[1]
if (!TestRunner.skipped.containsKey(reason)) {
TestRunner.skipped[reason] = 0
}
TestRunner.skipped[reason] = TestRunner.skipped[reason] + 1
return false
}
match = Patterns.nontestPattern.match(line)
if (match) {
return false
}
lineNum = lineNum + 1
}
if (inputLines.count > 0) {
_inputBytes = inputLines.join("\n")
}
return true
}
listContains_(list, item) {
for (i in list) {
if (i == item) return true
}
return false
}
run(app, type) {
var proc = Popen.new([app, _path])
if (_inputBytes) {
proc.stdin.write(_inputBytes)
}
proc.stdin.close()
var exitCode = proc.wait()
var out = proc.stdout.read()
var err = proc.stderr.read()
validate(type == "example", exitCode, out, err)
}
validate(isExample, exitCode, out, err) {
if (_compileErrors.count > 0 && _runtimeErrorMessage) {
fail("Test error: Cannot expect both compile and runtime errors.")
return
}
out = out.replace("\r\n", "\n")
err = err.replace("\r\n", "\n")
var errorLines = Patterns.splitPattern.split(err)
if (_runtimeErrorMessage) {
validateRuntimeError(errorLines)
} else {
validateCompileErrors(errorLines)
}
validateExitCode(exitCode, errorLines)
if (isExample) return
validateOutput(out)
}
validateRuntimeError(errorLines) {
if (errorLines.count < 2) {
fail("Expected runtime error \"" + _runtimeErrorMessage + "\" and got none.")
return
}
var line = 0
while (line < errorLines.count && Patterns.errorPattern.test(errorLines[line])) {
line = line + 1
}
if (line >= errorLines.count) {
fail("Expected runtime error \"" + _runtimeErrorMessage + "\" but only found compile errors.")
return
}
if (errorLines[line] != _runtimeErrorMessage) {
fail("Expected runtime error \"" + _runtimeErrorMessage + "\" and got:")
fail(errorLines[line])
}
var match = null
var stackLines = []
for (i in (line + 1)...errorLines.count) {
stackLines.add(errorLines[i])
match = Patterns.stackTracePattern.match(errorLines[i])
if (match) break
}
if (!match) {
fail("Expected stack trace and got:")
for (stackLine in stackLines) {
fail(stackLine)
}
} else {
var stackLine = Num.fromString(match.groups[1])
if (stackLine != _runtimeErrorLine) {
fail("Expected runtime error on line %(_runtimeErrorLine) but was on line %(stackLine).")
}
}
}
validateCompileErrors(errorLines) {
var foundErrors = []
for (line in errorLines) {
var match = Patterns.errorPattern.match(line)
if (match) {
var errorLine = Num.fromString(match.groups[1])
if (listContains_(_compileErrors, errorLine)) {
if (!listContains_(foundErrors, errorLine)) {
foundErrors.add(errorLine)
}
} else {
fail("Unexpected error:")
fail(line)
}
} else if (line != "") {
fail("Unexpected output on stderr:")
fail(line)
}
}
for (expected in _compileErrors) {
if (!listContains_(foundErrors, expected)) {
fail("Missing expected error on line %(expected).")
}
}
}
validateExitCode(exitCode, errorLines) {
if (exitCode == _exitCode) return
fail("Expected return code %(_exitCode) and got %(exitCode). Stderr:")
for (line in errorLines) {
_failures.add(line)
}
}
validateOutput(out) {
var outLines = Patterns.splitPattern.split(out)
if (outLines.count > 0 && outLines[-1] == "") {
outLines = outLines[0...-1]
}
var index = 0
for (line in outLines) {
if (index >= _output.count) {
fail("Got output \"%(line)\" when none was expected.")
} else if (_output[index][0] != line) {
fail("Expected output \"%(_output[index][0])\" on line %(_output[index][1]) and got \"%(line)\".")
}
index = index + 1
}
while (index < _output.count) {
fail("Missing expected output \"%(_output[index][0])\" on line %(_output[index][1]).")
index = index + 1
}
}
fail(message) {
_failures.add(message)
}
}
class TestRunner {
static passed { __passed }
static passed=(value) { __passed = value }
static failed { __failed }
static failed=(value) { __failed = value }
static numSkipped { __numSkipped }
static numSkipped=(value) { __numSkipped = value }
static skipped { __skipped }
static skipped=(value) { __skipped = value }
static expectations { __expectations }
static expectations=(value) { __expectations = value }
construct new() {
TestRunner.passed = 0
TestRunner.failed = 0
TestRunner.numSkipped = 0
TestRunner.skipped = {}
TestRunner.expectations = 0
var parser = ArgumentParser.new()
parser.addArgument("--suffix", {"default": ""})
parser.addArgument("suite", {"required": false, "default": null})
parser.addArgument("--skip-tests", {"action": "storeTrue"})
parser.addArgument("--skip-examples", {"action": "storeTrue"})
_args = parser.parseArgs()
_wrenDir = Path.new(Process.cwd)
_wrenApp = _wrenDir / "bin" / ("wren_cli" + _args["suffix"])
}
run() {
var testDir = _wrenDir / "test"
walk(testDir) {|path| runTest(path) }
printLine()
if (TestRunner.failed == 0) {
System.print("All " + Colors.green(TestRunner.passed) + " tests passed (" + TestRunner.expectations.toString + " expectations).")
} else {
System.print(Colors.green(TestRunner.passed) + " tests passed. " + Colors.red(TestRunner.failed) + " tests failed.")
}
var sortedKeys = sortKeys_(TestRunner.skipped)
for (key in sortedKeys) {
System.print("Skipped " + Colors.yellow(TestRunner.skipped[key]) + " tests: " + key)
}
if (TestRunner.failed != 0) {
Process.exit(1)
}
}
sortKeys_(map) {
var keys = []
for (key in map.keys) {
keys.add(key)
}
keys.sort()
return keys
}
walk(dir, callback) {
var entries = dir.iterdir()
var files = []
var dirs = []
for (entry in entries) {
if (entry.isDir()) {
dirs.add(entry)
} else {
files.add(entry)
}
}
dirs.sort {|a, b| a.toString < b.toString }
files.sort {|a, b| a.toString < b.toString }
for (file in files) {
callback.call(file)
}
for (subdir in dirs) {
walk(subdir, callback)
}
}
printLine() { printLine(null) }
printLine(line) {
System.write("\x1b[2K")
System.write("\r")
if (line) {
System.write(line)
Stdout.flush()
}
}
runTest(path) {
var pathStr = path.toString
if (!pathStr.endsWith(".wren")) return
if (_args["suite"]) {
var testPath = path.relativeTo(_wrenDir / "test").toString
if (!testPath.startsWith(_args["suite"])) return
}
var appRelPath = _wrenApp.relativeTo(_wrenDir).toString
printLine("(" + appRelPath + ") Passed: " + Colors.green(TestRunner.passed) + " Failed: " + Colors.red(TestRunner.failed) + " Skipped: " + Colors.yellow(TestRunner.numSkipped) + " ")
var normalizedPath = path.relativeTo(Path.cwd).toString.replace("\\", "/")
var test = Test.new(normalizedPath)
if (!test.parse()) {
return
}
test.run(_wrenApp.toString, "test")
if (test.failures.count == 0) {
TestRunner.passed = TestRunner.passed + 1
} else {
TestRunner.failed = TestRunner.failed + 1
printLine(Colors.red("FAIL") + ": " + normalizedPath)
System.print("")
for (failure in test.failures) {
System.print(" " + Colors.pink(failure))
}
System.print("")
}
}
}
Patterns.init()
var runner = TestRunner.new()
runner.run()
Reference in New Issue View Git Blame Copy Permalink