wren/util/test.wren

// retoor <retoor@molodetz.nl>

import "os" for Process, Platform
import "pathlib" for Path
import "regex" for Regex
import "argparse" for ArgumentParser
import "subprocess" for Popen
import "io" for Stdout

class Colors {
    static green(text) { Platform.isWindows ? text : "\x1b[32m%(text)\x1b[0m" }
    static red(text) { Platform.isWindows ? text : "\x1b[31m%(text)\x1b[0m" }
    static pink(text) { Platform.isWindows ? text : "\x1b[91m%(text)\x1b[0m" }
    static yellow(text) { Platform.isWindows ? text : "\x1b[33m%(text)\x1b[0m" }
}

class Patterns {
    static expectPattern { __expectPattern }
    static expectErrorLinePattern { __expectErrorLinePattern }
    static expectErrorPattern { __expectErrorPattern }
    static expectHandledRuntimeErrorPattern { __expectHandledRuntimeErrorPattern }
    static expectRuntimeErrorPattern { __expectRuntimeErrorPattern }
    static stdinPattern { __stdinPattern }
    static skipPattern { __skipPattern }
    static nontestPattern { __nontestPattern }
    static splitPattern { __splitPattern }
    static errorPattern { __errorPattern }
    static stackTracePattern { __stackTracePattern }

    static init() {
        __expectPattern = Regex.new("// expect: ?(.*)")
        __expectErrorLinePattern = Regex.new("// expect error line ([0-9]+)")
        __expectErrorPattern = Regex.new("// expect error")
        __expectHandledRuntimeErrorPattern = Regex.new("// expect handled runtime error: (.+)")
        __expectRuntimeErrorPattern = Regex.new("// expect runtime error: (.+)")
        __stdinPattern = Regex.new("// stdin: (.*)")
        __skipPattern = Regex.new("// skip: (.*)")
        __nontestPattern = Regex.new("// nontest")
        __splitPattern = Regex.new("\n|\r\n")
        __errorPattern = Regex.new("\\[.* line ([0-9]+)\\] Error")
        __stackTracePattern = Regex.new("test/.* line ([0-9]+)\\] in")
    }
}

class Test {
    construct new(path) {
        _path = path
        _output = []
        _compileErrors = []
        _runtimeErrorLine = 0
        _runtimeErrorMessage = null
        _exitCode = 0
        _inputBytes = null
        _failures = []
    }

    path { _path }
    failures { _failures }

    parse() {
        var inputLines = []
        var lineNum = 1

        var content = Path.new(_path).readText()
        var lines = Patterns.splitPattern.split(content)

        for (line in lines) {
            if (line.count == 0) {
                lineNum = lineNum + 1
                continue
            }

            var match = Patterns.expectPattern.match(line)
            if (match) {
                _output.add([match.groups[1], lineNum])
                TestRunner.expectations = TestRunner.expectations + 1
            }

            match = Patterns.expectErrorLinePattern.match(line)
            if (match) {
                var errLine = Num.fromString(match.groups[1])
                if (!listContains_(_compileErrors, errLine)) {
                    _compileErrors.add(errLine)
                }
                _exitCode = 65
                TestRunner.expectations = TestRunner.expectations + 1
            } else {
                match = Patterns.expectErrorPattern.match(line)
                if (match) {
                    if (!listContains_(_compileErrors, lineNum)) {
                        _compileErrors.add(lineNum)
                    }
                    _exitCode = 65
                    TestRunner.expectations = TestRunner.expectations + 1
                }
            }

            match = Patterns.expectHandledRuntimeErrorPattern.match(line)
            if (match) {
                _runtimeErrorLine = lineNum
                _runtimeErrorMessage = match.groups[1]
                TestRunner.expectations = TestRunner.expectations + 1
            } else {
                match = Patterns.expectRuntimeErrorPattern.match(line)
                if (match) {
                    _runtimeErrorLine = lineNum
                    _runtimeErrorMessage = match.groups[1]
                    _exitCode = 70
                    TestRunner.expectations = TestRunner.expectations + 1
                }
            }

            match = Patterns.stdinPattern.match(line)
            if (match) {
                inputLines.add(match.groups[1])
            }

            match = Patterns.skipPattern.match(line)
            if (match) {
                TestRunner.numSkipped = TestRunner.numSkipped + 1
                var reason = match.groups[1]
                if (!TestRunner.skipped.containsKey(reason)) {
                    TestRunner.skipped[reason] = 0
                }
                TestRunner.skipped[reason] = TestRunner.skipped[reason] + 1
                return false
            }

            match = Patterns.nontestPattern.match(line)
            if (match) {
                return false
            }

            lineNum = lineNum + 1
        }

        if (inputLines.count > 0) {
            _inputBytes = inputLines.join("\n")
        }

        return true
    }

    listContains_(list, item) {
        for (i in list) {
            if (i == item) return true
        }
        return false
    }

    run(app, type) {
        var proc = Popen.new([app, _path])

        if (_inputBytes) {
            proc.stdin.write(_inputBytes)
        }
        proc.stdin.close()

        var exitCode = proc.wait()
        var out = proc.stdout.read()
        var err = proc.stderr.read()

        validate(type == "example", exitCode, out, err)
    }

    validate(isExample, exitCode, out, err) {
        if (_compileErrors.count > 0 && _runtimeErrorMessage) {
            fail("Test error: Cannot expect both compile and runtime errors.")
            return
        }

        out = out.replace("\r\n", "\n")
        err = err.replace("\r\n", "\n")

        var errorLines = Patterns.splitPattern.split(err)

        if (_runtimeErrorMessage) {
            validateRuntimeError(errorLines)
        } else {
            validateCompileErrors(errorLines)
        }

        validateExitCode(exitCode, errorLines)

        if (isExample) return

        validateOutput(out)
    }

    validateRuntimeError(errorLines) {
        if (errorLines.count < 2) {
            fail("Expected runtime error \"" + _runtimeErrorMessage + "\" and got none.")
            return
        }

        var line = 0
        while (line < errorLines.count && Patterns.errorPattern.test(errorLines[line])) {
            line = line + 1
        }

        if (line >= errorLines.count) {
            fail("Expected runtime error \"" + _runtimeErrorMessage + "\" but only found compile errors.")
            return
        }

        if (errorLines[line] != _runtimeErrorMessage) {
            fail("Expected runtime error \"" + _runtimeErrorMessage + "\" and got:")
            fail(errorLines[line])
        }

        var match = null
        var stackLines = []
        for (i in (line + 1)...errorLines.count) {
            stackLines.add(errorLines[i])
            match = Patterns.stackTracePattern.match(errorLines[i])
            if (match) break
        }

        if (!match) {
            fail("Expected stack trace and got:")
            for (stackLine in stackLines) {
                fail(stackLine)
            }
        } else {
            var stackLine = Num.fromString(match.groups[1])
            if (stackLine != _runtimeErrorLine) {
                fail("Expected runtime error on line %(_runtimeErrorLine) but was on line %(stackLine).")
            }
        }
    }

    validateCompileErrors(errorLines) {
        var foundErrors = []

        for (line in errorLines) {
            var match = Patterns.errorPattern.match(line)
            if (match) {
                var errorLine = Num.fromString(match.groups[1])
                if (listContains_(_compileErrors, errorLine)) {
                    if (!listContains_(foundErrors, errorLine)) {
                        foundErrors.add(errorLine)
                    }
                } else {
                    fail("Unexpected error:")
                    fail(line)
                }
            } else if (line != "") {
                fail("Unexpected output on stderr:")
                fail(line)
            }
        }

        for (expected in _compileErrors) {
            if (!listContains_(foundErrors, expected)) {
                fail("Missing expected error on line %(expected).")
            }
        }
    }

    validateExitCode(exitCode, errorLines) {
        if (exitCode == _exitCode) return

        fail("Expected return code %(_exitCode) and got %(exitCode). Stderr:")
        for (line in errorLines) {
            _failures.add(line)
        }
    }

    validateOutput(out) {
        var outLines = Patterns.splitPattern.split(out)
        if (outLines.count > 0 && outLines[-1] == "") {
            outLines = outLines[0...-1]
        }

        var index = 0
        for (line in outLines) {
            if (index >= _output.count) {
                fail("Got output \"%(line)\" when none was expected.")
            } else if (_output[index][0] != line) {
                fail("Expected output \"%(_output[index][0])\" on line %(_output[index][1]) and got \"%(line)\".")
            }
            index = index + 1
        }

        while (index < _output.count) {
            fail("Missing expected output \"%(_output[index][0])\" on line %(_output[index][1]).")
            index = index + 1
        }
    }

    fail(message) {
        _failures.add(message)
    }
}

class TestRunner {
    static passed { __passed }
    static passed=(value) { __passed = value }
    static failed { __failed }
    static failed=(value) { __failed = value }
    static numSkipped { __numSkipped }
    static numSkipped=(value) { __numSkipped = value }
    static skipped { __skipped }
    static skipped=(value) { __skipped = value }
    static expectations { __expectations }
    static expectations=(value) { __expectations = value }

    construct new() {
        TestRunner.passed = 0
        TestRunner.failed = 0
        TestRunner.numSkipped = 0
        TestRunner.skipped = {}
        TestRunner.expectations = 0

        var parser = ArgumentParser.new()
        parser.addArgument("--suffix", {"default": ""})
        parser.addArgument("suite", {"required": false, "default": null})
        parser.addArgument("--skip-tests", {"action": "storeTrue"})
        parser.addArgument("--skip-examples", {"action": "storeTrue"})

        _args = parser.parseArgs()
        _wrenDir = Path.new(Process.cwd)
        _wrenApp = _wrenDir / "bin" / ("wren_cli" + _args["suffix"])
    }

    run() {
        var testDir = _wrenDir / "test"
        walk(testDir) {|path| runTest(path) }

        printLine()
        if (TestRunner.failed == 0) {
            System.print("All " + Colors.green(TestRunner.passed) + " tests passed (" + TestRunner.expectations.toString + " expectations).")
        } else {
            System.print(Colors.green(TestRunner.passed) + " tests passed. " + Colors.red(TestRunner.failed) + " tests failed.")
        }

        var sortedKeys = sortKeys_(TestRunner.skipped)
        for (key in sortedKeys) {
            System.print("Skipped " + Colors.yellow(TestRunner.skipped[key]) + " tests: " + key)
        }

        if (TestRunner.failed != 0) {
            Process.exit(1)
        }
    }

    sortKeys_(map) {
        var keys = []
        for (key in map.keys) {
            keys.add(key)
        }
        keys.sort()
        return keys
    }

    walk(dir, callback) {
        var entries = dir.iterdir()
        var files = []
        var dirs = []

        for (entry in entries) {
            if (entry.isDir()) {
                dirs.add(entry)
            } else {
                files.add(entry)
            }
        }

        dirs.sort {|a, b| a.toString < b.toString }
        files.sort {|a, b| a.toString < b.toString }

        for (file in files) {
            callback.call(file)
        }

        for (subdir in dirs) {
            walk(subdir, callback)
        }
    }

    printLine() { printLine(null) }

    printLine(line) {
        System.write("\x1b[2K")
        System.write("\r")
        if (line) {
            System.write(line)
            Stdout.flush()
        }
    }

    runTest(path) {
        var pathStr = path.toString

        if (!pathStr.endsWith(".wren")) return

        if (_args["suite"]) {
            var testPath = path.relativeTo(_wrenDir / "test").toString
            if (!testPath.startsWith(_args["suite"])) return
        }

        var appRelPath = _wrenApp.relativeTo(_wrenDir).toString
        printLine("(" + appRelPath + ") Passed: " + Colors.green(TestRunner.passed) + " Failed: " + Colors.red(TestRunner.failed) + " Skipped: " + Colors.yellow(TestRunner.numSkipped) + " ")

        var normalizedPath = path.relativeTo(Path.cwd).toString.replace("\\", "/")

        var test = Test.new(normalizedPath)

        if (!test.parse()) {
            return
        }

        test.run(_wrenApp.toString, "test")

        if (test.failures.count == 0) {
            TestRunner.passed = TestRunner.passed + 1
        } else {
            TestRunner.failed = TestRunner.failed + 1
            printLine(Colors.red("FAIL") + ": " + normalizedPath)
            System.print("")
            for (failure in test.failures) {
                System.print("      " + Colors.pink(failure))
            }
            System.print("")
        }
    }
}

Patterns.init()
var runner = TestRunner.new()
runner.run()