From 5d8ecad5a642e8ad7802aef875a922e3f3d46e97 Mon Sep 17 00:00:00 2001 From: Bob Nystrom Date: Wed, 16 Dec 2015 00:02:13 +0000 Subject: [PATCH] feat: add benchmark for Wren C API foreign method calls and refactor test class names Add a new benchmark that measures the performance of calling foreign methods through the Wren C embedding API, specifically testing argument passing and return values with a tight loop of 10 million calls. Refactor test API classes from generic "Api" to specific names like "Call", "Returns", "Value", and "ForeignClass" to avoid naming conflicts with the new benchmark module. Update the benchmark runner to use the test_api executable for API-specific benchmarks and remove the special API test argument handling from the test runner. --- Makefile | 4 +++ test/api/benchmark.c | 21 +++++++++++ test/api/benchmark.h | 3 ++ test/api/call.c | 10 +++--- test/api/call.wren | 2 +- test/api/foreign_class.c | 2 +- test/api/foreign_class.wren | 8 ++--- test/api/main.c | 43 ++++++++++------------- test/api/returns.c | 14 ++++---- test/api/returns.wren | 16 ++++----- test/api/value.c | 4 +-- test/api/value.wren | 6 ++-- test/benchmark/api_foreign_method.wren | 20 +++++++++++ util/benchmark.py | 15 +++++++- util/test.py | 4 --- util/xcode/wren.xcodeproj/project.pbxproj | 6 ++++ 16 files changed, 118 insertions(+), 60 deletions(-) create mode 100644 test/api/benchmark.c create mode 100644 test/api/benchmark.h create mode 100644 test/benchmark/api_foreign_method.wren diff --git a/Makefile b/Makefile index 5f5633fe..c402486a 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,10 @@ test: debug @ $(MAKE) -f util/wren.mk MODE=debug test @ ./util/test.py $(suite) +benchmark: release + @ $(MAKE) -f util/wren.mk test + @ ./util/benchmark.py $(suite) + # Generate the Wren site. docs: @ ./util/generate_docs.py diff --git a/test/api/benchmark.c b/test/api/benchmark.c new file mode 100644 index 00000000..5c7f4ccb --- /dev/null +++ b/test/api/benchmark.c @@ -0,0 +1,21 @@ +#include + +#include "benchmark.h" + +static void arguments(WrenVM* vm) +{ + double result = 0; + result += wrenGetArgumentDouble(vm, 1); + result += wrenGetArgumentDouble(vm, 2); + result += wrenGetArgumentDouble(vm, 3); + result += wrenGetArgumentDouble(vm, 4); + + wrenReturnDouble(vm, result); +} + +WrenForeignMethodFn benchmarkBindMethod(const char* signature) +{ + if (strcmp(signature, "static Benchmark.arguments(_,_,_,_)") == 0) return arguments; + + return NULL; +} diff --git a/test/api/benchmark.h b/test/api/benchmark.h new file mode 100644 index 00000000..de2bf4c1 --- /dev/null +++ b/test/api/benchmark.h @@ -0,0 +1,3 @@ +#include "wren.h" + +WrenForeignMethodFn benchmarkBindMethod(const char* signature); diff --git a/test/api/call.c b/test/api/call.c index 40ccf81f..5207a456 100644 --- a/test/api/call.c +++ b/test/api/call.c @@ -5,10 +5,10 @@ void callRunTests(WrenVM* vm) { - WrenValue* noParams = wrenGetMethod(vm, "main", "Api", "noParams"); - WrenValue* zero = wrenGetMethod(vm, "main", "Api", "zero()"); - WrenValue* one = wrenGetMethod(vm, "main", "Api", "one(_)"); - WrenValue* two = wrenGetMethod(vm, "main", "Api", "two(_,_)"); + WrenValue* noParams = wrenGetMethod(vm, "main", "Call", "noParams"); + WrenValue* zero = wrenGetMethod(vm, "main", "Call", "zero()"); + WrenValue* one = wrenGetMethod(vm, "main", "Call", "one(_)"); + WrenValue* two = wrenGetMethod(vm, "main", "Call", "two(_,_)"); // Different arity. wrenCall(vm, noParams, NULL, ""); @@ -16,7 +16,7 @@ void callRunTests(WrenVM* vm) wrenCall(vm, one, NULL, "i", 1); wrenCall(vm, two, NULL, "ii", 1, 2); - WrenValue* getValue = wrenGetMethod(vm, "main", "Api", "getValue(_)"); + WrenValue* getValue = wrenGetMethod(vm, "main", "Call", "getValue(_)"); // Returning a value. WrenValue* value = NULL; diff --git a/test/api/call.wren b/test/api/call.wren index a174ae2e..35feaa5e 100644 --- a/test/api/call.wren +++ b/test/api/call.wren @@ -1,4 +1,4 @@ -class Api { +class Call { static noParams { System.print("noParams") } diff --git a/test/api/foreign_class.c b/test/api/foreign_class.c index afe7f00e..cf9fbfeb 100644 --- a/test/api/foreign_class.c +++ b/test/api/foreign_class.c @@ -79,7 +79,7 @@ static void resourceFinalize(WrenVM* vm) WrenForeignMethodFn foreignClassBindMethod(const char* signature) { - if (strcmp(signature, "static Api.finalized") == 0) return apiFinalized; + if (strcmp(signature, "static ForeignClass.finalized") == 0) return apiFinalized; if (strcmp(signature, "Counter.increment(_)") == 0) return counterIncrement; if (strcmp(signature, "Counter.value") == 0) return counterValue; if (strcmp(signature, "Point.translate(_,_,_)") == 0) return pointTranslate; diff --git a/test/api/foreign_class.wren b/test/api/foreign_class.wren index 3f878271..c0b03535 100644 --- a/test/api/foreign_class.wren +++ b/test/api/foreign_class.wren @@ -1,4 +1,4 @@ -class Api { +class ForeignClass { foreign static finalized } @@ -64,14 +64,14 @@ var resources = [ ] System.gc() -System.print(Api.finalized) // expect: 0 +System.print(ForeignClass.finalized) // expect: 0 resources.removeAt(-1) System.gc() -System.print(Api.finalized) // expect: 1 +System.print(ForeignClass.finalized) // expect: 1 resources.clear() System.gc() -System.print(Api.finalized) // expect: 3 +System.print(ForeignClass.finalized) // expect: 3 diff --git a/test/api/main.c b/test/api/main.c index 7239b7ef..7674db05 100644 --- a/test/api/main.c +++ b/test/api/main.c @@ -4,27 +4,19 @@ #include "vm.h" #include "wren.h" +#include "benchmark.h" #include "call.h" #include "foreign_class.h" #include "returns.h" #include "value.h" -#define REGISTER_METHOD(name, camelCase) \ - if (strcmp(testName, #name) == 0) return camelCase##BindMethod(fullName) - -#define REGISTER_CLASS(name, camelCase) \ - if (strcmp(testName, #name) == 0) \ - { \ - camelCase##BindClass(className, &methods); \ - } - // The name of the currently executing API test. const char* testName; static WrenForeignMethodFn bindForeignMethod( WrenVM* vm, const char* module, const char* className, bool isStatic, const char* signature) -{ +{ if (strcmp(module, "main") != 0) return NULL; // For convenience, concatenate all of the method qualifiers into a single @@ -35,10 +27,20 @@ static WrenForeignMethodFn bindForeignMethod( strcat(fullName, className); strcat(fullName, "."); strcat(fullName, signature); - - REGISTER_METHOD(foreign_class, foreignClass); - REGISTER_METHOD(returns, returns); - REGISTER_METHOD(value, value); + + WrenForeignMethodFn method = NULL; + + method = benchmarkBindMethod(fullName); + if (method != NULL) return method; + + method = foreignClassBindMethod(fullName); + if (method != NULL) return method; + + method = returnsBindMethod(fullName); + if (method != NULL) return method; + + method = valueBindMethod(fullName); + if (method != NULL) return method; fprintf(stderr, "Unknown foreign method '%s' for test '%s'\n", fullName, testName); @@ -52,13 +54,13 @@ static WrenForeignClassMethods bindForeignClass( WrenForeignClassMethods methods = { NULL, NULL }; if (strcmp(module, "main") != 0) return methods; - REGISTER_CLASS(foreign_class, foreignClass); + foreignClassBindClass(className, &methods); return methods; } static void afterLoad(WrenVM* vm) { - if (strcmp(testName, "call") == 0) callRunTests(vm); + if (strstr(testName, "call.wren") != NULL) callRunTests(vm); } int main(int argc, const char* argv[]) @@ -70,14 +72,7 @@ int main(int argc, const char* argv[]) } testName = argv[1]; - - // The test script is at "test/api/.wren". - char testPath[256]; - strcpy(testPath, "test/api/"); - strcat(testPath, testName); - strcat(testPath, ".wren"); - setTestCallbacks(bindForeignMethod, bindForeignClass, afterLoad); - runFile(testPath); + runFile(testName); return 0; } diff --git a/test/api/returns.c b/test/api/returns.c index e1ba9155..f622ded9 100644 --- a/test/api/returns.c +++ b/test/api/returns.c @@ -39,13 +39,13 @@ static void returnBytes(WrenVM* vm) WrenForeignMethodFn returnsBindMethod(const char* signature) { - if (strcmp(signature, "static Api.implicitNull") == 0) return implicitNull; - if (strcmp(signature, "static Api.returnInt") == 0) return returnInt; - if (strcmp(signature, "static Api.returnFloat") == 0) return returnFloat; - if (strcmp(signature, "static Api.returnTrue") == 0) return returnTrue; - if (strcmp(signature, "static Api.returnFalse") == 0) return returnFalse; - if (strcmp(signature, "static Api.returnString") == 0) return returnString; - if (strcmp(signature, "static Api.returnBytes") == 0) return returnBytes; + if (strcmp(signature, "static Returns.implicitNull") == 0) return implicitNull; + if (strcmp(signature, "static Returns.returnInt") == 0) return returnInt; + if (strcmp(signature, "static Returns.returnFloat") == 0) return returnFloat; + if (strcmp(signature, "static Returns.returnTrue") == 0) return returnTrue; + if (strcmp(signature, "static Returns.returnFalse") == 0) return returnFalse; + if (strcmp(signature, "static Returns.returnString") == 0) return returnString; + if (strcmp(signature, "static Returns.returnBytes") == 0) return returnBytes; return NULL; } diff --git a/test/api/returns.wren b/test/api/returns.wren index b50edfcc..16a07e5e 100644 --- a/test/api/returns.wren +++ b/test/api/returns.wren @@ -1,4 +1,4 @@ -class Api { +class Returns { foreign static implicitNull foreign static returnInt @@ -11,13 +11,13 @@ class Api { foreign static returnBytes } -System.print(Api.implicitNull == null) // expect: true +System.print(Returns.implicitNull == null) // expect: true -System.print(Api.returnInt) // expect: 123456 -System.print(Api.returnFloat) // expect: 123.456 +System.print(Returns.returnInt) // expect: 123456 +System.print(Returns.returnFloat) // expect: 123.456 -System.print(Api.returnTrue) // expect: true -System.print(Api.returnFalse) // expect: false +System.print(Returns.returnTrue) // expect: true +System.print(Returns.returnFalse) // expect: false -System.print(Api.returnString) // expect: a string -System.print(Api.returnBytes.bytes.toList) // expect: [97, 0, 98, 0, 99] +System.print(Returns.returnString) // expect: a string +System.print(Returns.returnBytes.bytes.toList) // expect: [97, 0, 98, 0, 99] diff --git a/test/api/value.c b/test/api/value.c index 53022054..5b7b6225 100644 --- a/test/api/value.c +++ b/test/api/value.c @@ -17,8 +17,8 @@ static void getValue(WrenVM* vm) WrenForeignMethodFn valueBindMethod(const char* signature) { - if (strcmp(signature, "static Api.value=(_)") == 0) return setValue; - if (strcmp(signature, "static Api.value") == 0) return getValue; + if (strcmp(signature, "static Value.value=(_)") == 0) return setValue; + if (strcmp(signature, "static Value.value") == 0) return getValue; return NULL; } diff --git a/test/api/value.wren b/test/api/value.wren index c348869f..65e2194a 100644 --- a/test/api/value.wren +++ b/test/api/value.wren @@ -1,9 +1,9 @@ -class Api { +class Value { foreign static value=(value) foreign static value } -Api.value = ["list", "of", "strings"] +Value.value = ["list", "of", "strings"] // Do some stuff to trigger a GC (at least when GC stress testing enabled). var s = "string" @@ -11,4 +11,4 @@ for (i in 1...10) { s = s + " more" } -System.print(Api.value) // expect: [list, of, strings] +System.print(Value.value) // expect: [list, of, strings] diff --git a/test/benchmark/api_foreign_method.wren b/test/benchmark/api_foreign_method.wren new file mode 100644 index 00000000..35a7d17a --- /dev/null +++ b/test/benchmark/api_foreign_method.wren @@ -0,0 +1,20 @@ +class Benchmark { + foreign static arguments(a, b, c, d) +} + +var start = System.clock +var result = 0 +for (i in 1..1000000) { + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) + result = result + Benchmark.arguments(1, 2, 3, 4) +} +System.print(result) +System.print("elapsed: %(System.clock - start)") diff --git a/util/benchmark.py b/util/benchmark.py index 9e165952..36e7952d 100755 --- a/util/benchmark.py +++ b/util/benchmark.py @@ -51,6 +51,8 @@ def BENCHMARK(name, pattern): regex = re.compile(pattern + "\n" + r"elapsed: (\d+\.\d+)", re.MULTILINE) BENCHMARKS.append([name, regex, None]) +BENCHMARK("api_foreign_method", "100000000") + BENCHMARK("binary_trees", """stretch tree of depth 13 check: -1 8192 trees of depth 4 check: -8192 2048 trees of depth 6 check: -2048 @@ -143,9 +145,20 @@ def standard_deviation(times): def run_trial(benchmark, language): """Runs one benchmark one time for one language.""" + executable_args = language[1] + + # Hackish. If the benchmark name starts with "api_", it's testing the Wren + # C API, so run the test_api executable which has those test methods instead + # of the normal Wren build. + if benchmark[0].startswith("api_"): + executable_args = [ + os.path.join(WREN_DIR, "build", "release", "test", "wren") + ] + args = [] - args.extend(language[1]) + args.extend(executable_args) args.append(os.path.join(BENCHMARK_DIR, benchmark[0] + language[2])) + try: out = subprocess.check_output(args, universal_newlines=True) except OSError: diff --git a/util/test.py b/util/test.py index fdf0eac2..8e5bc0fb 100755 --- a/util/test.py +++ b/util/test.py @@ -111,10 +111,6 @@ class Test: def run(self, app, type): # Invoke wren and run the test. test_arg = self.path - if type == "api test": - # Just pass the suite name to API tests. - test_arg = basename(splitext(test_arg)[0]) - proc = Popen([app, test_arg], stdin=PIPE, stdout=PIPE, stderr=PIPE) # If a test takes longer than two seconds, kill it. diff --git a/util/xcode/wren.xcodeproj/project.pbxproj b/util/xcode/wren.xcodeproj/project.pbxproj index e30605b3..fe7b67a4 100644 --- a/util/xcode/wren.xcodeproj/project.pbxproj +++ b/util/xcode/wren.xcodeproj/project.pbxproj @@ -25,6 +25,7 @@ 29729F321BA70A620099CA20 /* io.c in Sources */ = {isa = PBXBuildFile; fileRef = 29729F2E1BA70A620099CA20 /* io.c */; }; 29729F331BA70A620099CA20 /* io.wren.inc in Sources */ = {isa = PBXBuildFile; fileRef = 29729F301BA70A620099CA20 /* io.wren.inc */; }; 2986F6D71ACF93BA00BCE26C /* wren_primitive.c in Sources */ = {isa = PBXBuildFile; fileRef = 2986F6D51ACF93BA00BCE26C /* wren_primitive.c */; }; + 29932D511C20D8C900099DEE /* benchmark.c in Sources */ = {isa = PBXBuildFile; fileRef = 29932D4F1C20D8C900099DEE /* benchmark.c */; }; 29A427341BDBE435001E6E22 /* wren_opt_meta.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */; }; 29A427351BDBE435001E6E22 /* wren_opt_meta.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */; }; 29A427361BDBE435001E6E22 /* wren_opt_meta.wren.inc in Sources */ = {isa = PBXBuildFile; fileRef = 29A427301BDBE435001E6E22 /* wren_opt_meta.wren.inc */; }; @@ -103,6 +104,8 @@ 29729F301BA70A620099CA20 /* io.wren.inc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.pascal; name = io.wren.inc; path = ../../src/module/io.wren.inc; sourceTree = ""; }; 2986F6D51ACF93BA00BCE26C /* wren_primitive.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = wren_primitive.c; path = ../../src/vm/wren_primitive.c; sourceTree = ""; }; 2986F6D61ACF93BA00BCE26C /* wren_primitive.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = wren_primitive.h; path = ../../src/vm/wren_primitive.h; sourceTree = ""; }; + 29932D4F1C20D8C900099DEE /* benchmark.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = benchmark.c; path = ../../test/api/benchmark.c; sourceTree = ""; }; + 29932D501C20D8C900099DEE /* benchmark.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = benchmark.h; path = ../../test/api/benchmark.h; sourceTree = ""; }; 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = wren_opt_meta.c; path = ../../src/optional/wren_opt_meta.c; sourceTree = ""; }; 29A4272F1BDBE435001E6E22 /* wren_opt_meta.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = wren_opt_meta.h; path = ../../src/optional/wren_opt_meta.h; sourceTree = ""; }; 29A427301BDBE435001E6E22 /* wren_opt_meta.wren.inc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.pascal; name = wren_opt_meta.wren.inc; path = ../../src/optional/wren_opt_meta.wren.inc; sourceTree = ""; }; @@ -239,6 +242,8 @@ 29D0099A1B7E394F000CE58C /* api_test */ = { isa = PBXGroup; children = ( + 29932D4F1C20D8C900099DEE /* benchmark.c */, + 29932D501C20D8C900099DEE /* benchmark.h */, 29D009A61B7E3993000CE58C /* main.c */, 293D46941BB43F9900200083 /* call.c */, 293D46951BB43F9900200083 /* call.h */, @@ -356,6 +361,7 @@ 291647C81BA5EC5E006142EE /* modules.c in Sources */, 29DC14A11BBA2FEC008A8274 /* scheduler.c in Sources */, 29A427391BDBE435001E6E22 /* wren_opt_random.c in Sources */, + 29932D511C20D8C900099DEE /* benchmark.c in Sources */, 29DC14A01BBA2FD6008A8274 /* timer.c in Sources */, 29DC149F1BBA2FCC008A8274 /* vm.c in Sources */, 29DC14A21BBA300A008A8274 /* wren_compiler.c in Sources */,