feat: add benchmark for Wren C API foreign method calls and refactor test class names

Browse Source 
Add a new benchmark that measures the performance of calling foreign methods through the Wren C embedding API, specifically testing argument passing and return values with a tight loop of 10 million calls. Refactor test API classes from generic "Api" to specific names like "Call", "Returns", "Value", and "ForeignClass" to avoid naming conflicts with the new benchmark module. Update the benchmark runner to use the test_api executable for API-specific benchmarks and remove the special API test argument handling from the test runner.
This commit is contained in:
Bob Nystrom 2015-12-16 00:02:13 +00:00
parent ba0d308fba
commit 5d8ecad5a6
16 changed files with 118 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -46,6 +46,10 @@ test: debug
@ $(MAKE) -f util/wren.mk MODE=debug test @ $(MAKE) -f util/wren.mk MODE=debug test
@ ./util/test.py $(suite) @ ./util/test.py $(suite)
benchmark: release
@ $(MAKE) -f util/wren.mk test
@ ./util/benchmark.py $(suite)
# Generate the Wren site. # Generate the Wren site.
docs: docs:
@ ./util/generate_docs.py @ ./util/generate_docs.py

21
test/api/benchmark.c Normal file
View File

@ -0,0 +1,21 @@
#include <string.h>
#include "benchmark.h"
static void arguments(WrenVM* vm)
{
double result = 0;
result += wrenGetArgumentDouble(vm, 1);
result += wrenGetArgumentDouble(vm, 2);
result += wrenGetArgumentDouble(vm, 3);
result += wrenGetArgumentDouble(vm, 4);
wrenReturnDouble(vm, result);
}
WrenForeignMethodFn benchmarkBindMethod(const char* signature)
{
if (strcmp(signature, "static Benchmark.arguments(_,_,_,_)") == 0) return arguments;
return NULL;
}

3
test/api/benchmark.h Normal file
View File

@ -0,0 +1,3 @@
#include "wren.h"
WrenForeignMethodFn benchmarkBindMethod(const char* signature);

10
test/api/call.c
View File

@ -5,10 +5,10 @@
void callRunTests(WrenVM* vm) void callRunTests(WrenVM* vm)
{ {
WrenValue* noParams = wrenGetMethod(vm, "main", "Api", "noParams"); WrenValue* noParams = wrenGetMethod(vm, "main", "Call", "noParams");
WrenValue* zero = wrenGetMethod(vm, "main", "Api", "zero()"); WrenValue* zero = wrenGetMethod(vm, "main", "Call", "zero()");
WrenValue* one = wrenGetMethod(vm, "main", "Api", "one(_)"); WrenValue* one = wrenGetMethod(vm, "main", "Call", "one(_)");
WrenValue* two = wrenGetMethod(vm, "main", "Api", "two(_,_)"); WrenValue* two = wrenGetMethod(vm, "main", "Call", "two(_,_)");
// Different arity. // Different arity.
wrenCall(vm, noParams, NULL, ""); wrenCall(vm, noParams, NULL, "");
@ -16,7 +16,7 @@ void callRunTests(WrenVM* vm)
wrenCall(vm, one, NULL, "i", 1); wrenCall(vm, one, NULL, "i", 1);
wrenCall(vm, two, NULL, "ii", 1, 2); wrenCall(vm, two, NULL, "ii", 1, 2);
WrenValue* getValue = wrenGetMethod(vm, "main", "Api", "getValue(_)"); WrenValue* getValue = wrenGetMethod(vm, "main", "Call", "getValue(_)");
// Returning a value. // Returning a value.
WrenValue* value = NULL; WrenValue* value = NULL;

2
test/api/call.wren
View File

@ -1,4 +1,4 @@
class Api { class Call {
static noParams { static noParams {
System.print("noParams") System.print("noParams")
} }

2
test/api/foreign_class.c
View File

@ -79,7 +79,7 @@ static void resourceFinalize(WrenVM* vm)
WrenForeignMethodFn foreignClassBindMethod(const char* signature) WrenForeignMethodFn foreignClassBindMethod(const char* signature)
{ {
if (strcmp(signature, "static Api.finalized") == 0) return apiFinalized; if (strcmp(signature, "static ForeignClass.finalized") == 0) return apiFinalized;
if (strcmp(signature, "Counter.increment(_)") == 0) return counterIncrement; if (strcmp(signature, "Counter.increment(_)") == 0) return counterIncrement;
if (strcmp(signature, "Counter.value") == 0) return counterValue; if (strcmp(signature, "Counter.value") == 0) return counterValue;
if (strcmp(signature, "Point.translate(_,_,_)") == 0) return pointTranslate; if (strcmp(signature, "Point.translate(_,_,_)") == 0) return pointTranslate;

8
test/api/foreign_class.wren
View File

@ -1,4 +1,4 @@
class Api { class ForeignClass {
foreign static finalized foreign static finalized
} }
@ -64,14 +64,14 @@ var resources = [
] ]
System.gc() System.gc()
System.print(Api.finalized) // expect: 0 System.print(ForeignClass.finalized) // expect: 0
resources.removeAt(-1) resources.removeAt(-1)
System.gc() System.gc()
System.print(Api.finalized) // expect: 1 System.print(ForeignClass.finalized) // expect: 1
resources.clear() resources.clear()
System.gc() System.gc()
System.print(Api.finalized) // expect: 3 System.print(ForeignClass.finalized) // expect: 3

39
test/api/main.c
View File

@ -4,20 +4,12 @@
#include "vm.h" #include "vm.h"
#include "wren.h" #include "wren.h"
#include "benchmark.h"
#include "call.h" #include "call.h"
#include "foreign_class.h" #include "foreign_class.h"
#include "returns.h" #include "returns.h"
#include "value.h" #include "value.h"
#define REGISTER_METHOD(name, camelCase) \
if (strcmp(testName, #name) == 0) return camelCase##BindMethod(fullName)
#define REGISTER_CLASS(name, camelCase) \
if (strcmp(testName, #name) == 0) \
{ \
camelCase##BindClass(className, &methods); \
}
// The name of the currently executing API test. // The name of the currently executing API test.
const char* testName; const char* testName;
@ -36,9 +28,19 @@ static WrenForeignMethodFn bindForeignMethod(
strcat(fullName, "."); strcat(fullName, ".");
strcat(fullName, signature); strcat(fullName, signature);
REGISTER_METHOD(foreign_class, foreignClass); WrenForeignMethodFn method = NULL;
REGISTER_METHOD(returns, returns);
REGISTER_METHOD(value, value); method = benchmarkBindMethod(fullName);
if (method != NULL) return method;
method = foreignClassBindMethod(fullName);
if (method != NULL) return method;
method = returnsBindMethod(fullName);
if (method != NULL) return method;
method = valueBindMethod(fullName);
if (method != NULL) return method;
fprintf(stderr, fprintf(stderr,
"Unknown foreign method '%s' for test '%s'\n", fullName, testName); "Unknown foreign method '%s' for test '%s'\n", fullName, testName);
@ -52,13 +54,13 @@ static WrenForeignClassMethods bindForeignClass(
WrenForeignClassMethods methods = { NULL, NULL }; WrenForeignClassMethods methods = { NULL, NULL };
if (strcmp(module, "main") != 0) return methods; if (strcmp(module, "main") != 0) return methods;
REGISTER_CLASS(foreign_class, foreignClass); foreignClassBindClass(className, &methods);
return methods; return methods;
} }
static void afterLoad(WrenVM* vm) { static void afterLoad(WrenVM* vm) {
if (strcmp(testName, "call") == 0) callRunTests(vm); if (strstr(testName, "call.wren") != NULL) callRunTests(vm);
} }
int main(int argc, const char* argv[]) int main(int argc, const char* argv[])
@ -70,14 +72,7 @@ int main(int argc, const char* argv[])
} }
testName = argv[1]; testName = argv[1];
// The test script is at "test/api/<test>.wren".
char testPath[256];
strcpy(testPath, "test/api/");
strcat(testPath, testName);
strcat(testPath, ".wren");
setTestCallbacks(bindForeignMethod, bindForeignClass, afterLoad); setTestCallbacks(bindForeignMethod, bindForeignClass, afterLoad);
runFile(testPath); runFile(testName);
return 0; return 0;
} }

14
test/api/returns.c
View File

@ -39,13 +39,13 @@ static void returnBytes(WrenVM* vm)
WrenForeignMethodFn returnsBindMethod(const char* signature) WrenForeignMethodFn returnsBindMethod(const char* signature)
{ {
if (strcmp(signature, "static Api.implicitNull") == 0) return implicitNull; if (strcmp(signature, "static Returns.implicitNull") == 0) return implicitNull;
if (strcmp(signature, "static Api.returnInt") == 0) return returnInt; if (strcmp(signature, "static Returns.returnInt") == 0) return returnInt;
if (strcmp(signature, "static Api.returnFloat") == 0) return returnFloat; if (strcmp(signature, "static Returns.returnFloat") == 0) return returnFloat;
if (strcmp(signature, "static Api.returnTrue") == 0) return returnTrue; if (strcmp(signature, "static Returns.returnTrue") == 0) return returnTrue;
if (strcmp(signature, "static Api.returnFalse") == 0) return returnFalse; if (strcmp(signature, "static Returns.returnFalse") == 0) return returnFalse;
if (strcmp(signature, "static Api.returnString") == 0) return returnString; if (strcmp(signature, "static Returns.returnString") == 0) return returnString;
if (strcmp(signature, "static Api.returnBytes") == 0) return returnBytes; if (strcmp(signature, "static Returns.returnBytes") == 0) return returnBytes;
return NULL; return NULL;
} }

16
test/api/returns.wren
View File

@ -1,4 +1,4 @@
class Api { class Returns {
foreign static implicitNull foreign static implicitNull
foreign static returnInt foreign static returnInt
@ -11,13 +11,13 @@ class Api {
foreign static returnBytes foreign static returnBytes
} }
System.print(Api.implicitNull == null) // expect: true System.print(Returns.implicitNull == null) // expect: true
System.print(Api.returnInt) // expect: 123456 System.print(Returns.returnInt) // expect: 123456
System.print(Api.returnFloat) // expect: 123.456 System.print(Returns.returnFloat) // expect: 123.456
System.print(Api.returnTrue) // expect: true System.print(Returns.returnTrue) // expect: true
System.print(Api.returnFalse) // expect: false System.print(Returns.returnFalse) // expect: false
System.print(Api.returnString) // expect: a string System.print(Returns.returnString) // expect: a string
System.print(Api.returnBytes.bytes.toList) // expect: [97, 0, 98, 0, 99] System.print(Returns.returnBytes.bytes.toList) // expect: [97, 0, 98, 0, 99]

4
test/api/value.c
View File

@ -17,8 +17,8 @@ static void getValue(WrenVM* vm)
WrenForeignMethodFn valueBindMethod(const char* signature) WrenForeignMethodFn valueBindMethod(const char* signature)
{ {
if (strcmp(signature, "static Api.value=(_)") == 0) return setValue; if (strcmp(signature, "static Value.value=(_)") == 0) return setValue;
if (strcmp(signature, "static Api.value") == 0) return getValue; if (strcmp(signature, "static Value.value") == 0) return getValue;
return NULL; return NULL;
} }

6
test/api/value.wren
View File

@ -1,9 +1,9 @@
class Api { class Value {
foreign static value=(value) foreign static value=(value)
foreign static value foreign static value
} }
Api.value = ["list", "of", "strings"] Value.value = ["list", "of", "strings"]
// Do some stuff to trigger a GC (at least when GC stress testing enabled). // Do some stuff to trigger a GC (at least when GC stress testing enabled).
var s = "string" var s = "string"
@ -11,4 +11,4 @@ for (i in 1...10) {
s = s + " more" s = s + " more"
} }
System.print(Api.value) // expect: [list, of, strings] System.print(Value.value) // expect: [list, of, strings]

20
test/benchmark/api_foreign_method.wren Normal file
View File

@ -0,0 +1,20 @@
class Benchmark {
foreign static arguments(a, b, c, d)
}
var start = System.clock
var result = 0
for (i in 1..1000000) {
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
}
System.print(result)
System.print("elapsed: %(System.clock - start)")

15
util/benchmark.py
View File

@ -51,6 +51,8 @@ def BENCHMARK(name, pattern):
regex = re.compile(pattern + "\n" + r"elapsed: (\d+\.\d+)", re.MULTILINE) regex = re.compile(pattern + "\n" + r"elapsed: (\d+\.\d+)", re.MULTILINE)
BENCHMARKS.append([name, regex, None]) BENCHMARKS.append([name, regex, None])
BENCHMARK("api_foreign_method", "100000000")
BENCHMARK("binary_trees", """stretch tree of depth 13 check: -1 BENCHMARK("binary_trees", """stretch tree of depth 13 check: -1
8192 trees of depth 4 check: -8192 8192 trees of depth 4 check: -8192
2048 trees of depth 6 check: -2048 2048 trees of depth 6 check: -2048
@ -143,9 +145,20 @@ def standard_deviation(times):
def run_trial(benchmark, language): def run_trial(benchmark, language):
"""Runs one benchmark one time for one language.""" """Runs one benchmark one time for one language."""
executable_args = language[1]
# Hackish. If the benchmark name starts with "api_", it's testing the Wren
# C API, so run the test_api executable which has those test methods instead
# of the normal Wren build.
if benchmark[0].startswith("api_"):
executable_args = [
os.path.join(WREN_DIR, "build", "release", "test", "wren")
]
args = [] args = []
args.extend(language[1]) args.extend(executable_args)
args.append(os.path.join(BENCHMARK_DIR, benchmark[0] + language[2])) args.append(os.path.join(BENCHMARK_DIR, benchmark[0] + language[2]))
try: try:
out = subprocess.check_output(args, universal_newlines=True) out = subprocess.check_output(args, universal_newlines=True)
except OSError: except OSError:

4
util/test.py
View File

@ -111,10 +111,6 @@ class Test:
def run(self, app, type): def run(self, app, type):
# Invoke wren and run the test. # Invoke wren and run the test.
test_arg = self.path test_arg = self.path
if type == "api test":
# Just pass the suite name to API tests.
test_arg = basename(splitext(test_arg)[0])
proc = Popen([app, test_arg], stdin=PIPE, stdout=PIPE, stderr=PIPE) proc = Popen([app, test_arg], stdin=PIPE, stdout=PIPE, stderr=PIPE)
# If a test takes longer than two seconds, kill it. # If a test takes longer than two seconds, kill it.

6
util/xcode/wren.xcodeproj/project.pbxproj
View File

@ -25,6 +25,7 @@
29729F321BA70A620099CA20 /* io.c in Sources */ = {isa = PBXBuildFile; fileRef = 29729F2E1BA70A620099CA20 /* io.c */; }; 29729F321BA70A620099CA20 /* io.c in Sources */ = {isa = PBXBuildFile; fileRef = 29729F2E1BA70A620099CA20 /* io.c */; };
29729F331BA70A620099CA20 /* io.wren.inc in Sources */ = {isa = PBXBuildFile; fileRef = 29729F301BA70A620099CA20 /* io.wren.inc */; }; 29729F331BA70A620099CA20 /* io.wren.inc in Sources */ = {isa = PBXBuildFile; fileRef = 29729F301BA70A620099CA20 /* io.wren.inc */; };
2986F6D71ACF93BA00BCE26C /* wren_primitive.c in Sources */ = {isa = PBXBuildFile; fileRef = 2986F6D51ACF93BA00BCE26C /* wren_primitive.c */; }; 2986F6D71ACF93BA00BCE26C /* wren_primitive.c in Sources */ = {isa = PBXBuildFile; fileRef = 2986F6D51ACF93BA00BCE26C /* wren_primitive.c */; };
29932D511C20D8C900099DEE /* benchmark.c in Sources */ = {isa = PBXBuildFile; fileRef = 29932D4F1C20D8C900099DEE /* benchmark.c */; };
29A427341BDBE435001E6E22 /* wren_opt_meta.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */; }; 29A427341BDBE435001E6E22 /* wren_opt_meta.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */; };
29A427351BDBE435001E6E22 /* wren_opt_meta.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */; }; 29A427351BDBE435001E6E22 /* wren_opt_meta.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */; };
29A427361BDBE435001E6E22 /* wren_opt_meta.wren.inc in Sources */ = {isa = PBXBuildFile; fileRef = 29A427301BDBE435001E6E22 /* wren_opt_meta.wren.inc */; }; 29A427361BDBE435001E6E22 /* wren_opt_meta.wren.inc in Sources */ = {isa = PBXBuildFile; fileRef = 29A427301BDBE435001E6E22 /* wren_opt_meta.wren.inc */; };
@ -103,6 +104,8 @@
29729F301BA70A620099CA20 /* io.wren.inc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.pascal; name = io.wren.inc; path = ../../src/module/io.wren.inc; sourceTree = "<group>"; }; 29729F301BA70A620099CA20 /* io.wren.inc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.pascal; name = io.wren.inc; path = ../../src/module/io.wren.inc; sourceTree = "<group>"; };
2986F6D51ACF93BA00BCE26C /* wren_primitive.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = wren_primitive.c; path = ../../src/vm/wren_primitive.c; sourceTree = "<group>"; }; 2986F6D51ACF93BA00BCE26C /* wren_primitive.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = wren_primitive.c; path = ../../src/vm/wren_primitive.c; sourceTree = "<group>"; };
2986F6D61ACF93BA00BCE26C /* wren_primitive.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = wren_primitive.h; path = ../../src/vm/wren_primitive.h; sourceTree = "<group>"; }; 2986F6D61ACF93BA00BCE26C /* wren_primitive.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = wren_primitive.h; path = ../../src/vm/wren_primitive.h; sourceTree = "<group>"; };
29932D4F1C20D8C900099DEE /* benchmark.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = benchmark.c; path = ../../test/api/benchmark.c; sourceTree = "<group>"; };
29932D501C20D8C900099DEE /* benchmark.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = benchmark.h; path = ../../test/api/benchmark.h; sourceTree = "<group>"; };
29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = wren_opt_meta.c; path = ../../src/optional/wren_opt_meta.c; sourceTree = "<group>"; }; 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = wren_opt_meta.c; path = ../../src/optional/wren_opt_meta.c; sourceTree = "<group>"; };
29A4272F1BDBE435001E6E22 /* wren_opt_meta.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = wren_opt_meta.h; path = ../../src/optional/wren_opt_meta.h; sourceTree = "<group>"; }; 29A4272F1BDBE435001E6E22 /* wren_opt_meta.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = wren_opt_meta.h; path = ../../src/optional/wren_opt_meta.h; sourceTree = "<group>"; };
29A427301BDBE435001E6E22 /* wren_opt_meta.wren.inc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.pascal; name = wren_opt_meta.wren.inc; path = ../../src/optional/wren_opt_meta.wren.inc; sourceTree = "<group>"; }; 29A427301BDBE435001E6E22 /* wren_opt_meta.wren.inc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.pascal; name = wren_opt_meta.wren.inc; path = ../../src/optional/wren_opt_meta.wren.inc; sourceTree = "<group>"; };
@ -239,6 +242,8 @@
29D0099A1B7E394F000CE58C /* api_test */ = { 29D0099A1B7E394F000CE58C /* api_test */ = {
isa = PBXGroup; isa = PBXGroup;
children = ( children = (
29932D4F1C20D8C900099DEE /* benchmark.c */,
29932D501C20D8C900099DEE /* benchmark.h */,
29D009A61B7E3993000CE58C /* main.c */, 29D009A61B7E3993000CE58C /* main.c */,
293D46941BB43F9900200083 /* call.c */, 293D46941BB43F9900200083 /* call.c */,
293D46951BB43F9900200083 /* call.h */, 293D46951BB43F9900200083 /* call.h */,
@ -356,6 +361,7 @@
291647C81BA5EC5E006142EE /* modules.c in Sources */, 291647C81BA5EC5E006142EE /* modules.c in Sources */,
29DC14A11BBA2FEC008A8274 /* scheduler.c in Sources */, 29DC14A11BBA2FEC008A8274 /* scheduler.c in Sources */,
29A427391BDBE435001E6E22 /* wren_opt_random.c in Sources */, 29A427391BDBE435001E6E22 /* wren_opt_random.c in Sources */,
29932D511C20D8C900099DEE /* benchmark.c in Sources */,
29DC14A01BBA2FD6008A8274 /* timer.c in Sources */, 29DC14A01BBA2FD6008A8274 /* timer.c in Sources */,
29DC149F1BBA2FCC008A8274 /* vm.c in Sources */, 29DC149F1BBA2FCC008A8274 /* vm.c in Sources */,
29DC14A21BBA300A008A8274 /* wren_compiler.c in Sources */, 29DC14A21BBA300A008A8274 /* wren_compiler.c in Sources */,