feat: add benchmark for Wren C API foreign method calls and refactor test class names

Browse Source 
Add a new benchmark that measures the performance of calling foreign methods through the Wren C embedding API, specifically testing argument passing and return values with a tight loop of 10 million calls. Refactor test API classes from generic "Api" to specific names like "Call", "Returns", "Value", and "ForeignClass" to avoid naming conflicts with the new benchmark module. Update the benchmark runner to use the test_api executable for API-specific benchmarks and remove the special API test argument handling from the test runner.
This commit is contained in:
Bob Nystrom 2015-12-16 00:02:13 +00:00
parent ba0d308fba
commit 5d8ecad5a6
16 changed files with 118 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -46,6 +46,10 @@ test: debug
@ $(MAKE) -f util/wren.mk MODE=debug test
@ ./util/test.py $(suite)
benchmark: release
@ $(MAKE) -f util/wren.mk test
@ ./util/benchmark.py $(suite)
# Generate the Wren site.
docs:
@ ./util/generate_docs.py

21
test/api/benchmark.c Normal file
View File

@ -0,0 +1,21 @@
#include <string.h>
#include "benchmark.h"
static void arguments(WrenVM* vm)
{
double result = 0;
result += wrenGetArgumentDouble(vm, 1);
result += wrenGetArgumentDouble(vm, 2);
result += wrenGetArgumentDouble(vm, 3);
result += wrenGetArgumentDouble(vm, 4);
wrenReturnDouble(vm, result);
}
WrenForeignMethodFn benchmarkBindMethod(const char* signature)
{
if (strcmp(signature, "static Benchmark.arguments(_,_,_,_)") == 0) return arguments;
return NULL;
}

3
test/api/benchmark.h Normal file
View File

@ -0,0 +1,3 @@
#include "wren.h"
WrenForeignMethodFn benchmarkBindMethod(const char* signature);

10
test/api/call.c
View File

@ -5,10 +5,10 @@
void callRunTests(WrenVM* vm)
{
WrenValue* noParams = wrenGetMethod(vm, "main", "Api", "noParams");
WrenValue* zero = wrenGetMethod(vm, "main", "Api", "zero()");
WrenValue* one = wrenGetMethod(vm, "main", "Api", "one(_)");
WrenValue* two = wrenGetMethod(vm, "main", "Api", "two(_,_)");
WrenValue* noParams = wrenGetMethod(vm, "main", "Call", "noParams");
WrenValue* zero = wrenGetMethod(vm, "main", "Call", "zero()");
WrenValue* one = wrenGetMethod(vm, "main", "Call", "one(_)");
WrenValue* two = wrenGetMethod(vm, "main", "Call", "two(_,_)");
// Different arity.
wrenCall(vm, noParams, NULL, "");
@ -16,7 +16,7 @@ void callRunTests(WrenVM* vm)
wrenCall(vm, one, NULL, "i", 1);
wrenCall(vm, two, NULL, "ii", 1, 2);
WrenValue* getValue = wrenGetMethod(vm, "main", "Api", "getValue(_)");
WrenValue* getValue = wrenGetMethod(vm, "main", "Call", "getValue(_)");
// Returning a value.
WrenValue* value = NULL;

2
test/api/call.wren
View File

@ -1,4 +1,4 @@
class Api {
class Call {
static noParams {
System.print("noParams")
}

2
test/api/foreign_class.c
View File

@ -79,7 +79,7 @@ static void resourceFinalize(WrenVM* vm)
WrenForeignMethodFn foreignClassBindMethod(const char* signature)
{
if (strcmp(signature, "static Api.finalized") == 0) return apiFinalized;
if (strcmp(signature, "static ForeignClass.finalized") == 0) return apiFinalized;
if (strcmp(signature, "Counter.increment(_)") == 0) return counterIncrement;
if (strcmp(signature, "Counter.value") == 0) return counterValue;
if (strcmp(signature, "Point.translate(_,_,_)") == 0) return pointTranslate;

8
test/api/foreign_class.wren
View File

@ -1,4 +1,4 @@
class Api {
class ForeignClass {
foreign static finalized
}
@ -64,14 +64,14 @@ var resources = [
]
System.gc()
System.print(Api.finalized) // expect: 0
System.print(ForeignClass.finalized) // expect: 0
resources.removeAt(-1)
System.gc()
System.print(Api.finalized) // expect: 1
System.print(ForeignClass.finalized) // expect: 1
resources.clear()
System.gc()
System.print(Api.finalized) // expect: 3
System.print(ForeignClass.finalized) // expect: 3

43
test/api/main.c
View File

@ -4,27 +4,19 @@
#include "vm.h"
#include "wren.h"
#include "benchmark.h"
#include "call.h"
#include "foreign_class.h"
#include "returns.h"
#include "value.h"
#define REGISTER_METHOD(name, camelCase) \
if (strcmp(testName, #name) == 0) return camelCase##BindMethod(fullName)
#define REGISTER_CLASS(name, camelCase) \
if (strcmp(testName, #name) == 0) \
{ \
camelCase##BindClass(className, &methods); \
}
// The name of the currently executing API test.
const char* testName;
static WrenForeignMethodFn bindForeignMethod(
WrenVM* vm, const char* module, const char* className,
bool isStatic, const char* signature)
{
{
if (strcmp(module, "main") != 0) return NULL;
// For convenience, concatenate all of the method qualifiers into a single
@ -35,10 +27,20 @@ static WrenForeignMethodFn bindForeignMethod(
strcat(fullName, className);
strcat(fullName, ".");
strcat(fullName, signature);
REGISTER_METHOD(foreign_class, foreignClass);
REGISTER_METHOD(returns, returns);
REGISTER_METHOD(value, value);
WrenForeignMethodFn method = NULL;
method = benchmarkBindMethod(fullName);
if (method != NULL) return method;
method = foreignClassBindMethod(fullName);
if (method != NULL) return method;
method = returnsBindMethod(fullName);
if (method != NULL) return method;
method = valueBindMethod(fullName);
if (method != NULL) return method;
fprintf(stderr,
"Unknown foreign method '%s' for test '%s'\n", fullName, testName);
@ -52,13 +54,13 @@ static WrenForeignClassMethods bindForeignClass(
WrenForeignClassMethods methods = { NULL, NULL };
if (strcmp(module, "main") != 0) return methods;
REGISTER_CLASS(foreign_class, foreignClass);
foreignClassBindClass(className, &methods);
return methods;
}
static void afterLoad(WrenVM* vm) {
if (strcmp(testName, "call") == 0) callRunTests(vm);
if (strstr(testName, "call.wren") != NULL) callRunTests(vm);
}
int main(int argc, const char* argv[])
@ -70,14 +72,7 @@ int main(int argc, const char* argv[])
}
testName = argv[1];
// The test script is at "test/api/<test>.wren".
char testPath[256];
strcpy(testPath, "test/api/");
strcat(testPath, testName);
strcat(testPath, ".wren");
setTestCallbacks(bindForeignMethod, bindForeignClass, afterLoad);
runFile(testPath);
runFile(testName);
return 0;
}

14
test/api/returns.c
View File

@ -39,13 +39,13 @@ static void returnBytes(WrenVM* vm)
WrenForeignMethodFn returnsBindMethod(const char* signature)
{
if (strcmp(signature, "static Api.implicitNull") == 0) return implicitNull;
if (strcmp(signature, "static Api.returnInt") == 0) return returnInt;
if (strcmp(signature, "static Api.returnFloat") == 0) return returnFloat;
if (strcmp(signature, "static Api.returnTrue") == 0) return returnTrue;
if (strcmp(signature, "static Api.returnFalse") == 0) return returnFalse;
if (strcmp(signature, "static Api.returnString") == 0) return returnString;
if (strcmp(signature, "static Api.returnBytes") == 0) return returnBytes;
if (strcmp(signature, "static Returns.implicitNull") == 0) return implicitNull;
if (strcmp(signature, "static Returns.returnInt") == 0) return returnInt;
if (strcmp(signature, "static Returns.returnFloat") == 0) return returnFloat;
if (strcmp(signature, "static Returns.returnTrue") == 0) return returnTrue;
if (strcmp(signature, "static Returns.returnFalse") == 0) return returnFalse;
if (strcmp(signature, "static Returns.returnString") == 0) return returnString;
if (strcmp(signature, "static Returns.returnBytes") == 0) return returnBytes;
return NULL;
}

16
test/api/returns.wren
View File

@ -1,4 +1,4 @@
class Api {
class Returns {
foreign static implicitNull
foreign static returnInt
@ -11,13 +11,13 @@ class Api {
foreign static returnBytes
}
System.print(Api.implicitNull == null) // expect: true
System.print(Returns.implicitNull == null) // expect: true
System.print(Api.returnInt) // expect: 123456
System.print(Api.returnFloat) // expect: 123.456
System.print(Returns.returnInt) // expect: 123456
System.print(Returns.returnFloat) // expect: 123.456
System.print(Api.returnTrue) // expect: true
System.print(Api.returnFalse) // expect: false
System.print(Returns.returnTrue) // expect: true
System.print(Returns.returnFalse) // expect: false
System.print(Api.returnString) // expect: a string
System.print(Api.returnBytes.bytes.toList) // expect: [97, 0, 98, 0, 99]
System.print(Returns.returnString) // expect: a string
System.print(Returns.returnBytes.bytes.toList) // expect: [97, 0, 98, 0, 99]

4
test/api/value.c
View File

@ -17,8 +17,8 @@ static void getValue(WrenVM* vm)
WrenForeignMethodFn valueBindMethod(const char* signature)
{
if (strcmp(signature, "static Api.value=(_)") == 0) return setValue;
if (strcmp(signature, "static Api.value") == 0) return getValue;
if (strcmp(signature, "static Value.value=(_)") == 0) return setValue;
if (strcmp(signature, "static Value.value") == 0) return getValue;
return NULL;
}

6
test/api/value.wren
View File

@ -1,9 +1,9 @@
class Api {
class Value {
foreign static value=(value)
foreign static value
}
Api.value = ["list", "of", "strings"]
Value.value = ["list", "of", "strings"]
// Do some stuff to trigger a GC (at least when GC stress testing enabled).
var s = "string"
@ -11,4 +11,4 @@ for (i in 1...10) {
s = s + " more"
}
System.print(Api.value) // expect: [list, of, strings]
System.print(Value.value) // expect: [list, of, strings]

20
test/benchmark/api_foreign_method.wren Normal file
View File

@ -0,0 +1,20 @@
class Benchmark {
foreign static arguments(a, b, c, d)
}
var start = System.clock
var result = 0
for (i in 1..1000000) {
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
result = result + Benchmark.arguments(1, 2, 3, 4)
}
System.print(result)
System.print("elapsed: %(System.clock - start)")

15
util/benchmark.py
View File

@ -51,6 +51,8 @@ def BENCHMARK(name, pattern):
regex = re.compile(pattern + "\n" + r"elapsed: (\d+\.\d+)", re.MULTILINE)
BENCHMARKS.append([name, regex, None])
BENCHMARK("api_foreign_method", "100000000")
BENCHMARK("binary_trees", """stretch tree of depth 13 check: -1
8192 trees of depth 4 check: -8192
2048 trees of depth 6 check: -2048
@ -143,9 +145,20 @@ def standard_deviation(times):
def run_trial(benchmark, language):
"""Runs one benchmark one time for one language."""
executable_args = language[1]
# Hackish. If the benchmark name starts with "api_", it's testing the Wren
# C API, so run the test_api executable which has those test methods instead
# of the normal Wren build.
if benchmark[0].startswith("api_"):
executable_args = [
os.path.join(WREN_DIR, "build", "release", "test", "wren")
]
args = []
args.extend(language[1])
args.extend(executable_args)
args.append(os.path.join(BENCHMARK_DIR, benchmark[0] + language[2]))
try:
out = subprocess.check_output(args, universal_newlines=True)
except OSError:

4
util/test.py
View File

@ -111,10 +111,6 @@ class Test:
def run(self, app, type):
# Invoke wren and run the test.
test_arg = self.path
if type == "api test":
# Just pass the suite name to API tests.
test_arg = basename(splitext(test_arg)[0])
proc = Popen([app, test_arg], stdin=PIPE, stdout=PIPE, stderr=PIPE)
# If a test takes longer than two seconds, kill it.

6
util/xcode/wren.xcodeproj/project.pbxproj
View File

@ -25,6 +25,7 @@
29729F321BA70A620099CA20 /* io.c in Sources */ = {isa = PBXBuildFile; fileRef = 29729F2E1BA70A620099CA20 /* io.c */; };
29729F331BA70A620099CA20 /* io.wren.inc in Sources */ = {isa = PBXBuildFile; fileRef = 29729F301BA70A620099CA20 /* io.wren.inc */; };
2986F6D71ACF93BA00BCE26C /* wren_primitive.c in Sources */ = {isa = PBXBuildFile; fileRef = 2986F6D51ACF93BA00BCE26C /* wren_primitive.c */; };
29932D511C20D8C900099DEE /* benchmark.c in Sources */ = {isa = PBXBuildFile; fileRef = 29932D4F1C20D8C900099DEE /* benchmark.c */; };
29A427341BDBE435001E6E22 /* wren_opt_meta.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */; };
29A427351BDBE435001E6E22 /* wren_opt_meta.c in Sources */ = {isa = PBXBuildFile; fileRef = 29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */; };
29A427361BDBE435001E6E22 /* wren_opt_meta.wren.inc in Sources */ = {isa = PBXBuildFile; fileRef = 29A427301BDBE435001E6E22 /* wren_opt_meta.wren.inc */; };
@ -103,6 +104,8 @@
29729F301BA70A620099CA20 /* io.wren.inc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.pascal; name = io.wren.inc; path = ../../src/module/io.wren.inc; sourceTree = "<group>"; };
2986F6D51ACF93BA00BCE26C /* wren_primitive.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = wren_primitive.c; path = ../../src/vm/wren_primitive.c; sourceTree = "<group>"; };
2986F6D61ACF93BA00BCE26C /* wren_primitive.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = wren_primitive.h; path = ../../src/vm/wren_primitive.h; sourceTree = "<group>"; };
29932D4F1C20D8C900099DEE /* benchmark.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = benchmark.c; path = ../../test/api/benchmark.c; sourceTree = "<group>"; };
29932D501C20D8C900099DEE /* benchmark.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = benchmark.h; path = ../../test/api/benchmark.h; sourceTree = "<group>"; };
29A4272E1BDBE435001E6E22 /* wren_opt_meta.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = wren_opt_meta.c; path = ../../src/optional/wren_opt_meta.c; sourceTree = "<group>"; };
29A4272F1BDBE435001E6E22 /* wren_opt_meta.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = wren_opt_meta.h; path = ../../src/optional/wren_opt_meta.h; sourceTree = "<group>"; };
29A427301BDBE435001E6E22 /* wren_opt_meta.wren.inc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.pascal; name = wren_opt_meta.wren.inc; path = ../../src/optional/wren_opt_meta.wren.inc; sourceTree = "<group>"; };
@ -239,6 +242,8 @@
29D0099A1B7E394F000CE58C /* api_test */ = {
isa = PBXGroup;
children = (
29932D4F1C20D8C900099DEE /* benchmark.c */,
29932D501C20D8C900099DEE /* benchmark.h */,
29D009A61B7E3993000CE58C /* main.c */,
293D46941BB43F9900200083 /* call.c */,
293D46951BB43F9900200083 /* call.h */,
@ -356,6 +361,7 @@
291647C81BA5EC5E006142EE /* modules.c in Sources */,
29DC14A11BBA2FEC008A8274 /* scheduler.c in Sources */,
29A427391BDBE435001E6E22 /* wren_opt_random.c in Sources */,
29932D511C20D8C900099DEE /* benchmark.c in Sources */,
29DC14A01BBA2FD6008A8274 /* timer.c in Sources */,
29DC149F1BBA2FCC008A8274 /* vm.c in Sources */,
29DC14A21BBA300A008A8274 /* wren_compiler.c in Sources */,