retoor/wren
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: remove JS benchmarks and reduce depth for faster binary_trees/fib runs

Browse Source 
Switch binary_trees max depth from 14 to 12 across all languages and reduce fib argument from 30 to 28, dropping JS entirely since its performance is incomparable. Update expected output patterns in run_bench script to match new tab-free format and use best-time comparison instead of mean.
This commit is contained in:
Bob Nystrom
2013-12-13 00:59:57 +00:00
parent 5e62b5856e
commit 8842efbab7
12 changed files with 82 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmark/run_bench
+61 -29
View File
@@ -7,42 +7,40 @@ import re
import subprocess
import sys
# How many times to run a given benchmark. Should be an odd number to get the
# right median.
NUM_TRIALS = 7
BENCHMARKS = []
def BENCHMARK(name, pattern):
regex = re.compile(pattern + "\n" + r"elapsed: (\d+\.\d+)", re.MULTILINE)
BENCHMARKS.append([name, regex, None])
BENCHMARK("binary_trees", """stretch tree of depth 15\t check: -1
32768\t trees of depth 4\t check: -32768
8192\t trees of depth 6\t check: -8192
2048\t trees of depth 8\t check: -2048
512\t trees of depth 10\t check: -512
128\t trees of depth 12\t check: -128
32\t trees of depth 14\t check: -32
long lived tree of depth 14\t check: -1""")
BENCHMARK("binary_trees", """stretch tree of depth 13 check: -1
8192 trees of depth 4 check: -8192
2048 trees of depth 6 check: -2048
512 trees of depth 8 check: -512
128 trees of depth 10 check: -128
32 trees of depth 12 check: -32
long lived tree of depth 12 check: -1""")
BENCHMARK("fib", r"""832040
832040
832040
832040
832040""")
BENCHMARK("fib", r"""317811
317811
317811
317811
317811""")
BENCHMARK("method_call", r"""true
false""")
LANGUAGES = [
("wren", "../build/Release/wren", ".wren"),
("js", "node", ".js"),
("lua", "lua", ".lua"),
("python", "python", ".py"),
("ruby", "ruby", ".rb")
]
# How many times to run a given benchmark. Should be an odd number to get the
# right median.
NUM_TRIALS = 7
results = []
def green(text):
@@ -62,16 +60,17 @@ def yellow(text):
def calc_stats(nums):
"""Calculates the mean, median, and std deviation of a list of numbers."""
"""Calculates the best, mean, and median of a list of numbers."""
mean = sum(nums) / len(nums)
nums.sort()
median = nums[(len(nums) - 1) / 2]
diffs = ((n - mean) * (n - mean) for n in nums)
std_dev = math.sqrt(sum(diffs) / len(nums))
return [mean, median, std_dev]
return [nums[0], mean, median, std_dev]
def run_benchmark_once(benchmark, language):
def run_trial(benchmark, language):
"""Runs one benchmark one time for one language."""
args = [language[1], benchmark[0] + language[2]]
out = subprocess.check_output(args, universal_newlines=True)
match = benchmark[1].match(out)
@@ -84,6 +83,7 @@ def run_benchmark_once(benchmark, language):
def run_benchmark_language(benchmark, language):
"""Runs one benchmark for a number of trials for one language."""
name = "{0} - {1}".format(benchmark[0], language[0])
print "{0:22s}".format(name),
@@ -93,7 +93,7 @@ def run_benchmark_language(benchmark, language):
times = []
for i in range(0, NUM_TRIALS):
time = run_benchmark_once(benchmark, language)
time = run_trial(benchmark, language)
if not time:
return
times.append(time)
@@ -105,7 +105,7 @@ def run_benchmark_language(benchmark, language):
comparison = ""
if language[0] == "wren":
if benchmark[2] != None:
ratio = 100 * stats[1] / benchmark[2]
ratio = 100 * stats[0] / benchmark[2]
comparison = "{0:.2f}% of baseline".format(ratio)
if ratio > 105:
comparison = red(comparison)
@@ -123,14 +123,15 @@ def run_benchmark_language(benchmark, language):
if ratio > 1:
comparison = green(comparison)
print " mean: {0:.2f} median: {1:.2f} std_dev: {2:.2f} {3:s}".format(
print " best: {0:.2f} mean: {1:.2f} median: {2:.2f} {3:s}".format(
stats[0], stats[1], stats[2], comparison)
results.append([name, times, stats[1]])
results.append([name, times, stats[0]])
return stats
def run_benchmark(benchmark, languages):
"""Runs one benchmark for the given languages (or all of them)."""
for language in LANGUAGES:
if not languages or language[0] in languages:
run_benchmark_language(benchmark, language)
@@ -138,6 +139,37 @@ def run_benchmark(benchmark, languages):
del results[0:len(results)]
# TODO(bob): Hook this up so it can be called.
def solo_benchmark(benchmark, language):
"""Runs a single language benchmark repeatedly, graphing the results."""
base = benchmark[2]
total = 0
for i in range(0, NUM_TRIALS):
time = run_trial(benchmark, language)
total += time
ratio = 100 * time / base
# TODO(bob): Show scale.
line = [" "] * 51
line[25] = "|"
index = 25 + int((time - base) * 200)
if index < 0: index = 0
if index > 50: index = 50
line[index] = "*"
comparison = "{0:.4f} ({1:6.2f}%) {2}".format(time, ratio, "".join(line))
if ratio > 105:
comparison = red(comparison)
if ratio < 95:
comparison = green(comparison)
print comparison
total /= NUM_TRIALS
print "----"
print "{0:.4f} ({1:6.2f}%)".format(total, 100 * total / base)
def graph_results():
print
@@ -168,10 +200,10 @@ def read_baseline():
if os.path.exists("baseline.txt"):
with open("baseline.txt") as f:
for line in f.readlines():
name, mean, median = line.split(",")
name, best = line.split(",")
for benchmark in BENCHMARKS:
if benchmark[0] == name:
benchmark[2] = float(median)
benchmark[2] = float(best)
def generate_baseline():
@@ -179,8 +211,7 @@ def generate_baseline():
baseline_text = ""
for benchmark in BENCHMARKS:
stats = run_benchmark_language(benchmark, LANGUAGES[0])
baseline_text += ("{},{},{}\n".format(
benchmark[0], stats[0], stats[1]))
baseline_text += ("{},{}\n".format(benchmark[0], stats[0]))
# Write them to a file.
with open("baseline.txt", 'w') as out:
@@ -207,6 +238,7 @@ def main():
read_baseline()
# Run all benchmarks.
if args.benchmark == "all":
for benchmark in BENCHMARKS:
run_benchmark(benchmark, args.language)