Bob Nystrom
65615f4aa8
Extract hardcoded trial count into NUM_TRIALS constant set to 7, collect all timing results into a list, and add graph_results function that prints a horizontal ASCII bar chart scaling each run's times relative to the global maximum across all benchmarks and languages.
92 lines
2.1 KiB
Python
Executable File
92 lines
2.1 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
import math
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
FIB_OUTPUT_PATTERN = re.compile(r"""832040
|
|
832040
|
|
832040
|
|
832040
|
|
832040
|
|
elapsed: (\d+\.\d+)""", re.MULTILINE)
|
|
|
|
BENCHMARKS = [
|
|
("fib", FIB_OUTPUT_PATTERN)
|
|
]
|
|
|
|
LANGUAGES = [
|
|
("wren", "../build/Release/wren", ".wren"),
|
|
("lua", "lua", ".lua"),
|
|
("python", "python", ".py"),
|
|
("ruby", "ruby", ".rb")
|
|
]
|
|
|
|
# How many times to run a given benchmark. Should be an odd number to get the
|
|
# right median.
|
|
NUM_TRIALS = 7
|
|
|
|
results = []
|
|
|
|
def calc_stats(nums):
|
|
"""Calculates the mean, median, and std deviation of a list of numbers."""
|
|
mean = sum(nums) / len(nums)
|
|
nums.sort()
|
|
median = nums[(len(nums) - 1) / 2]
|
|
diffs = ((n - mean) * (n - mean) for n in nums)
|
|
std_dev = math.sqrt(sum(diffs) / len(nums))
|
|
return [mean, median, std_dev]
|
|
|
|
|
|
def run_benchmark_once(benchmark, language):
|
|
args = [language[1], benchmark[0] + language[2]]
|
|
out = subprocess.check_output(args, universal_newlines=True)
|
|
match = benchmark[1].match(out)
|
|
if match:
|
|
return float(match.group(1))
|
|
else:
|
|
return None
|
|
|
|
|
|
def run_benchmark(benchmark, language):
|
|
print "{0} - {1:10s}".format(benchmark[0], language[0]),
|
|
|
|
times = []
|
|
for i in range(0, NUM_TRIALS):
|
|
times.append(run_benchmark_once(benchmark, language))
|
|
sys.stdout.write(".")
|
|
|
|
if None in times:
|
|
print "error"
|
|
return
|
|
|
|
times.sort()
|
|
stats = calc_stats(times)
|
|
print " mean: {0:.4f} median: {1:.4f} std_dev: {2:.4f}".format(
|
|
stats[0], stats[1], stats[2])
|
|
|
|
results.append([benchmark[0] + " - " + language[0], times])
|
|
|
|
def graph_results():
|
|
print
|
|
|
|
# Scale everything by the highest time.
|
|
highest = 0
|
|
for result in results:
|
|
time = max(result[1])
|
|
if time > highest: highest = time
|
|
|
|
print "{0:15s} 0.0 {1:76.4f}".format("", highest)
|
|
for result in results:
|
|
line = ["-"] * 80
|
|
for time in result[1]:
|
|
line[int(time / highest * 79)] = "O"
|
|
print "{0:15s} {1}".format(result[0], "".join(line))
|
|
|
|
for benchmark in BENCHMARKS:
|
|
for language in LANGUAGES:
|
|
run_benchmark(benchmark, language)
|
|
|
|
graph_results()
|