fix: add explicit return statements in benchmark functions and add method_call benchmark suite

benchmark/run_bench

@@ -0,0 +1,140 @@
+#!/usr/bin/python
+
+import math
+import os
+import re
+import subprocess
+import sys
+
+BENCHMARKS = []
+
+def BENCHMARK(name, pattern):
+  regex = re.compile(pattern + "\n" + r"elapsed: (\d+\.\d+)", re.MULTILINE)
+  BENCHMARKS.append((name, regex))
+
+BENCHMARK("binary_trees", """stretch tree of depth 15\t check: -1
+32768\t trees of depth 4\t check: -32768
+8192\t trees of depth 6\t check: -8192
+2048\t trees of depth 8\t check: -2048
+512\t trees of depth 10\t check: -512
+128\t trees of depth 12\t check: -128
+32\t trees of depth 14\t check: -32
+long lived tree of depth 14\t check: -1""")
+
+BENCHMARK("fib", r"""832040
+832040
+832040
+832040
+832040""")
+
+BENCHMARK("method_call", r"""true
+false""")
+
+LANGUAGES = [
+  ("wren",   "../build/Release/wren", ".wren"),
+  ("js",     "node",                  ".js"),
+  ("lua",    "lua",                   ".lua"),
+  ("python", "python",                ".py"),
+  ("ruby",   "ruby",                  ".rb")
+]
+
+# How many times to run a given benchmark. Should be an odd number to get the
+# right median.
+NUM_TRIALS = 5
+
+results = []
+
+def calc_stats(nums):
+  """Calculates the mean, median, and std deviation of a list of numbers."""
+  mean = sum(nums) / len(nums)
+  nums.sort()
+  median = nums[(len(nums) - 1) / 2]
+  diffs = ((n - mean) * (n - mean) for n in nums)
+  std_dev = math.sqrt(sum(diffs) / len(nums))
+  return [mean, median, std_dev]
+
+
+def run_benchmark_once(benchmark, language):
+  args = [language[1], benchmark[0] + language[2]]
+  out = subprocess.check_output(args, universal_newlines=True)
+  match = benchmark[1].match(out)
+  if match:
+    return float(match.group(1))
+  else:
+    print "Incorrect output:"
+    print out
+    return None
+
+
+def run_benchmark_language(benchmark, language):
+  name = "{0} - {1}".format(benchmark[0], language[0])
+  print "{0:22s}".format(name),
+
+  if not os.path.exists(benchmark[0] + language[2]):
+    print "No implementation for this language"
+    return
+
+  times = []
+  for i in range(0, NUM_TRIALS):
+    time = run_benchmark_once(benchmark, language)
+    if not time:
+      return
+    times.append(time)
+    sys.stdout.write(".")
+
+  times.sort()
+  stats = calc_stats(times)
+  print " mean: {0:.4f} median: {1:.4f} std_dev: {2:.4f}".format(
+      stats[0], stats[1], stats[2])
+
+  results.append([name, times])
+
+
+def run_benchmark(benchmark):
+  for language in LANGUAGES:
+    run_benchmark_language(benchmark, language)
+  graph_results()
+  del results[0:len(results)]
+
+
+def graph_results():
+  print
+
+  INCREMENT = {
+    '-': 'o',
+    'o': 'O',
+    'O': '0',
+    '0': '0'
+  }
+
+  # Scale everything by the highest time.
+  highest = 0
+  for result in results:
+    time = max(result[1])
+    if time > highest: highest = time
+
+  print "{0:22s}0.0 {1:64.4f}".format("", highest)
+  for result in results:
+    line = ["-"] * 68
+    for time in result[1]:
+      index = int(time / highest * 67)
+      line[index] = INCREMENT[line[index]]
+    print "{0:22s}{1}".format(result[0], "".join(line))
+  print
+
+
+def main():
+  # If a benchmark name is passed, just run it.
+  if len(sys.argv) == 2:
+    benchmark_name = sys.argv[1]
+    for benchmark in BENCHMARKS:
+      if benchmark[0] == benchmark_name:
+        run_benchmark(benchmark)
+    return
+
+  # Otherwise run them all.
+  for benchmark in BENCHMARKS:
+    run_benchmark(benchmark)
+
+
+main()