From aaa8b369faf3ba75e8702b2d78db4c1e252a79e4 Mon Sep 17 00:00:00 2001
From: Simon Michael <simon@joyful.com>
Date: Sat, 13 Feb 2010 18:39:49 +0000
Subject: [PATCH] tools: benchmarking updates, add experimental
 criterion/progression runners

---
 Makefile                           | 34 ++++++++++++++++++++++--------
 tools/criterionbench.hs            | 10 +++++++++
 tools/progressionbench.hs          |  9 ++++++++
 tools/{bench.hs => simplebench.hs} |  0
 4 files changed, 44 insertions(+), 9 deletions(-)
 create mode 100644 tools/criterionbench.hs
 create mode 100644 tools/progressionbench.hs
 rename tools/{bench.hs => simplebench.hs} (100%)

diff --git a/Makefile b/Makefile
index 4166e1b20..e37c3f4cc 100644
--- a/Makefile
+++ b/Makefile
@@ -14,10 +14,10 @@ PROFCMD=-f 1000x1000x10.ledger balance
 # command to run during "make coverage"
 COVCMD=test
 
-# executables to run during "make benchmark". They should be on the path
+# executables to run during "make simplebench". They should be on the path
 # or in the current directory. hledger executables should generally be the
 # standard cabal builds, also constrained to parsec 2 for now.
-BENCHEXES=hledger-0.6 hledger-0.7 ledger-3pre
+BENCHEXES=hledger-0.7 hledger-0.8 ledger-3pre
 
 # misc. tools
 PANDOC=pandoc
@@ -113,9 +113,17 @@ tools/unittest: tools/unittest.hs
 tools/doctest: tools/doctest.hs
 	ghc --make tools/doctest.hs
 
-# build the benchmark runner. Requires tabular.
-tools/bench: tools/bench.hs
-	ghc --make tools/bench.hs
+# build the simple benchmark runner. Requires tabular.
+tools/simplebench: tools/simplebench.hs
+	ghc --make tools/simplebench.hs
+
+# build the criterion-based benchmark runner. Requires criterion.
+tools/criterionbench: tools/criterionbench.hs
+	ghc --make tools/criterionbench.hs
+
+# build the progression-based benchmark runner. Requires progression.
+tools/progressionbench: tools/progressionbench.hs
+	ghc --make tools/progressionbench.hs
 
 # build the generateledger tool
 tools/generateledger: tools/generateledger.hs
@@ -195,13 +203,21 @@ fullcabaltest: setversion
 		&& cabal upload dist/hledger-$(VERSION).tar.gz --check -v3 \
 		&& echo $@ passed) || echo $@ FAILED
 
-# run performance benchmarks and save results in profs/. 
+# run performance benchmarks and save textual results in profs/.
 # Requires some commands defined in bench.tests and some BENCHEXES defined above.
-benchmark: sampleledgers bench.tests tools/bench
+simplebench bench: sampleledgers bench.tests tools/simplebench
 	PATH=.:$(PATH) tools/bench -fbench.tests $(BENCHEXES) | tee profs/$(TIME).bench
 	@rm -f benchresults.*
 	@(cd profs; rm -f latest.bench; ln -s $(TIME).bench latest.bench)
 
+# run criterion benchmark tests and save graphical results
+criterionbench: sampleledgers tools/criterionbench
+	tools/criterionbench -t png -k png
+
+# run progression benchmark tests and save graphical results
+progressionbench: sampleledgers tools/progressionbench
+	tools/progressionbench -- -t png -k png
+
 # generate, save and display a standard profile
 prof: sampleledgers hledgerp
 	@echo "Profiling $(PROFCMD)"
@@ -401,7 +417,7 @@ hoogleindex:
 #
 # - The VERSION file must be updated manually before a release.
 #
-# - "make benchmark" depends on version numbers in BENCHEXES, these also
+# - "make simplebench" depends on version numbers in BENCHEXES, these also
 #   must be updated manually.
 #
 # - "make" updates the version in most other places, and defines PATCHES.
@@ -489,7 +505,7 @@ pushbinary:
 	-rsync -aP $(BINARYFILENAME).gz joyful.com:/repos/hledger/website/binaries/
 
 # show project stats useful for release notes
-stats: showlastreleasedate showreleaseauthors showloc showcov showerrors showlocalchanges showreleasechanges benchmark
+stats: showlastreleasedate showreleaseauthors showloc showcov showerrors showlocalchanges showreleasechanges simplebench
 
 showreleaseauthors:
 	@echo Patch authors since last release:
diff --git a/tools/criterionbench.hs b/tools/criterionbench.hs
new file mode 100644
index 000000000..aeb634b78
--- /dev/null
+++ b/tools/criterionbench.hs
@@ -0,0 +1,10 @@
+{- Criterion-based benchmarks. Criterion displays and minimises the impact
+of time variance and charts the results. -}
+
+import Criterion.Main
+import System.Environment (withArgs)
+import qualified HledgerMain
+
+main = defaultMain [
+        bench "balance_100x100x10" $ nfIO $ withArgs ["balance", "-f", "100x100x10.ledger", ">/dev/null"] HledgerMain.main
+       ]
diff --git a/tools/progressionbench.hs b/tools/progressionbench.hs
new file mode 100644
index 000000000..2b60e8923
--- /dev/null
+++ b/tools/progressionbench.hs
@@ -0,0 +1,9 @@
+{- Progression-based benchmarks. Progression charts the difference between successive benchmark runs. -}
+
+import Criterion.Main hiding (defaultMain, defaultMainWith)
+import Progression.Main
+import System.Environment (withArgs)
+import qualified HledgerMain
+
+main = defaultMain $
+        bench "balance_100x100x10" $ nfIO $ withArgs ["balance", "-f", "100x100x10.ledger", ">/dev/null"] HledgerMain.main
diff --git a/tools/bench.hs b/tools/simplebench.hs
similarity index 100%
rename from tools/bench.hs
rename to tools/simplebench.hs