cli: a more robust and featureful bench test

The bench script invoked by cabal bench or stack bench now links the
benchmarked code directly, avoiding any dependence on the shell and
$PATH. Also, it reports the journal parse time and the various reports
on the parsed data separately, and runs quicker.

You can get more accurate benchmark times by running with --criterion.
This will usually give much the same numbers and takes much longer.

Benchmarking via simplebench is still available, by running with
--simplebench. In this mode it benchmarks whatever commands are
configured in bench/default.bench. A disadvantage is that it uses the
first "hledger" executable in $PATH, so check which one it reports.

hledger/bench/bench.hs

@@ -1,15 +1,79 @@
+-- bench
+-- By default, show approximate times for some standard hledger operations on a sample journal.
+-- With --criterion, show accurate times (slow).
+-- With --simplebench, show approximate times for the commands in default.bench, using the first hledger executable on $PATH.
+
 import Control.Concurrent (threadDelay)
+import Criterion.Main     (defaultMainWith, defaultConfig, bench, nfIO)
+-- import Criterion.Types
 import SimpleBench        (defaultMain)
-import System.Environment (withArgs)
+import System.Directory   (getCurrentDirectory)
+import System.Environment (getArgs, withArgs)
+import System.Info        (os)
+-- import System.IO          (hFlush, stdout)
+import System.Process     (readProcess)
+import System.TimeIt      (timeItT)
+import Text.Printf
+import Hledger.Cli
+
+-- sample journal file to use for benchmarks
+inputfile = "bench/10000x1000x10.journal"
+
+outputfile = "/dev/null" -- hide output of benchmarked commands (XXX unixism)
+-- outputfile = "-" -- show output of benchmarked commands
+
+-- a delay to avoid truncation of final output by "stack bench"
+-- https://github.com/commercialhaskell/stack/issues/413
+stackFinalOutputDelaySeconds = 0 -- 10
 
 main = do
-  -- expects to be run from the parent directory, as by cabal
+ -- withArgs ["--simplebench"] $ do
-  withArgs [
+ -- withArgs ["--criterion"] $ do
-    "-fbench/default.bench"
+  args <- getArgs
-   ,"dist/build/hledger/hledger"
+  if "--criterion" `elem` args
-   -- ,"-v"
+    then withArgs [] benchWithCriterion
-   ] defaultMain
+    else if "--simplebench" `elem` args
+         then benchWithSimplebench
+         else benchWithTimeit
+  -- hFlush stdout
+  threadDelay (stackFinalOutputDelaySeconds * 1000000)
 
-  -- a little delay to avoid truncation of final output by stack
+benchWithTimeit = do
-  -- in a slow-rendering terminal, such as an emacs shell
+  getCurrentDirectory >>= printf "Benchmarking hledger in %s with timeit\n"
-  threadDelay 500000
+  let opts = defcliopts{output_file_=Just outputfile}
+  (t0,j) <- timeit ("read "++inputfile) $ either error id <$> readJournalFile Nothing Nothing True inputfile
+  (t1,_) <- timeit ("print") $ print' opts j
+  (t2,_) <- timeit ("register") $ register opts j
+  (t3,_) <- timeit ("balance") $ balance  opts j
+  (t4,_) <- timeit ("stats") $ stats opts j
+  printf "Total: %0.2fs\n" (sum [t0,t1,t2,t3,t4])
+
+timeit :: String -> IO a -> IO (Double, a)
+timeit name action = do
+  printf "%s%s" name (replicate (40 - length name) ' ')
+  (t,a) <- timeItT action
+  printf "[%.2fs]\n" t
+  return (t,a)
+
+benchWithCriterion = do
+  getCurrentDirectory >>= printf "Benchmarking hledger in %s with criterion\n"
+  let opts = defcliopts{output_file_=Just "/dev/null"}
+  j <- either error id <$> readJournalFile Nothing Nothing True inputfile
+  Criterion.Main.defaultMainWith defaultConfig $ [
+    bench ("read "++inputfile) $ nfIO $ (either error const <$> readJournalFile Nothing Nothing True inputfile),
+    bench ("print")            $ nfIO $ print'   opts j,
+    bench ("register")         $ nfIO $ register opts j,
+    bench ("balance")          $ nfIO $ balance  opts j,
+    bench ("stats")            $ nfIO $ stats    opts j
+    ]
+
+benchWithSimplebench = do
+  let whichcmd = if os == "mingw32" then "where" else "which"
+  exe <- init <$> readProcess whichcmd ["hledger"] ""
+  pwd <- getCurrentDirectory
+  printf "Benchmarking %s in %s with simplebench and shell\n" exe pwd
+  flip withArgs SimpleBench.defaultMain [
+     "-fbench/default.bench"
+    ,"-v"
+    ,"hledger"
+    ] 

hledger/hledger.cabal

@@ -214,8 +214,10 @@ benchmark bench
                     hledger,
                     base >= 4.3 && < 5,
                     base-compat >= 0.8.1,
+                    criterion,
                     html,
                     tabular >= 0.2 && < 0.3,
+                    timeit,
                     process,
                     filepath,
                     directory

stack.yaml

@@ -4,4 +4,5 @@ packages:
 - hledger-web
 flags: {}
 resolver: nightly-2015-06-17
-extra-deps: []
+extra-deps:
+- timeit-1.0.0.0