feat: cli: Add tsv output (#869)

All commands that suport csv output now also support tsv output. The
data is identical, but the fields are separated by tab characters and
there is no quoting or escaping. Tab, carriage return, and newline
characters in data are converted to spaces (this should rarely if ever
happen in practice).

hledger-lib/Hledger/Read/CsvUtils.hs

@@ -13,6 +13,7 @@ CSV utilities.
 module Hledger.Read.CsvUtils (
   CSV, CsvRecord, CsvValue,
   printCSV,
+  printTSV,
   -- * Tests
   tests_CsvUtils,
 )
@@ -41,9 +42,15 @@ printCSV = TB.toLazyText . unlinesB . map printRecord
     where printRecord = foldMap TB.fromText . intersperse "," . map printField
           printField = wrap "\"" "\"" . T.replace "\"" "\"\""
 
+printTSV :: [CsvRecord] -> TL.Text
+printTSV = TB.toLazyText . unlinesB . map printRecord
+    where printRecord = foldMap TB.fromText . intersperse "\t" . map printField
+          printField = T.map replaceWhitespace
+          replaceWhitespace c | c `elem` ['\t', '\n', '\r'] = ' '
+          replaceWhitespace c = c
+
 --- ** tests
 
 tests_CsvUtils :: TestTree
 tests_CsvUtils = testGroup "CsvUtils" [
   ]
-

hledger/Hledger/Cli/CliOptions.hs

@@ -640,7 +640,7 @@ defaultOutputFormat :: String
 defaultOutputFormat = "txt"
 
 outputFormats :: [String]
-outputFormats = [defaultOutputFormat, "csv", "html"]
+outputFormats = [defaultOutputFormat, "csv", "tsv", "html"]
 
 -- | Get the output format from the --output-format option,
 -- otherwise from a recognised file extension in the --output-file option,

hledger/Hledger/Cli/Commands/Aregister.hs

@@ -29,7 +29,7 @@ import Lucid as L hiding (value_)
 import System.Console.CmdArgs.Explicit (flagNone, flagReq)
 
 import Hledger
-import Hledger.Read.CsvUtils (CSV, CsvRecord, printCSV)
+import Hledger.Read.CsvUtils (CSV, CsvRecord, printCSV, printTSV)
 import Hledger.Cli.CliOptions
 import Hledger.Cli.Utils
 import Text.Tabular.AsciiWide hiding (render)
@@ -58,7 +58,7 @@ aregistermode = hledgerCommandMode
       ++ " or $COLUMNS). -wN,M sets description width as well."
      )
   ,flagNone ["align-all"] (setboolopt "align-all") "guarantee alignment across all lines (slower)"
-  ,outputFormatFlag ["txt","html","csv","json"]
+  ,outputFormatFlag ["txt","html","csv","tsv","json"]
   ,outputFileFlag
   ])
   [generalflagsgroup1]
@@ -107,6 +107,7 @@ aregister opts@CliOpts{rawopts_=rawopts,reportspec_=rspec} j = do
     render | fmt=="txt"  = accountTransactionsReportAsText opts (_rsQuery rspec') thisacctq
            | fmt=="html" = accountTransactionsReportAsHTML opts (_rsQuery rspec') thisacctq
            | fmt=="csv"  = printCSV . accountTransactionsReportAsCsv wd (_rsQuery rspec') thisacctq
+           | fmt=="tsv"  = printTSV . accountTransactionsReportAsCsv wd (_rsQuery rspec') thisacctq
            | fmt=="json" = toJsonText
            | otherwise   = error' $ unsupportedOutputFormatError fmt  -- PARTIAL:
       where

hledger/Hledger/Cli/Commands/Aregister.md

@@ -62,7 +62,7 @@ at the cost of more time and memory, use the `--align-all` flag.
 This command also supports the
 [output destination](hledger.html#output-destination) and
 [output format](hledger.html#output-format) options.
-The output formats supported are `txt`, `csv`, and `json`.
+The output formats supported are `txt`, `csv`, `tsv`, and `json`.
 
 ### aregister and posting dates
 

hledger/Hledger/Cli/Commands/Balance.hs

@@ -277,7 +277,7 @@ import qualified Text.Tabular.AsciiWide as Tab
 import Hledger
 import Hledger.Cli.CliOptions
 import Hledger.Cli.Utils
-import Hledger.Read.CsvUtils (CSV, printCSV)
+import Hledger.Read.CsvUtils (CSV, printCSV, printTSV)
 
 
 -- | Command line options for this command.
@@ -332,7 +332,7 @@ balancemode = hledgerCommandMode
         ,"'tidy'        : every attribute in its own column"
         ])
     -- output:
-    ,outputFormatFlag ["txt","html","csv","json"]
+    ,outputFormatFlag ["txt","html","csv","tsv","json"]
     ,outputFileFlag
     ]
   )
@@ -353,6 +353,7 @@ balance opts@CliOpts{reportspec_=rspec} j = case balancecalc_ of
             "txt"  -> budgetReportAsText ropts
             "json" -> (<>"\n") . toJsonText
             "csv"  -> printCSV . budgetReportAsCsv ropts
+            "tsv"  -> printTSV . budgetReportAsCsv ropts
             _      -> error' $ unsupportedOutputFormatError fmt
       writeOutputLazyText opts $ render budgetreport
 
@@ -361,6 +362,7 @@ balance opts@CliOpts{reportspec_=rspec} j = case balancecalc_ of
             render = case fmt of
               "txt"  -> multiBalanceReportAsText ropts
               "csv"  -> printCSV . multiBalanceReportAsCsv ropts
+              "tsv"  -> printTSV . multiBalanceReportAsCsv ropts
               "html" -> (<>"\n") . L.renderText . multiBalanceReportAsHtml ropts
               "json" -> (<>"\n") . toJsonText
               _      -> const $ error' $ unsupportedOutputFormatError fmt  -- PARTIAL:
@@ -371,6 +373,7 @@ balance opts@CliOpts{reportspec_=rspec} j = case balancecalc_ of
             render = case fmt of
               "txt"  -> \ropts1 -> TB.toLazyText . balanceReportAsText ropts1
               "csv"  -> \ropts1 -> printCSV . balanceReportAsCsv ropts1
+              "tsv"  -> \ropts1 -> printTSV . balanceReportAsCsv ropts1
               -- "html" -> \ropts -> (<>"\n") . L.renderText . multiBalanceReportAsHtml ropts . balanceReportAsMultiBalanceReport ropts
               "json" -> const $ (<>"\n") . toJsonText
               _      -> error' $ unsupportedOutputFormatError fmt  -- PARTIAL:
@@ -378,8 +381,9 @@ balance opts@CliOpts{reportspec_=rspec} j = case balancecalc_ of
   where
     styles = journalCommodityStylesWith HardRounding j
     ropts@ReportOpts{..} = _rsReportOpts rspec
-    -- Tidy csv should be consistent between single period and multiperiod reports.
+    -- Tidy csv/tsv should be consistent between single period and multiperiod reports.
-    multiperiod = interval_ /= NoInterval || (layout_ == LayoutTidy && fmt == "csv")
+    multiperiod = interval_ /= NoInterval || (layout_ == LayoutTidy && delimited)
+    delimited   = fmt == "csv" || fmt == "tsv"
     fmt         = outputFormatFromOpts opts
 
 -- XXX this allows rough HTML rendering of a flat BalanceReport, but it can't handle tree mode etc.

hledger/Hledger/Cli/Commands/Balance.md

@@ -74,7 +74,7 @@ Many of these work with the higher-level commands as well.
 This command supports the
 [output destination](#output-destination) and
 [output format](#output-format) options,
-with output formats `txt`, `csv`, `json`, and (multi-period reports only:) `html`. 
+with output formats `txt`, `csv`, `tsv`, `json`, and (multi-period reports only:) `html`.
 In `txt` output in a colour-supporting terminal, negative amounts are shown in red.
 
 The `--related`/`-r` flag shows the balance of the *other* postings in the

hledger/Hledger/Cli/Commands/Balancesheet.md

@@ -48,4 +48,4 @@ This command also supports the
 [output destination](hledger.html#output-destination) and
 [output format](hledger.html#output-format) options
 The output formats supported are
-`txt`, `csv`, `html`, and (experimental) `json`.
+`txt`, `csv`, `tsv`, `html`, and (experimental) `json`.

hledger/Hledger/Cli/Commands/Balancesheetequity.md

@@ -51,4 +51,4 @@ This command also supports the
 [output destination](hledger.html#output-destination) and
 [output format](hledger.html#output-format) options
 The output formats supported are
-`txt`, `csv`, `html`, and (experimental) `json`.
+`txt`, `csv`, `tsv`, `html`, and (experimental) `json`.

hledger/Hledger/Cli/Commands/Cashflow.md

@@ -48,4 +48,4 @@ This command also supports the
 [output destination](hledger.html#output-destination) and
 [output format](hledger.html#output-format) options
 The output formats supported are
-`txt`, `csv`, `html`, and (experimental) `json`.
+`txt`, `csv`, `tsv`, `html`, and (experimental) `json`.

hledger/Hledger/Cli/Commands/Incomestatement.md

@@ -49,4 +49,4 @@ This command also supports the
 [output destination](hledger.html#output-destination) and
 [output format](hledger.html#output-format) options
 The output formats supported are
-`txt`, `csv`, `html`, and (experimental) `json`.
+`txt`, `csv`, `tsv`, `html`, and (experimental) `json`.

hledger/Hledger/Cli/Commands/Print.hs

@@ -24,7 +24,7 @@ import Lens.Micro ((^.), _Just, has)
 import System.Console.CmdArgs.Explicit
 
 import Hledger
-import Hledger.Read.CsvUtils (CSV, printCSV)
+import Hledger.Read.CsvUtils (CSV, printCSV, printTSV)
 import Hledger.Cli.CliOptions
 import Hledger.Cli.Utils
 import System.Exit (exitFailure)
@@ -54,7 +54,7 @@ printmode = hledgerCommandMode
   ,let arg = "DESC" in
    flagReq  ["match","m"] (\s opts -> Right $ setopt "match" s opts) arg
     ("fuzzy search for one recent transaction with description closest to "++arg)
-  ,outputFormatFlag ["txt","csv","json","sql"]
+  ,outputFormatFlag ["txt","csv","tsv","json","sql"]
   ,outputFileFlag
   ])
   [generalflagsgroup1]
@@ -107,6 +107,7 @@ printEntries opts@CliOpts{rawopts_=rawopts, reportspec_=rspec} j =
     fmt = outputFormatFromOpts opts
     render | fmt=="txt"  = entriesReportAsText opts      . styleAmounts styles
            | fmt=="csv"  = printCSV . entriesReportAsCsv . styleAmounts styles
+           | fmt=="tsv"  = printTSV . entriesReportAsCsv . styleAmounts styles
            | fmt=="json" = toJsonText                    . styleAmounts styles
            | fmt=="sql"  = entriesReportAsSql            . styleAmounts styles
            | otherwise   = error' $ unsupportedOutputFormatError fmt  -- PARTIAL:

hledger/Hledger/Cli/Commands/Print.md

@@ -115,7 +115,7 @@ This command also supports the
 [output destination](hledger.html#output-destination) and
 [output format](hledger.html#output-format) options
 The output formats supported are
-`txt`, `csv`, and (experimental) `json` and `sql`.
+`txt`, `csv`, `tsv`, and (experimental) `json` and `sql`.
 
 Here's an example of print's CSV output:
 

hledger/Hledger/Cli/Commands/Register.hs

@@ -27,7 +27,7 @@ import qualified Data.Text.Lazy.Builder as TB
 import System.Console.CmdArgs.Explicit (flagNone, flagReq)
 
 import Hledger hiding (per)
-import Hledger.Read.CsvUtils (CSV, CsvRecord, printCSV)
+import Hledger.Read.CsvUtils (CSV, CsvRecord, printCSV, printTSV)
 import Hledger.Cli.CliOptions
 import Hledger.Cli.Utils
 import Text.Tabular.AsciiWide hiding (render)
@@ -59,7 +59,7 @@ registermode = hledgerCommandMode
       ++ " or $COLUMNS). -wN,M sets description width as well."
      )
   ,flagNone ["align-all"] (setboolopt "align-all") "guarantee alignment across all lines (slower)"
-  ,outputFormatFlag ["txt","csv","json"]
+  ,outputFormatFlag ["txt","csv","tsv","json"]
   ,outputFileFlag
   ])
   [generalflagsgroup1]
@@ -88,6 +88,7 @@ register opts@CliOpts{rawopts_=rawopts, reportspec_=rspec} j
     rpt = postingsReport rspec j
     render | fmt=="txt"  = postingsReportAsText opts
            | fmt=="csv"  = printCSV . postingsReportAsCsv
+           | fmt=="tsv"  = printTSV . postingsReportAsCsv
            | fmt=="json" = toJsonText
            | otherwise   = error' $ unsupportedOutputFormatError fmt  -- PARTIAL:
       where fmt = outputFormatFromOpts opts

hledger/Hledger/Cli/Commands/Register.md

@@ -142,5 +142,4 @@ This command also supports the
 [output destination](hledger.html#output-destination) and
 [output format](hledger.html#output-format) options
 The output formats supported are
-`txt`, `csv`, and (experimental) `json`.
+`txt`, `csv`, `tsv`, and (experimental) `json`.
-

hledger/Hledger/Cli/CompoundBalanceCommand.hs

@@ -20,7 +20,7 @@ import qualified Data.Text.Lazy as TL
 import qualified Data.Text.Lazy.Builder as TB
 import Data.Time.Calendar (Day, addDays)
 import System.Console.CmdArgs.Explicit as C
-import Hledger.Read.CsvUtils (CSV, printCSV)
+import Hledger.Read.CsvUtils (CSV, printCSV, printTSV)
 import Lucid as L hiding (value_)
 import Text.Tabular.AsciiWide as Tab hiding (render)
 
@@ -93,7 +93,7 @@ compoundBalanceCommandMode CompoundBalanceCommandSpec{..} =
         ,"'tall'        : each commodity on a new line"
         ,"'bare'        : bare numbers, symbols in a column"
         ])
-    ,outputFormatFlag ["txt","html","csv","json"]
+    ,outputFormatFlag ["txt","html","csv","tsv","json"]
     ,outputFileFlag
     ])
     [generalflagsgroup1]
@@ -178,6 +178,7 @@ compoundBalanceCommand CompoundBalanceCommandSpec{..} opts@CliOpts{reportspec_=r
     render = case outputFormatFromOpts opts of
       "txt"  -> compoundBalanceReportAsText ropts'
       "csv"  -> printCSV . compoundBalanceReportAsCsv ropts'
+      "tsv"  -> printTSV . compoundBalanceReportAsCsv ropts'
       "html" -> L.renderText . compoundBalanceReportAsHtml ropts'
       "json" -> toJsonText
       x      -> error' $ unsupportedOutputFormatError x

hledger/hledger.m4.md

@@ -459,7 +459,7 @@ $ hledger print -o -        # write to stdout (the default)
 Some commands offer other kinds of output, not just text on the terminal.
 Here are those commands and the formats currently supported:
 
-| -                  | txt              | csv              | html               | json | sql |
+| -                  | txt              | csv/tsv          | html               | json | sql |
 |--------------------|------------------|------------------|--------------------|------|-----|
 | aregister          | Y                | Y                | Y                  | Y    |     |
 | balance            | Y *<sup>1</sup>* | Y *<sup>1</sup>* | Y *<sup>1,2</sup>* | Y    |     |

hledger/test/balance/budget.test

@@ -581,6 +581,16 @@ $ hledger bal -f- --budget -TA not:income -O csv
 "expenses:bills:f","$10","0","$10","0","$10","0"
 "Total:","$80","$370","$80","$370","$80","$370"
 
+# TSV output works.
+$ hledger bal -f- --budget -TA not:income -O tsv
+Account	2019-01-01..2019-01-02	budget	Total	budget	Average	budget
+expenses:bills	$80	$370	$80	$370	$80	$370
+expenses:bills:a	$10	$20	$10	$20	$10	$20
+expenses:bills:b	$40	$200	$40	$200	$40	$200
+expenses:bills:c		$50		$50		$50
+expenses:bills:f	$10	0	$10	0	$10	0
+Total:	$80	$370	$80	$370	$80	$370
+
 # ** 30. You would expect this to show a budget goal in jan, feb, mar.
 # But by the usual report date logic, which picks the oldest and newest 
 # transaction date (1/15 and 3/15) as start and end date by default, 

hledger/test/balance/layout.test

@@ -10,6 +10,13 @@ $ hledger -f bcexample.hledger bal assets.*etrade -3 -O csv
 "total","70.00 GLD, 17.00 ITOT, 5120.50 USD, 36.00 VEA, 294.00 VHT"
 >=0
 
+$ hledger -f bcexample.hledger bal assets.*etrade -3 -O tsv
+>
+account	balance
+Assets:US:ETrade	70.00 GLD, 17.00 ITOT, 5120.50 USD, 36.00 VEA, 294.00 VHT
+total	70.00 GLD, 17.00 ITOT, 5120.50 USD, 36.00 VEA, 294.00 VHT
+>=0
+
 # ** 2. Balance report csv output with one line per commodity (--layout=bare).
 $ hledger -f bcexample.hledger bal assets.*etrade -3 -O csv --layout=bare
 >
@@ -26,6 +33,21 @@ $ hledger -f bcexample.hledger bal assets.*etrade -3 -O csv --layout=bare
 "total","VHT","294.00"
 >=0
 
+$ hledger -f bcexample.hledger bal assets.*etrade -3 -O tsv --layout=bare
+>
+account	commodity	balance
+Assets:US:ETrade	GLD	70.00
+Assets:US:ETrade	ITOT	17.00
+Assets:US:ETrade	USD	5120.50
+Assets:US:ETrade	VEA	36.00
+Assets:US:ETrade	VHT	294.00
+total	GLD	70.00
+total	ITOT	17.00
+total	USD	5120.50
+total	VEA	36.00
+total	VHT	294.00
+>=0
+
 # ** 3. Balance report output with no commodity column.
 $ hledger -f bcexample.hledger bal assets.*etrade -3
 >
@@ -66,6 +88,13 @@ $ hledger -f bcexample.hledger bal -T -Y assets.*etrade -3 -O csv
 "total","10.00 ITOT, 337.18 USD, 12.00 VEA, 106.00 VHT","70.00 GLD, 18.00 ITOT, -98.12 USD, 10.00 VEA, 18.00 VHT","-11.00 ITOT, 4881.44 USD, 14.00 VEA, 170.00 VHT","70.00 GLD, 17.00 ITOT, 5120.50 USD, 36.00 VEA, 294.00 VHT"
 >=0
 
+$ hledger -f bcexample.hledger bal -T -Y assets.*etrade -3 -O tsv
+>
+account	2012	2013	2014	total
+Assets:US:ETrade	10.00 ITOT, 337.18 USD, 12.00 VEA, 106.00 VHT	70.00 GLD, 18.00 ITOT, -98.12 USD, 10.00 VEA, 18.00 VHT	-11.00 ITOT, 4881.44 USD, 14.00 VEA, 170.00 VHT	70.00 GLD, 17.00 ITOT, 5120.50 USD, 36.00 VEA, 294.00 VHT
+total	10.00 ITOT, 337.18 USD, 12.00 VEA, 106.00 VHT	70.00 GLD, 18.00 ITOT, -98.12 USD, 10.00 VEA, 18.00 VHT	-11.00 ITOT, 4881.44 USD, 14.00 VEA, 170.00 VHT	70.00 GLD, 17.00 ITOT, 5120.50 USD, 36.00 VEA, 294.00 VHT
+>=0
+
 # ** 6. Multicolumn balance report csv output with --layout=bare.
 $ hledger -f bcexample.hledger bal -T -Y assets.*etrade -3 -O csv --layout=bare
 >
@@ -82,6 +111,21 @@ $ hledger -f bcexample.hledger bal -T -Y assets.*etrade -3 -O csv --layout=bare
 "total","VHT","106.00","18.00","170.00","294.00"
 >=0
 
+$ hledger -f bcexample.hledger bal -T -Y assets.*etrade -3 -O tsv --layout=bare
+>
+account	commodity	2012	2013	2014	total
+Assets:US:ETrade	GLD	0	70.00	0	70.00
+Assets:US:ETrade	ITOT	10.00	18.00	-11.00	17.00
+Assets:US:ETrade	USD	337.18	-98.12	4881.44	5120.50
+Assets:US:ETrade	VEA	12.00	10.00	14.00	36.00
+Assets:US:ETrade	VHT	106.00	18.00	170.00	294.00
+total	GLD	0	70.00	0	70.00
+total	ITOT	10.00	18.00	-11.00	17.00
+total	USD	337.18	-98.12	4881.44	5120.50
+total	VEA	12.00	10.00	14.00	36.00
+total	VHT	106.00	18.00	170.00	294.00
+>=0
+
 # ** 7. Multicolumn balance report with --layout=bare.
 $ hledger -f bcexample.hledger bal -Y assets.*etrade -3 --average --layout=bare --no-total
 >
@@ -310,6 +354,26 @@ $ hledger -f bcexample.hledger bal -T -Y assets.*etrade -3 -O csv --layout=tidy
 "Assets:US:ETrade","2014","2014-01-01","2014-12-31","VHT","170.00"
 >=0
 
+$ hledger -f bcexample.hledger bal -T -Y assets.*etrade -3 -O tsv --layout=tidy
+>
+account	period	start_date	end_date	commodity	value
+Assets:US:ETrade	2012	2012-01-01	2012-12-31	GLD	0
+Assets:US:ETrade	2012	2012-01-01	2012-12-31	ITOT	10.00
+Assets:US:ETrade	2012	2012-01-01	2012-12-31	USD	337.18
+Assets:US:ETrade	2012	2012-01-01	2012-12-31	VEA	12.00
+Assets:US:ETrade	2012	2012-01-01	2012-12-31	VHT	106.00
+Assets:US:ETrade	2013	2013-01-01	2013-12-31	GLD	70.00
+Assets:US:ETrade	2013	2013-01-01	2013-12-31	ITOT	18.00
+Assets:US:ETrade	2013	2013-01-01	2013-12-31	USD	-98.12
+Assets:US:ETrade	2013	2013-01-01	2013-12-31	VEA	10.00
+Assets:US:ETrade	2013	2013-01-01	2013-12-31	VHT	18.00
+Assets:US:ETrade	2014	2014-01-01	2014-12-31	GLD	0
+Assets:US:ETrade	2014	2014-01-01	2014-12-31	ITOT	-11.00
+Assets:US:ETrade	2014	2014-01-01	2014-12-31	USD	4881.44
+Assets:US:ETrade	2014	2014-01-01	2014-12-31	VEA	14.00
+Assets:US:ETrade	2014	2014-01-01	2014-12-31	VHT	170.00
+>=0
+
 # ** 16. Single column balance report csv output with --layout=tidy
 $ hledger -f bcexample.hledger bal -T assets.*etrade -3 -O csv --layout=tidy
 >
@@ -321,6 +385,16 @@ $ hledger -f bcexample.hledger bal -T assets.*etrade -3 -O csv --layout=tidy
 "Assets:US:ETrade","2012-01-01..2014-10-11","2012-01-01","2014-10-11","VHT","294.00"
 >=0
 
+$ hledger -f bcexample.hledger bal -T assets.*etrade -3 -O tsv --layout=tidy
+>
+account	period	start_date	end_date	commodity	value
+Assets:US:ETrade	2012-01-01..2014-10-11	2012-01-01	2014-10-11	GLD	70.00
+Assets:US:ETrade	2012-01-01..2014-10-11	2012-01-01	2014-10-11	ITOT	17.00
+Assets:US:ETrade	2012-01-01..2014-10-11	2012-01-01	2014-10-11	USD	5120.50
+Assets:US:ETrade	2012-01-01..2014-10-11	2012-01-01	2014-10-11	VEA	36.00
+Assets:US:ETrade	2012-01-01..2014-10-11	2012-01-01	2014-10-11	VHT	294.00
+>=0
+
 <
 2021-01-01 Test
     Assets:Bank           INR 1.00

hledger/test/register/csv.test

@@ -8,3 +8,9 @@ $ hledger -f- register -O csv
 "txnidx","date","code","description","account","amount","total"
 "1","2019-01-01","","","(a)","10000000.0","10000000.0"
 >=
+
+# ** 2. Tsv output will not display thousands separators
+$ hledger -f- register -O tsv
+txnidx	date	code	description	account	amount	total
+1	2019-01-01			(a)	10000000.0	10000000.0
+>=