slaesvuo/hledger
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cli: import command! hledger import *.csv adds new txns to main file

Browse Source
This commit is contained in:
Simon Michael 2017-09-17 18:57:42 -07:00
parent 0720cf9341
commit 7edacdf443
13 changed files with 293 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hledger-lib/Hledger/Read.hs
View File

@ -284,7 +284,7 @@ readJournalFileWithOpts iopts prefixedfile = do
Right j | new_ iopts -> do
ds <- previousLatestDates f
let (newj, newds) = journalFilterSinceLatestDates ds j
when (not $ null newds) $ saveLatestDates newds f
when (new_save_ iopts && not (null newds)) $ saveLatestDates newds f
return $ Right newj
Right j -> return $ Right j

4
hledger-lib/Hledger/Read/Common.hs
View File

@ -56,13 +56,14 @@ data InputOpts = InputOpts {
,anon_ :: Bool -- ^ do light anonymisation/obfuscation of the data
,ignore_assertions_ :: Bool -- ^ don't check balance assertions
,new_ :: Bool -- ^ read only new transactions since this file was last read
,new_save_ :: Bool -- ^ save latest new transactions state for next time
,pivot_ :: String -- ^ use the given field's value as the account name
} deriving (Show, Data) --, Typeable)
instance Default InputOpts where def = definputopts
definputopts :: InputOpts
definputopts = InputOpts def def def def def def def
definputopts = InputOpts def def def def def def True def
rawOptsToInputOpts :: RawOpts -> InputOpts
rawOptsToInputOpts rawopts = InputOpts{
@ -73,6 +74,7 @@ rawOptsToInputOpts rawopts = InputOpts{
,anon_ = boolopt "anon" rawopts
,ignore_assertions_ = boolopt "ignore-assertions" rawopts
,new_ = boolopt "new" rawopts
,new_save_ = True
,pivot_ = stringopt "pivot" rawopts
}

7
hledger-lib/doc/hledger_csv.5
View File

@ -247,6 +247,13 @@ If the CSV includes a running balance, you can assign that to the
\f[C]balance\f[] pseudo field; whenever the running balance value is
non\-empty, it will be asserted as the balance after the
\f[C]account1\f[] posting.
.SS Reading multiple CSV files
.PP
You can read multiple CSV files at once using multiple \f[C]\-f\f[]
arguments on the command line, and hledger will look for a
correspondingly\-named rules file for each.
Note if you use the \f[C]\-\-rules\-file\f[] option, this one rules file
will be used for all the CSV files being read.
.SH "REPORTING BUGS"

32
hledger-lib/doc/hledger_csv.5.info
View File

@ -198,6 +198,7 @@ File: hledger_csv.5.info, Node: CSV TIPS, Prev: CSV RULES, Up: Top
* CSV accounts::
* CSV amounts::
* CSV balance assertions::
* Reading multiple CSV files::

File: hledger_csv.5.info, Node: CSV ordering, Next: CSV accounts, Up: CSV TIPS
@ -246,7 +247,7 @@ fields (giving more control, eg to put the currency symbol on the
right).

File: hledger_csv.5.info, Node: CSV balance assertions, Prev: CSV amounts, Up: CSV TIPS
File: hledger_csv.5.info, Node: CSV balance assertions, Next: Reading multiple CSV files, Prev: CSV amounts, Up: CSV TIPS
2.4 CSV balance assertions
==========================
@ -255,6 +256,17 @@ If the CSV includes a running balance, you can assign that to the
'balance' pseudo field; whenever the running balance value is non-empty,
it will be asserted as the balance after the 'account1' posting.

File: hledger_csv.5.info, Node: Reading multiple CSV files, Prev: CSV balance assertions, Up: CSV TIPS
2.5 Reading multiple CSV files
==============================
You can read multiple CSV files at once using multiple '-f' arguments on
the command line, and hledger will look for a correspondingly-named
rules file for each. Note if you use the '--rules-file' option, this
one rules file will be used for all the CSV files being read.

Tag Table:
Node: Top74
@ -276,13 +288,15 @@ Node: newest-first5011
Ref: #newest-first5127
Node: CSV TIPS5538
Ref: #csv-tips5634
Node: CSV ordering5721
Ref: #csv-ordering5841
Node: CSV accounts6022
Ref: #csv-accounts6162
Node: CSV amounts6416
Ref: #csv-amounts6564
Node: CSV balance assertions7339
Ref: #csv-balance-assertions7488
Node: CSV ordering5752
Ref: #csv-ordering5872
Node: CSV accounts6053
Ref: #csv-accounts6193
Node: CSV amounts6447
Ref: #csv-amounts6595
Node: CSV balance assertions7370
Ref: #csv-balance-assertions7554
Node: Reading multiple CSV files7759
Ref: #reading-multiple-csv-files7931

End Tag Table

10
hledger-lib/doc/hledger_csv.5.txt
View File

@ -169,10 +169,16 @@ CSV TIPS
ance pseudo field; whenever the running balance value is non-empty, it
will be asserted as the balance after the account1 posting.
Reading multiple CSV files
You can read multiple CSV files at once using multiple -f arguments on
the command line, and hledger will look for a correspondingly-named
rules file for each. Note if you use the --rules-file option, this one
rules file will be used for all the CSV files being read.
REPORTING BUGS
Report bugs at http://bugs.hledger.org (or on the #hledger IRC channel
Report bugs at http://bugs.hledger.org (or on the #hledger IRC channel
or hledger mail list)
@ -186,7 +192,7 @@ COPYRIGHT
SEE ALSO
hledger(1), hledger-ui(1), hledger-web(1), hledger-api(1),
hledger(1), hledger-ui(1), hledger-web(1), hledger-api(1),
hledger_csv(5), hledger_journal(5), hledger_timeclock(5), hledger_time-
dot(5), ledger(1)

3
hledger/Hledger/Cli/Commands.hs
View File

@ -22,6 +22,7 @@ module Hledger.Cli.Commands (
,module Hledger.Cli.Commands.Checkdupes
,module Hledger.Cli.Commands.Equity
,module Hledger.Cli.Commands.Help
,module Hledger.Cli.Commands.Import
,module Hledger.Cli.Commands.Incomestatement
,module Hledger.Cli.Commands.Prices
,module Hledger.Cli.Commands.Print
@ -60,6 +61,7 @@ import Hledger.Cli.Commands.Checkdates
import Hledger.Cli.Commands.Checkdupes
import Hledger.Cli.Commands.Equity
import Hledger.Cli.Commands.Help
import Hledger.Cli.Commands.Import
import Hledger.Cli.Commands.Incomestatement
import Hledger.Cli.Commands.Prices
import Hledger.Cli.Commands.Print
@ -86,6 +88,7 @@ builtinCommands = [
,(checkdupesmode , checkdupes)
,(equitymode , equity)
,(helpmode , help')
,(importmode , importcmd)
,(incomestatementmode , incomestatement)
,(pricesmode , prices)
,(printmode , print')

58
hledger/Hledger/Cli/Commands/Import.hs Executable file
View File

@ -0,0 +1,58 @@
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE QuasiQuotes #-}
module Hledger.Cli.Commands.Import (
importmode
,importcmd
)
where
import Control.Monad
import Data.String.Here
import Hledger
import Hledger.Cli.CliOptions
import Hledger.Cli.Commands.Add (journalAddTransaction)
-- import Hledger.Cli.Commands.Print (print')
import System.Console.CmdArgs.Explicit
import Text.Printf
importmode = hledgerCommandMode
[here| import
Read new transactions added to each FILE since last run, and add them to
the main journal file. Or with --dry-run, just print the transactions
that would be added.
Input files are provided as arguments, or glob patterns. So eg to add new
transactions from all CSV files to the main journal: hledger import *.csv
New transactions are detected like print --new (using .latest.FILE state files)
FLAGS
|]
[flagNone ["dry-run"] (\opts -> setboolopt "dry-run" opts) "just show the transactions to be imported"]
[generalflagsgroup1]
[]
([], Just $ argsFlag "FILE [...]")
importcmd opts@CliOpts{rawopts_=rawopts,inputopts_=iopts} j = do
let
inputfiles = listofstringopt "args" rawopts
dryrun = boolopt "dry-run" rawopts
iopts' = iopts{new_=True, new_save_=not dryrun}
case inputfiles of
[] -> error' "please provide one or more input files as arguments"
fs -> do
enewj <- readJournalFilesWithOpts iopts' fs
case enewj of
Left e -> error' e
Right newj ->
case jtxns newj of
[] -> putStrLn "no new transactions"
newts | dryrun -> do
printf "would import %d new transactions:\n\n" (length newts)
-- TODO how to force output here ?
-- length (jtxns newj) `seq` print' opts{rawopts_=("explicit",""):rawopts} newj
mapM_ (putStr . showTransactionUnelided) newts
newts -> do
foldM (flip journalAddTransaction opts) j newts -- gets forced somehow.. (how ?)
printf "imported %d new transactions\n" (length newts)

18
hledger/doc/commands.m4.md
View File

@ -388,6 +388,19 @@ DESCRIPTION
...
}})
## import
Read new transactions added to each FILE since last run, and add them to
the main journal file.
`--dry-run`
: just show the transactions to be imported
Input files are provided as arguments, or glob patterns. So eg to add new
transactions from all CSV files to the main journal: hledger import *.csv
New transactions are detected like print --new (using .latest.FILE state files).
## incomestatement
Show an income statement. Alias: is.
@ -525,7 +538,7 @@ one whose description is most similar to STR, and is most recent. STR should con
least two characters. If there is no similar-enough match, no transaction will be shown.
With `--new`, for each FILE being read, hledger reads (and writes) a special
.latest.FILE file in the same directory, containing the latest transaction date(s)
state file (`.latest.FILE` in the same directory), containing the latest transaction date(s)
that were seen last time FILE was read. When this file is found, only transactions
with newer dates (and new transactions on the latest date) are printed.
This is useful for ignoring already-seen entries in import data, such as downloaded CSV files.
@ -535,7 +548,8 @@ $ hledger -f bank1.csv print --new
# shows transactions added since last print --new on this file
```
This assumes that transactions added to FILE always have same or increasing dates,
and that transactions on the same day do not get reordered.
and that transactions on the same day do not get reordered.
See also the [import](#import) command.
The print command also supports
[output destination](#output-destination)

22
hledger/doc/hledger.1
View File

@ -1951,6 +1951,22 @@ DESCRIPTION
\&...
\f[]
.fi
.SS import
.PP
Read new transactions added to each FILE since last run, and add them to
the main journal file.
.TP
.B \f[C]\-\-dry\-run\f[]
just show the transactions to be imported
.RS
.RE
.PP
Input files are provided as arguments, or glob patterns.
So eg to add new transactions from all CSV files to the main journal:
hledger import *.csv
.PP
New transactions are detected like print \-\-new (using .latest.FILE
state files).
.SS incomestatement
.PP
Show an income statement.
@ -2140,8 +2156,9 @@ STR should contain at least two characters.
If there is no similar\-enough match, no transaction will be shown.
.PP
With \f[C]\-\-new\f[], for each FILE being read, hledger reads (and
writes) a special .latest.FILE file in the same directory, containing
the latest transaction date(s) that were seen last time FILE was read.
writes) a special state file (\f[C]\&.latest.FILE\f[] in the same
directory), containing the latest transaction date(s) that were seen
last time FILE was read.
When this file is found, only transactions with newer dates (and new
transactions on the latest date) are printed.
This is useful for ignoring already\-seen entries in import data, such
@ -2158,6 +2175,7 @@ $\ hledger\ \-f\ bank1.csv\ print\ \-\-new
This assumes that transactions added to FILE always have same or
increasing dates, and that transactions on the same day do not get
reordered.
See also the import command.
.PP
The print command also supports output destination and CSV output.
Here\[aq]s an example of print\[aq]s CSV output:

238
hledger/doc/hledger.1.info
View File

@ -800,6 +800,7 @@ detailed command help.
* check-dupes::
* equity::
* help::
* import::
* incomestatement::
* prices::
* print::
@ -1520,7 +1521,7 @@ balances to zero and back. Can be useful for bringing account balances
across file boundaries.

File: hledger.1.info, Node: help, Next: incomestatement, Prev: equity, Up: COMMANDS
File: hledger.1.info, Node: help, Next: import, Prev: equity, Up: COMMANDS
4.11 help
=========
@ -1557,9 +1558,29 @@ DESCRIPTION
...

File: hledger.1.info, Node: incomestatement, Next: prices, Prev: help, Up: COMMANDS
File: hledger.1.info, Node: import, Next: incomestatement, Prev: help, Up: COMMANDS
4.12 incomestatement
4.12 import
===========
Read new transactions added to each FILE since last run, and add them to
the main journal file.
'--dry-run'
just show the transactions to be imported
Input files are provided as arguments, or glob patterns. So eg to
add new transactions from all CSV files to the main journal: hledger
import *.csv
New transactions are detected like print -new (using .latest.FILE
state files).

File: hledger.1.info, Node: incomestatement, Next: prices, Prev: import, Up: COMMANDS
4.13 incomestatement
====================
Show an income statement. Alias: is.
@ -1635,7 +1656,7 @@ report mode with '--change'/'--cumulative'/'--historical'.

File: hledger.1.info, Node: prices, Next: print, Prev: incomestatement, Up: COMMANDS
4.13 prices
4.14 prices
===========
Print all market prices from the journal.
@ -1643,7 +1664,7 @@ Print all market prices from the journal.

File: hledger.1.info, Node: print, Next: print-unique, Prev: prices, Up: COMMANDS
4.14 print
4.15 print
==========
Show transactions from the journal. Aliases: p, txns.
@ -1713,18 +1734,19 @@ is most recent. STR should contain at least two characters. If there
is no similar-enough match, no transaction will be shown.
With '--new', for each FILE being read, hledger reads (and writes) a
special .latest.FILE file in the same directory, containing the latest
transaction date(s) that were seen last time FILE was read. When this
file is found, only transactions with newer dates (and new transactions
on the latest date) are printed. This is useful for ignoring
already-seen entries in import data, such as downloaded CSV files. Eg:
special state file ('.latest.FILE' in the same directory), containing
the latest transaction date(s) that were seen last time FILE was read.
When this file is found, only transactions with newer dates (and new
transactions on the latest date) are printed. This is useful for
ignoring already-seen entries in import data, such as downloaded CSV
files. Eg:
$ hledger -f bank1.csv print --new
# shows transactions added since last print --new on this file
This assumes that transactions added to FILE always have same or
increasing dates, and that transactions on the same day do not get
reordered.
reordered. See also the import command.
The print command also supports output destination and CSV output.
Here's an example of print's CSV output:
@ -1759,7 +1781,7 @@ $ hledger print -Ocsv

File: hledger.1.info, Node: print-unique, Next: register, Prev: print, Up: COMMANDS
4.15 print-unique
4.16 print-unique
=================
Print transactions which do not reuse an already-seen description.
@ -1767,7 +1789,7 @@ Print transactions which do not reuse an already-seen description.

File: hledger.1.info, Node: register, Next: register-match, Prev: print-unique, Up: COMMANDS
4.16 register
4.17 register
=============
Show postings and their running total. Aliases: r, reg.
@ -1872,7 +1894,7 @@ length and comparable to the others in the report.

File: hledger.1.info, Node: Custom register output, Up: register
4.16.1 Custom register output
4.17.1 Custom register output
-----------------------------
register uses the full terminal width by default, except on windows.
@ -1904,7 +1926,7 @@ output.

File: hledger.1.info, Node: register-match, Next: rewrite, Prev: register, Up: COMMANDS
4.17 register-match
4.18 register-match
===================
Print the one posting whose transaction description is closest to DESC,
@ -1914,7 +1936,7 @@ already-seen transactions when importing.

File: hledger.1.info, Node: rewrite, Next: stats, Prev: register-match, Up: COMMANDS
4.18 rewrite
4.19 rewrite
============
Print all transactions, adding custom postings to the matched ones.
@ -1922,7 +1944,7 @@ Print all transactions, adding custom postings to the matched ones.

File: hledger.1.info, Node: stats, Next: tags, Prev: rewrite, Up: COMMANDS
4.19 stats
4.20 stats
==========
Show some journal statistics.
@ -1954,7 +1976,7 @@ output destination.

File: hledger.1.info, Node: tags, Next: test, Prev: stats, Up: COMMANDS
4.20 tags
4.21 tags
=========
List all the tag names in use.
@ -1962,7 +1984,7 @@ List all the tag names in use.

File: hledger.1.info, Node: test, Prev: tags, Up: COMMANDS
4.21 test
4.22 test
=========
Run built-in unit tests.
@ -2192,93 +2214,95 @@ Node: QUERIES21498
Ref: #queries21602
Node: COMMANDS25569
Ref: #commands25683
Node: accounts26655
Ref: #accounts26755
Node: activity27748
Ref: #activity27860
Node: add28219
Ref: #add28320
Node: balance30978
Ref: #balance31091
Node: Flat mode34111
Ref: #flat-mode34238
Node: Depth limited balance reports34658
Ref: #depth-limited-balance-reports34861
Node: Multicolumn balance reports35281
Ref: #multicolumn-balance-reports35492
Node: Custom balance output40140
Ref: #custom-balance-output40324
Node: Colour support42417
Ref: #colour-support42578
Node: Output destination42751
Ref: #output-destination42909
Node: CSV output43179
Ref: #csv-output43298
Node: balancesheet43695
Ref: #balancesheet43833
Node: balancesheetequity45740
Ref: #balancesheetequity45891
Node: cashflow46680
Ref: #cashflow46810
Node: check-dates48661
Ref: #check-dates48790
Node: check-dupes48907
Ref: #check-dupes49034
Node: equity49171
Ref: #equity49283
Node: help49446
Ref: #help49558
Node: incomestatement50632
Ref: #incomestatement50766
Node: prices52658
Ref: #prices52775
Node: print52818
Ref: #print52930
Node: print-unique57737
Ref: #print-unique57865
Node: register57933
Ref: #register58062
Node: Custom register output62563
Ref: #custom-register-output62694
Node: register-match63991
Ref: #register-match64127
Node: rewrite64310
Ref: #rewrite64429
Node: stats64498
Ref: #stats64603
Node: tags65484
Ref: #tags65584
Node: test65616
Ref: #test65702
Node: ADD-ON COMMANDS66070
Ref: #add-on-commands66182
Node: Official add-ons67469
Ref: #official-add-ons67611
Node: api67698
Ref: #api67789
Node: ui67841
Ref: #ui67942
Node: web68000
Ref: #web68091
Node: Third party add-ons68137
Ref: #third-party-add-ons68314
Node: diff68449
Ref: #diff68548
Node: iadd68647
Ref: #iadd68763
Node: interest68846
Ref: #interest68969
Node: irr69064
Ref: #irr69164
Node: Experimental add-ons69242
Ref: #experimental-add-ons69396
Node: autosync69687
Ref: #autosync69801
Node: budget70040
Ref: #budget70164
Node: chart70230
Ref: #chart70349
Node: check70420
Ref: #check70524
Node: accounts26666
Ref: #accounts26766
Node: activity27759
Ref: #activity27871
Node: add28230
Ref: #add28331
Node: balance30989
Ref: #balance31102
Node: Flat mode34122
Ref: #flat-mode34249
Node: Depth limited balance reports34669
Ref: #depth-limited-balance-reports34872
Node: Multicolumn balance reports35292
Ref: #multicolumn-balance-reports35503
Node: Custom balance output40151
Ref: #custom-balance-output40335
Node: Colour support42428
Ref: #colour-support42589
Node: Output destination42762
Ref: #output-destination42920
Node: CSV output43190
Ref: #csv-output43309
Node: balancesheet43706
Ref: #balancesheet43844
Node: balancesheetequity45751
Ref: #balancesheetequity45902
Node: cashflow46691
Ref: #cashflow46821
Node: check-dates48672
Ref: #check-dates48801
Node: check-dupes48918
Ref: #check-dupes49045
Node: equity49182
Ref: #equity49294
Node: help49457
Ref: #help49560
Node: import50634
Ref: #import50750
Node: incomestatement51145
Ref: #incomestatement51281
Node: prices53173
Ref: #prices53290
Node: print53333
Ref: #print53445
Node: print-unique58291
Ref: #print-unique58419
Node: register58487
Ref: #register58616
Node: Custom register output63117
Ref: #custom-register-output63248
Node: register-match64545
Ref: #register-match64681
Node: rewrite64864
Ref: #rewrite64983
Node: stats65052
Ref: #stats65157
Node: tags66038
Ref: #tags66138
Node: test66170
Ref: #test66256
Node: ADD-ON COMMANDS66624
Ref: #add-on-commands66736
Node: Official add-ons68023
Ref: #official-add-ons68165
Node: api68252
Ref: #api68343
Node: ui68395
Ref: #ui68496
Node: web68554
Ref: #web68645
Node: Third party add-ons68691
Ref: #third-party-add-ons68868
Node: diff69003
Ref: #diff69102
Node: iadd69201
Ref: #iadd69317
Node: interest69400
Ref: #interest69523
Node: irr69618
Ref: #irr69718
Node: Experimental add-ons69796
Ref: #experimental-add-ons69950
Node: autosync70241
Ref: #autosync70355
Node: budget70594
Ref: #budget70718
Node: chart70784
Ref: #chart70903
Node: check70974
Ref: #check71078

End Tag Table

27
hledger/doc/hledger.1.txt
View File

@ -1374,6 +1374,20 @@ COMMANDS
hledger is a cross-platform program for tracking money, time, or any
...
import
Read new transactions added to each FILE since last run, and add them
to the main journal file.
--dry-run
just show the transactions to be imported
Input files are provided as arguments, or glob patterns. So eg to add
new transactions from all CSV files to the main journal: hledger import
*.csv
New transactions are detected like print --new (using .latest.FILE
state files).
incomestatement
Show an income statement. Alias: is.
@ -1511,18 +1525,19 @@ COMMANDS
no similar-enough match, no transaction will be shown.
With --new, for each FILE being read, hledger reads (and writes) a spe-
cial .latest.FILE file in the same directory, containing the latest
transaction date(s) that were seen last time FILE was read. When this
file is found, only transactions with newer dates (and new transactions
on the latest date) are printed. This is useful for ignoring
already-seen entries in import data, such as downloaded CSV files. Eg:
cial state file (.latest.FILE in the same directory), containing the
latest transaction date(s) that were seen last time FILE was read.
When this file is found, only transactions with newer dates (and new
transactions on the latest date) are printed. This is useful for
ignoring already-seen entries in import data, such as downloaded CSV
files. Eg:
$ hledger -f bank1.csv print --new
# shows transactions added since last print --new on this file
This assumes that transactions added to FILE always have same or
increasing dates, and that transactions on the same day do not get
reordered.
reordered. See also the import command.
The print command also supports output destination and CSV output.
Here's an example of print's CSV output:

1
hledger/hledger.cabal
View File

@ -134,6 +134,7 @@ library
Hledger.Cli.Commands.Checkdupes
Hledger.Cli.Commands.Equity
Hledger.Cli.Commands.Help
Hledger.Cli.Commands.Import
Hledger.Cli.Commands.Incomestatement
Hledger.Cli.Commands.Prices
Hledger.Cli.Commands.Print

1
hledger/package.yaml
View File

@ -114,6 +114,7 @@ library:
- Hledger.Cli.Commands.Checkdupes
- Hledger.Cli.Commands.Equity
- Hledger.Cli.Commands.Help
- Hledger.Cli.Commands.Import
- Hledger.Cli.Commands.Incomestatement
- Hledger.Cli.Commands.Prices
- Hledger.Cli.Commands.Print