From 669fa706c0bde55e4e8174acc3d09351c2cd3850 Mon Sep 17 00:00:00 2001 From: Simon Michael Date: Fri, 15 Sep 2017 09:55:17 -0700 Subject: [PATCH] print: --new shows only transactions added since last time First cut, error messages could be refined etc. --- hledger-lib/Hledger/Read.hs | 72 +++++++++++++++++++++++++-- hledger-lib/Hledger/Read/Common.hs | 4 +- hledger/Hledger/Cli/Commands/Print.hs | 14 +++--- hledger/doc/commands.m4.md | 45 +++++++++++------ 4 files changed, 108 insertions(+), 27 deletions(-) diff --git a/hledger-lib/Hledger/Read.hs b/hledger-lib/Hledger/Read.hs index c961ffc11..7721bbdc9 100644 --- a/hledger-lib/Hledger/Read.hs +++ b/hledger-lib/Hledger/Read.hs @@ -42,18 +42,20 @@ import qualified Control.Exception as C import Control.Monad.Except import Data.List import Data.Maybe +import Data.Ord import Data.Text (Text) import qualified Data.Text as T +import Data.Time (Day) import Safe import System.Directory (doesFileExist, getHomeDirectory) import System.Environment (getEnv) import System.Exit (exitFailure) -import System.FilePath ((), takeExtension) -import System.IO (stderr) +import System.FilePath +import System.IO import Test.HUnit import Text.Printf -import Hledger.Data.Dates (getCurrentDay) +import Hledger.Data.Dates (getCurrentDay, parsedate, showDate) import Hledger.Data.Types import Hledger.Read.Common import qualified Hledger.Read.JournalReader as JournalReader @@ -259,7 +261,7 @@ tryReaders readers mrulesfile assrt path t = firstSuccessOrFirstError [] readers path' = fromMaybe "(string)" path ---- New versions of readJournal* with easier arguments +--- New versions of readJournal* with easier arguments, and --new/last-seen handling. readJournalFilesWithOpts :: InputOpts -> [FilePath] -> IO (Either String Journal) readJournalFilesWithOpts iopts = @@ -275,7 +277,67 @@ readJournalFileWithOpts iopts prefixedfile = do (mfmt, f) = splitReaderPrefix prefixedfile iopts' = iopts{mformat_=firstJust [mfmt, mformat_ iopts]} requireJournalFileExists f - readFileOrStdinAnyLineEnding f >>= readJournalWithOpts iopts' (Just f) + t <- readFileOrStdinAnyLineEnding f + ej <- readJournalWithOpts iopts' (Just f) t + case ej of + Left e -> return $ Left e + Right j | new_ iopts -> do + lastdates <- lastSeen f + let (newj, newlastdates) = journalFilterSinceLastDates lastdates j + when (not $ null newlastdates) $ saveLastSeen newlastdates f + return $ Right newj + Right j -> return $ Right j + +-- | Given zero or more date values (all the same, representing the +-- latest previously seen transaction date, and how many transactions +-- were seen on that date), remove transactions with earlier dates +-- from the journal, and the same number of transactions on the +-- latest date, if any, leaving only transactions that we can assume +-- are newer. Also returns the new last dates of the new journal. +journalFilterSinceLastDates :: [Day] -> Journal -> (Journal, [Day]) +journalFilterSinceLastDates [] j = (j, latestDates $ map tdate $ jtxns j) +journalFilterSinceLastDates ds@(d:_) j = (j', ds') + where + samedateorlaterts = filter ((>= d).tdate) $ jtxns j + (samedatets, laterts) = span ((== d).tdate) $ sortBy (comparing tdate) samedateorlaterts + newsamedatets = drop (length ds) samedatets + j' = j{jtxns=newsamedatets++laterts} + ds' = latestDates $ map tdate $ samedatets++laterts + +-- | Get all instances of the latest date in an unsorted list of dates. +-- Ie, if the latest date appears once, return it in a one-element list, +-- if it appears three times (anywhere), return three of it. +latestDates :: [Day] -> [Day] +latestDates = headDef [] . take 1 . group . reverse . sort + +-- | Where to save last-seen transactions info for the given file path +-- (.FILE.seen). +seenFileFor :: FilePath -> FilePath +seenFileFor f = dir fname' <.> "seen" + where + (dir, fname) = splitFileName f + fname' | "." `isPrefixOf` fname = fname + | otherwise = '.':fname + +-- | What were the latest transaction dates seen the last time this +-- journal file was read ? If there were multiple transactions on the +-- latest date, that number of dates is returned, otherwise just one. +-- Or none if no transactions were seen. +lastSeen :: FilePath -> IO [Day] +lastSeen f = do + let seenfile = seenFileFor f + exists <- doesFileExist seenfile + if exists + then map (parsedate . strip) . lines . strip . T.unpack <$> readFileStrictly seenfile + else return [] + +readFileStrictly :: FilePath -> IO Text +readFileStrictly f = readFile' f >>= \t -> C.evaluate (T.length t) >> return t + +-- | Remember that these transaction dates were the latest seen when +-- reading this journal file. +saveLastSeen :: [Day] -> FilePath -> IO () +saveLastSeen dates f = writeFile (seenFileFor f) $ unlines $ map showDate dates readJournalWithOpts :: InputOpts -> Maybe FilePath -> Text -> IO (Either String Journal) readJournalWithOpts iopts mfile txt = diff --git a/hledger-lib/Hledger/Read/Common.hs b/hledger-lib/Hledger/Read/Common.hs index ea3aa4946..435dc8995 100644 --- a/hledger-lib/Hledger/Read/Common.hs +++ b/hledger-lib/Hledger/Read/Common.hs @@ -55,13 +55,14 @@ data InputOpts = InputOpts { ,aliases_ :: [String] -- ^ account name aliases to apply ,anon_ :: Bool -- ^ do light anonymisation/obfuscation of the data ,ignore_assertions_ :: Bool -- ^ don't check balance assertions + ,new_ :: Bool -- ^ read only new transactions since this file was last read ,pivot_ :: String -- ^ use the given field's value as the account name } deriving (Show, Data) --, Typeable) instance Default InputOpts where def = definputopts definputopts :: InputOpts -definputopts = InputOpts def def def def def def +definputopts = InputOpts def def def def def def def rawOptsToInputOpts :: RawOpts -> InputOpts rawOptsToInputOpts rawopts = InputOpts{ @@ -71,6 +72,7 @@ rawOptsToInputOpts rawopts = InputOpts{ ,aliases_ = map (T.unpack . stripquotes . T.pack) $ listofstringopt "alias" rawopts ,anon_ = boolopt "anon" rawopts ,ignore_assertions_ = boolopt "ignore-assertions" rawopts + ,new_ = boolopt "new" rawopts ,pivot_ = stringopt "pivot" rawopts } diff --git a/hledger/Hledger/Cli/Commands/Print.hs b/hledger/Hledger/Cli/Commands/Print.hs index 62f169eee..a152b985c 100644 --- a/hledger/Hledger/Cli/Commands/Print.hs +++ b/hledger/Hledger/Cli/Commands/Print.hs @@ -31,13 +31,13 @@ printmode = (defCommandMode $ ["print"] ++ aliases) { modeHelp = "show transaction journal entries, sorted by date. With --date2, sort by secondary date instead." `withAliases` aliases ,modeGroupFlags = Group { groupUnnamed = [ - let matcharg = "STR" - in - flagReq ["match","m"] (\s opts -> Right $ setopt "match" s opts) matcharg - ("show the transaction whose description is most similar to "++matcharg - ++ ", and is most recent"), - flagNone ["explicit","x"] (setboolopt "explicit") - "show all amounts explicitly" + let arg = "STR" in + flagReq ["match","m"] (\s opts -> Right $ setopt "match" s opts) arg + ("show the transaction whose description is most similar to "++arg++", and is most recent") + ,flagNone ["explicit","x"] (setboolopt "explicit") + "show all amounts explicitly" + ,flagNone ["new"] (setboolopt "new") + "show only more recent transactions added to each file since last run" ] ++ outputflags ,groupHidden = [] diff --git a/hledger/doc/commands.m4.md b/hledger/doc/commands.m4.md index b101af352..5ead7fbd2 100644 --- a/hledger/doc/commands.m4.md +++ b/hledger/doc/commands.m4.md @@ -464,12 +464,15 @@ Print all market prices from the journal. ## print Show transactions from the journal. Aliases: p, txns. -`-x --explicit` -: show all amounts explicitly - `-m STR --match=STR ` : show the transaction whose description is most similar to STR, and is most recent +` --new` +: show only more recent transactions added to each file since last run + +`-x --explicit` +: show all amounts explicitly + `-O FMT --output-format=FMT ` : select the output format. Supported formats: txt, csv. @@ -501,22 +504,36 @@ $ hledger print assets:bank:checking $-1 ``` -The print command displays full journal entries (transactions) from the journal file, tidily formatted. +The print command displays full journal entries (transactions) from the journal file in date order, tidily formatted. +print's output is always a valid [hledger journal](/journal.html). +It preserves all transaction information, but it does not preserve directives or inter-transaction comments -As of hledger 1.2, print's output is always a valid [hledger journal](/journal.html). -However it may not preserve all original content, eg it does not print directives or inter-transaction comments. - -Normally, transactions' implicit/explicit amount style is preserved: -when an amount is omitted in the journal, it will be omitted in the output. -You can use the `-x/--explicit` flag to make all amounts explicit, which can be +Normally, the journal entry's explicit or implicit amount style is preserved. +Ie when an amount is omitted in the journal, it will be omitted in the output. +You can use the `-x`/`--explicit` flag to make all amounts explicit, which can be useful for troubleshooting or for making your journal more readable and robust against data entry errors. -Note, in this mode postings with a multi-commodity amount -(possible with an implicit amount in a multi-commodity transaction) +Note, `-x` will cause postings with a multi-commodity amount +(these can arise when a multi-commodity transaction has an implicit amount) will be split into multiple single-commodity postings, for valid journal output. -With -B/--cost, amounts with [transaction prices](/journal.html#transaction-prices) -are converted to cost (using the transaction price). +With `-B`/`--cost`, amounts with [transaction prices](/journal.html#transaction-prices) +are converted to cost using that price. + +With `-m`/`--match` and a STR argument, print will show at most one transaction: the one +one whose description is most similar to STR, and is most recent. STR should contain at +least two characters. If there is no similar-enough match, no transaction will be shown. + +With `--new`, for each FILE being read, hledger reads (and writes) a special .FILE.seen file in the same directory, +containing the latest transaction date(s) that were seen last time FILE was read. +When this file is found, only transactions with newer dates (and new transactions on the latest date) are printed. +This is useful for ignoring already-seen entries in import data, such as downloaded CSV files. +Eg: +```console +$ hledger -f bank1.csv print --new +# shows transactions added since last print --new on this file +``` +It assumes that only same-or-newer-dated transactions are added to FILE, and that the order of same-date transactions remains stable. The print command also supports [output destination](#output-destination)