From c07ad29a87260359b77cb5c87b03dc678c415a96 Mon Sep 17 00:00:00 2001 From: Stephen Morgan Date: Mon, 23 Aug 2021 15:30:54 +1000 Subject: [PATCH] imp!: forecast: Implements more intuitive logic for the forecast interval. (#1648) The forecast period begins on: - the start date supplied to the `--forecast` argument, if present - otherwise, the later of - the report start date if specified with -b/-p/date: - the day after the latest normal (non-periodic) transaction in the journal, if any - otherwise today. It ends on: - the end date supplied to the `--forecast` argument, if present - otherwise the report end date if specified with -e/-p/date: - otherwise 180 days (6 months) from today. Note that the previous behaviour did not quite match the documentation, so this also acts as a bug fix for #1665. --- hledger-lib/Hledger/Read.hs | 2 + hledger-lib/Hledger/Read/Common.hs | 65 +++++++----------- hledger-lib/Hledger/Read/InputOptions.hs | 65 ++++++++++++------ hledger-ui/Hledger/UI/UIState.hs | 5 +- hledger/hledger.m4.md | 17 ++--- hledger/test/forecast.test | 86 +++++++++++++++++++----- 6 files changed, 152 insertions(+), 88 deletions(-) diff --git a/hledger-lib/Hledger/Read.hs b/hledger-lib/Hledger/Read.hs index 9180e8876..fa27f07e3 100644 --- a/hledger-lib/Hledger/Read.hs +++ b/hledger-lib/Hledger/Read.hs @@ -36,6 +36,7 @@ module Hledger.Read ( findReader, splitReaderPrefix, module Hledger.Read.Common, + module Hledger.Read.InputOptions, -- * Tests tests_Read, @@ -69,6 +70,7 @@ import System.IO (hPutStr, stderr) import Hledger.Data.Dates (getCurrentDay, parsedateM, showDate) import Hledger.Data.Types import Hledger.Read.Common +import Hledger.Read.InputOptions import Hledger.Read.JournalReader as JournalReader import Hledger.Read.CsvReader (tests_CsvReader) -- import Hledger.Read.TimedotReader (tests_TimedotReader) diff --git a/hledger-lib/Hledger/Read/Common.hs b/hledger-lib/Hledger/Read/Common.hs index de643b766..fa4dc1299 100644 --- a/hledger-lib/Hledger/Read/Common.hs +++ b/hledger-lib/Hledger/Read/Common.hs @@ -32,7 +32,6 @@ module Hledger.Read.Common ( InputOpts(..), definputopts, rawOptsToInputOpts, - forecastPeriodFromRawOpts, rawOptsToCommodityStylesOpts, -- * parsing utilities @@ -148,7 +147,7 @@ import qualified Data.Map as M import qualified Data.Semigroup as Sem import Data.Text (Text) import qualified Data.Text as T -import Data.Time.Calendar (Day, addDays, fromGregorianValid, toGregorian) +import Data.Time.Calendar (Day, fromGregorianValid, toGregorian) import Data.Time.Clock.POSIX (getPOSIXTime) import Data.Time.LocalTime (LocalTime(..), TimeOfDay(..)) import Data.Word (Word8) @@ -160,7 +159,7 @@ import Text.Megaparsec.Custom finalErrorBundlePretty, parseErrorAt, parseErrorAtRegion) import Hledger.Data -import Hledger.Query (Query(..), filterQuery, parseQueryTerm, queryEndDate, queryIsDate, simplifyQuery) +import Hledger.Query (Query(..), filterQuery, parseQueryTerm, queryEndDate, queryStartDate, queryIsDate, simplifyQuery) import Hledger.Reports.ReportOptions (ReportOpts(..), queryFromFlags, rawOptsToReportOpts) import Hledger.Utils import Text.Printf (printf) @@ -234,6 +233,14 @@ rawOptsToInputOpts :: RawOpts -> IO InputOpts rawOptsToInputOpts rawopts = do d <- getCurrentDay + let noinferprice = boolopt "strict" rawopts || stringopt "args" rawopts == "balancednoautoconversion" + + -- Do we really need to do all this work just to get the requested end date? This is duplicating + -- much of reportOptsToSpec. + ropts = rawOptsToReportOpts d rawopts + argsquery = lefts . rights . map (parseQueryTerm d) $ querystring_ ropts + datequery = simplifyQuery . filterQuery queryIsDate . And $ queryFromFlags ropts : argsquery + return InputOpts{ -- files_ = listofstringopt "file" rawopts mformat_ = Nothing @@ -244,6 +251,7 @@ rawOptsToInputOpts rawopts = do ,new_save_ = True ,pivot_ = stringopt "pivot" rawopts ,forecast_ = forecastPeriodFromRawOpts d rawopts + ,reportspan_ = DateSpan (queryStartDate False datequery) (queryEndDate False datequery) ,auto_ = boolopt "auto" rawopts ,balancingopts_ = balancingOpts{ ignore_assertions_ = boolopt "ignore-assertions" rawopts @@ -252,36 +260,21 @@ rawOptsToInputOpts rawopts = do } ,strict_ = boolopt "strict" rawopts } - where noinferprice = boolopt "strict" rawopts || stringopt "args" rawopts == "balancednoautoconversion" -- | Get the date span from --forecast's PERIODEXPR argument, if any. -- This will fail with a usage error if the period expression cannot be parsed, -- or if it contains a report interval. forecastPeriodFromRawOpts :: Day -> RawOpts -> Maybe DateSpan -forecastPeriodFromRawOpts d rawopts = case maybestringopt "forecast" rawopts of - Nothing -> Nothing - Just "" -> Just forecastspanDefault - Just arg -> - either - (\e -> usageError $ "could not parse forecast period : "++customErrorBundlePretty e) - (\(interval, requestedspan) -> - case interval of - NoInterval -> Just $ requestedspan `spanDefaultsFrom` forecastspanDefault - _ -> usageError $ unlines - [ "--forecast's argument should not contain a report interval" - , "(" ++ show interval ++ " in \"" ++ arg ++ "\")" - ]) - (parsePeriodExpr d $ stripquotes $ T.pack arg) +forecastPeriodFromRawOpts d rawopts = do + arg <- maybestringopt "forecast" rawopts + let period = parsePeriodExpr d . stripquotes $ T.pack arg + return $ if null arg then nulldatespan else either badParse (getSpan arg) period where - -- "They end on or before the specified report end date, or 180 days from today if unspecified." - mspecifiedend = dbg2 "specifieddates" $ queryEndDate False datequery - forecastendDefault = dbg2 "forecastendDefault" $ addDays 180 d - forecastspanDefault = DateSpan Nothing $ mspecifiedend <|> Just forecastendDefault - -- Do we really need to do all this work just to get the requested end date? This is duplicating - -- much of reportOptsToSpec. - ropts = rawOptsToReportOpts d rawopts - argsquery = lefts . rights . map (parseQueryTerm d) $ querystring_ ropts - datequery = simplifyQuery . filterQuery queryIsDate . And $ queryFromFlags ropts : argsquery + badParse e = usageError $ "could not parse forecast period : "++customErrorBundlePretty e + getSpan arg (interval, requestedspan) = case interval of + NoInterval -> requestedspan + _ -> usageError $ "--forecast's argument should not contain a report interval (" + ++ show interval ++ " in \"" ++ arg ++ "\")" --- ** parsing utilities @@ -371,7 +364,7 @@ parseAndFinaliseJournal' parser iopts f txt = do -- - infer transaction-implied market prices from transaction prices -- journalFinalise :: InputOpts -> FilePath -> Text -> ParsedJournal -> ExceptT String IO Journal -journalFinalise InputOpts{forecast_,auto_,balancingopts_,strict_} f txt pj = do +journalFinalise iopts@InputOpts{auto_,balancingopts_,strict_} f txt pj = do t <- liftIO getPOSIXTime d <- liftIO getCurrentDay -- Infer and apply canonical styles for each commodity (or throw an error). @@ -390,7 +383,7 @@ journalFinalise InputOpts{forecast_,auto_,balancingopts_,strict_} f txt pj = do journalCheckCommoditiesDeclared j -- Add forecast transactions if enabled - journalAddForecast d forecast_ j + journalAddForecast (forecastPeriod d iopts j) j -- Add auto postings if enabled & (if auto_ && not (null $ jtxnmodifiers j) then journalAddAutoPostings d balancingopts_ else pure) -- Balance all transactions and maybe check balance assertions. @@ -412,9 +405,9 @@ journalAddAutoPostings d bopts = -- -- The start & end date for generated periodic transactions are determined in -- a somewhat complicated way; see the hledger manual -> Periodic transactions. -journalAddForecast :: Day -> Maybe DateSpan -> Journal -> Journal -journalAddForecast _ Nothing j = j -journalAddForecast d (Just requestedspan) j = j{jtxns = jtxns j ++ forecasttxns} +journalAddForecast :: Maybe DateSpan -> Journal -> Journal +journalAddForecast Nothing j = j +journalAddForecast (Just forecastspan) j = j{jtxns = jtxns j ++ forecasttxns} where forecasttxns = map (txnTieKnot . transactionTransformPostings (postingApplyCommodityStyles $ journalCommodityStyles j)) @@ -422,14 +415,6 @@ journalAddForecast d (Just requestedspan) j = j{jtxns = jtxns j ++ forecasttxns} . concatMap (`runPeriodicTransaction` forecastspan) $ jperiodictxns j - -- "They can start no earlier than: the day following the latest normal transaction in the journal (or today if there are none)." - mjournalend = dbg2 "journalEndDate" $ journalEndDate False j -- ignore secondary dates - forecastbeginDefault = dbg2 "forecastbeginDefault" $ mjournalend <|> Just d - - -- "They end on or before the specified report end date, or 180 days from today if unspecified." - forecastspan = dbg2 "forecastspan" $ dbg2 "forecastspan flag" requestedspan - `spanDefaultsFrom` DateSpan forecastbeginDefault (Just $ addDays 180 d) - -- | Check that all the journal's transactions have payees declared with -- payee directives, returning an error message otherwise. journalCheckPayeesDeclared :: Journal -> Either String () diff --git a/hledger-lib/Hledger/Read/InputOptions.hs b/hledger-lib/Hledger/Read/InputOptions.hs index c99233c1c..ed515bb33 100644 --- a/hledger-lib/Hledger/Read/InputOptions.hs +++ b/hledger-lib/Hledger/Read/InputOptions.hs @@ -6,30 +6,36 @@ Similar to CliOptions.inputflags, simplifies the journal-reading functions. -} module Hledger.Read.InputOptions ( - -- * Types and helpers for input options - InputOpts(..) - , definputopts -) -where +-- * Types and helpers for input options + InputOpts(..) +, definputopts +, forecastPeriod +) where + +import Control.Applicative ((<|>)) +import Data.Time (Day, addDays) import Hledger.Data.Types -import Hledger.Data.Transaction -import Hledger.Data.Dates() +import Hledger.Data.Transaction (BalancingOpts(..), balancingOpts) +import Hledger.Data.Journal (journalEndDate) +import Hledger.Data.Dates (nulldatespan) +import Hledger.Utils data InputOpts = InputOpts { -- files_ :: [FilePath] - mformat_ :: Maybe StorageFormat -- ^ a file/storage format to try, unless overridden - -- by a filename prefix. Nothing means try all. - ,mrules_file_ :: Maybe FilePath -- ^ a conversion rules file to use (when reading CSV) - ,aliases_ :: [String] -- ^ account name aliases to apply - ,anon_ :: Bool -- ^ do light anonymisation/obfuscation of the data - ,new_ :: Bool -- ^ read only new transactions since this file was last read - ,new_save_ :: Bool -- ^ save latest new transactions state for next time - ,pivot_ :: String -- ^ use the given field's value as the account name - ,forecast_ :: Maybe DateSpan -- ^ span in which to generate forecast transactions - ,auto_ :: Bool -- ^ generate automatic postings when journal is parsed - ,balancingopts_ :: BalancingOpts -- ^ options for balancing transactions - ,strict_ :: Bool -- ^ do extra error checking (eg, all posted accounts are declared, no prices are inferred) + mformat_ :: Maybe StorageFormat -- ^ a file/storage format to try, unless overridden + -- by a filename prefix. Nothing means try all. + ,mrules_file_ :: Maybe FilePath -- ^ a conversion rules file to use (when reading CSV) + ,aliases_ :: [String] -- ^ account name aliases to apply + ,anon_ :: Bool -- ^ do light anonymisation/obfuscation of the data + ,new_ :: Bool -- ^ read only new transactions since this file was last read + ,new_save_ :: Bool -- ^ save latest new transactions state for next time + ,pivot_ :: String -- ^ use the given field's value as the account name + ,forecast_ :: Maybe DateSpan -- ^ span in which to generate forecast transactions + ,reportspan_ :: DateSpan -- ^ a dirty hack keeping the query dates in InputOpts. This rightfully lives in ReportSpec, but is duplicated here. + ,auto_ :: Bool -- ^ generate automatic postings when journal is parsed + ,balancingopts_ :: BalancingOpts -- ^ options for balancing transactions + ,strict_ :: Bool -- ^ do extra error checking (eg, all posted accounts are declared, no prices are inferred) } deriving (Show) definputopts :: InputOpts @@ -42,7 +48,28 @@ definputopts = InputOpts , new_save_ = True , pivot_ = "" , forecast_ = Nothing + , reportspan_ = nulldatespan , auto_ = False , balancingopts_ = balancingOpts , strict_ = False } + +-- | Get the Maybe the DateSpan to generate forecast options from. +-- This begins on: +-- - the start date supplied to the `--forecast` argument, if present +-- - otherwise, the later of +-- - the report start date if specified with -b/-p/date: +-- - the day after the latest normal (non-periodic) transaction in the journal, if any +-- - otherwise today. +-- It ends on: +-- - the end date supplied to the `--forecast` argument, if present +-- - otherwise the report end date if specified with -e/-p/date: +-- - otherwise 180 days (6 months) from today. +forecastPeriod :: Day -> InputOpts -> Journal -> Maybe DateSpan +forecastPeriod d iopts j = do + DateSpan requestedStart requestedEnd <- forecast_ iopts + let forecastStart = requestedStart <|> max mjournalend reportStart <|> Just d + forecastEnd = requestedEnd <|> reportEnd <|> Just (addDays 180 d) + mjournalend = dbg2 "journalEndDate" $ journalEndDate False j -- ignore secondary dates + DateSpan reportStart reportEnd = reportspan_ iopts + return . dbg2 "forecastspan" $ DateSpan forecastStart forecastEnd diff --git a/hledger-ui/Hledger/UI/UIState.hs b/hledger-ui/Hledger/UI/UIState.hs index ebe5e067c..27817a0f8 100644 --- a/hledger-ui/Hledger/UI/UIState.hs +++ b/hledger-ui/Hledger/UI/UIState.hs @@ -7,7 +7,6 @@ module Hledger.UI.UIState where import Brick.Widgets.Edit -import Control.Applicative ((<|>)) import Data.List ((\\), foldl', sort) import Data.Semigroup (Max(..)) import qualified Data.Text as T @@ -157,11 +156,11 @@ toggleHistorical ui@UIState{aopts=uopts@UIOpts{cliopts_=copts@CliOpts{reportspec -- (which are usually but not necessarily future-dated). -- In normal mode, both of these are hidden. toggleForecast :: Day -> UIState -> UIState -toggleForecast d ui@UIState{aopts=UIOpts{cliopts_=copts@CliOpts{inputopts_=iopts}}} = +toggleForecast d ui@UIState{aopts=UIOpts{cliopts_=CliOpts{inputopts_=iopts}}} = uiSetForecast ui $ case forecast_ iopts of Just _ -> Nothing - Nothing -> forecastPeriodFromRawOpts d (rawopts_ copts) <|> Just nulldatespan + Nothing -> forecastPeriod d iopts{forecast_=Just nulldatespan} (ajournal ui) -- | Helper: set forecast mode (with the given forecast period) on or off in the UI state. uiSetForecast :: UIState -> Maybe DateSpan -> UIState diff --git a/hledger/hledger.m4.md b/hledger/hledger.m4.md index b9613e76e..6c6d0d0ad 100644 --- a/hledger/hledger.m4.md +++ b/hledger/hledger.m4.md @@ -3165,15 +3165,16 @@ transactions generated "just now": `_generated-transaction:~ PERIODICEXPR`. Periodic transactions are generated within some forecast period. -By default, this - -- begins on the later of +This begins on: +- the start date supplied to the `--forecast` argument, if present +- otherwise, the later of - the report start date if specified with -b/-p/date: - - the day after the latest normal (non-periodic) transaction in the journal, - or today if there are no normal transactions. - -- ends on the report end date if specified with -e/-p/date:, - or 6 months (180 days) from today. + - the day after the latest normal (non-periodic) transaction in the journal, if any +- otherwise today. +It ends on: +- the end date supplied to the `--forecast` argument, if present +- otherwise the report end date if specified with -e/-p/date: +- otherwise 180 days (6 months) from today. This means that periodic transactions will begin only after the latest recorded transaction. And a recorded transaction dated in the future can diff --git a/hledger/test/forecast.test b/hledger/test/forecast.test index 2d7150cb4..49c86d93d 100644 --- a/hledger/test/forecast.test +++ b/hledger/test/forecast.test @@ -196,21 +196,6 @@ $ hledger -f - reg --forecast date:202001 2020-01-28 (a) 1,000.00 USD 2,000.00 USD >=0 -< -2021-01-01 - (a) 1000 - -~ daily - (a) 1 - -# 11. Forecast transactions are generated up to the day before the requested end date -$ hledger -f - reg -b 2021-01-01 -e 2021-01-05 --forecast -2021-01-01 (a) 1000 1000 -2021-01-02 (a) 1 1001 -2021-01-03 (a) 1 1002 -2021-01-04 (a) 1 1003 ->=0 - < 2021-09-01 Normal Balance Assertion Works Checking = -60 @@ -224,7 +209,7 @@ $ hledger -f - reg -b 2021-01-01 -e 2021-01-05 --forecast Checking = -120 Costs -# 12. Forecast transactions work with balance assignments +# 11. Forecast transactions work with balance assignments $ hledger -f - print -x --forecast -e 2021-11 2021-09-01 Normal Balance Assertion Works Checking -60 = -60 @@ -250,7 +235,7 @@ $ hledger -f - print -x --forecast -e 2021-11 income:client1 -10 USD assets:receivables:contractor1 -# 13. Generated forecast for weekday transactions +# 12. Generated forecast for weekday transactions $ hledger -f - reg --forecast -b "2021-09-01" -e "2021-09-15" --forecast -w 100 2021-09-01 income:client1 -10 USD -10 USD assets:receivables:contractor1 10 USD 0 @@ -282,7 +267,7 @@ $ hledger -f - reg --forecast -b "2021-09-01" -e "2021-09-15" --forecast -w 100 income:client1 -10 USD assets:receivables:contractor1 -# 14. Generated forecast for weekend transactions +# 13. Generated forecast for weekend transactions $ hledger -f - reg --forecast -b "2021-09-01" -e "2021-09-15" --forecast -w 100 2021-09-04 income:client1 -10 USD -10 USD assets:receivables:contractor1 10 USD 0 @@ -293,3 +278,68 @@ $ hledger -f - reg --forecast -b "2021-09-01" -e "2021-09-15" --forecast -w 100 2021-09-12 income:client1 -10 USD -10 USD assets:receivables:contractor1 10 USD 0 >=0 + +< +2021-01-01 + (a) 1000 + +~ daily + (a) 1 + +# 14. Arguments to --forecast take precedence over anything. Only generate up to the day before the end date. +$ hledger -f - reg --forecast="2020-01-01..2020-01-05" -b 2019-12-01 -e 2020-02-01 -H +2020-01-01 (a) 1 1 +2020-01-02 (a) 1 2 +2020-01-03 (a) 1 3 +2020-01-04 (a) 1 4 +>=0 + +# 15. With no arguments to --forecast, we use the report start date if it's after the journal end date. +$ hledger -f - reg --forecast -b 2021-02-01 -e 2021-02-05 -H +2021-02-01 (a) 1 1001 +2021-02-02 (a) 1 1002 +2021-02-03 (a) 1 1003 +2021-02-04 (a) 1 1004 +>=0 + +# 16. With no arguments to --forecast, we use journal end date if it's after the report start date. +$ hledger -f - reg --forecast -b 2020-12-01 -e 2021-01-05 -H +2021-01-01 (a) 1000 1000 +2021-01-02 (a) 1 1001 +2021-01-03 (a) 1 1002 +2021-01-04 (a) 1 1003 +>=0 + +# 17. With no arguments to --forecast, and no report start, generate from journal end to 180 days from today. +# We use here the fact that we are at least 180 days from 2021-01-01. This test will fail if you travel back in time! +$ hledger -f - reg --forecast -H +> /1 1360/ +>=0 + +< +~ daily + (a) 1 + +# 18. No real transactions. +# Arguments to --forecast take precedence over anything. Only generate up to the day before the end date. +$ hledger -f - reg --forecast="2020-01-01..2020-01-05" -b 2019-12-01 -e 2020-01-05 -H +2020-01-01 (a) 1 1 +2020-01-02 (a) 1 2 +2020-01-03 (a) 1 3 +2020-01-04 (a) 1 4 +>=0 + +# 19. No real transactions. +# With no arguments to --forecast, we use the report start date. +$ hledger -f - reg --forecast -b 2021-02-01 -e 2021-02-05 -H +2021-02-01 (a) 1 1 +2021-02-02 (a) 1 2 +2021-02-03 (a) 1 3 +2021-02-04 (a) 1 4 +>=0 + +# 20. No real transactions. +# With no arguments to --forecast, and no report start, generate from today to 180 days from today. +$ hledger -f - reg --forecast -H +> /1 180/ +>=0