fix: lib: Filter postings before performing valuation and costing in

MultiBalanceReport, PostingsReport, EntriesReport, and
AccountTransactionsReport. (#1625, #1630)

hledger-lib/Hledger/Reports/AccountTransactionsReport.hs

@@ -113,20 +113,21 @@ accountTransactionsReport rspec@ReportSpec{_rsReportOpts=ropts} j thisacctq = it
     -- sort by the transaction's register date, for accurate starting balance
     -- these are not yet filtered by tdate, we want to search them all for priorps
     transactions =
-        ptraceAtWith 5 (("ts5:\n"++).pshowTransactions)
+        ptraceAtWith 5 (("ts4:\n"++).pshowTransactions)
       . sortOn (transactionRegisterDate reportq thisacctq)
       . jtxns
-      . ptraceAtWith 5 (("ts4:\n"++).pshowTransactions.jtxns)
+      . ptraceAtWith 5 (("ts3:\n"++).pshowTransactions.jtxns)
+      -- maybe convert these transactions to cost or value
+      . journalApplyValuationFromOpts rspec
+      . (if filtertxns then filterJournalTransactions reportq else id)
       -- keep just the transactions affecting this account (via possibly realness or status-filtered postings)
       . traceAt 3 ("thisacctq: "++show thisacctq)
-      . ptraceAtWith 5 (("ts3:\n"++).pshowTransactions.jtxns)
+      . ptraceAtWith 5 (("ts2:\n"++).pshowTransactions.jtxns)
       . filterJournalTransactions thisacctq
       . filterJournalPostings (And [realq, statusq])
+      . ptraceAtWith 5 (("ts1:\n"++).pshowTransactions.jtxns)
       -- apply any cur:SYM filters in reportq
-      . ptraceAtWith 5 (("ts2:\n"++).pshowTransactions.jtxns)
+      $ if queryIsNull symq then j else filterJournalAmounts symq j
-      . (if queryIsNull symq then id else filterJournalAmounts symq)
-      -- maybe convert these transactions to cost or value
-      $ journalApplyValuationFromOpts rspec j
 
     startbal
       | balanceaccum_ ropts == Historical = sumPostings priorps
@@ -150,10 +151,7 @@ accountTransactionsReport rspec@ReportSpec{_rsReportOpts=ropts} j thisacctq = it
     -- Make it an option for now.
     filtertxns = txn_dates_ ropts
 
-    items = reverse $
+    items = reverse $ accountTransactionsReportItems reportq thisacctq startbal maNegate transactions
-            accountTransactionsReportItems reportq thisacctq startbal maNegate $
-            (if filtertxns then filter (reportq `matchesTransaction`) else id) $
-            transactions
 
 pshowTransactions :: [Transaction] -> String
 pshowTransactions = pshow . map (\t -> unwords [show $ tdate t, T.unpack $ tdescription t])
@@ -172,8 +170,7 @@ accountTransactionsReportItem reportq thisacctq signfn bal torig = balItem
     -- 201407: I've lost my grip on this, let's just hope for the best
     -- 201606: we now calculate change and balance from filtered postings, check this still works well for all callers XXX
     where
-      tfiltered@Transaction{tpostings=reportps} = filterTransactionPostings reportq torig
+      tacct@Transaction{tpostings=reportps} = torig{tdate=transactionRegisterDate reportq thisacctq torig}
-      tacct = tfiltered{tdate=transactionRegisterDate reportq thisacctq tfiltered}
       balItem = case reportps of
            [] -> (bal, Nothing)  -- no matched postings in this transaction, skip it
            _  -> (b, Just (torig, tacct, numotheraccts > 1, otheracctstr, a, b))

hledger-lib/Hledger/Reports/EntriesReport.hs

@@ -35,8 +35,9 @@ type EntriesReportItem = Transaction
 -- | Select transactions for an entries report.
 entriesReport :: ReportSpec -> Journal -> EntriesReport
 entriesReport rspec@ReportSpec{_rsReportOpts=ropts} =
-    sortBy (comparing $ transactionDateFn ropts) . jtxns . filterJournalTransactions (_rsQuery rspec)
+    sortBy (comparing $ transactionDateFn ropts) . jtxns
     . journalApplyValuationFromOpts rspec{_rsReportOpts=ropts{show_costs_=True}}
+    . filterJournalTransactions (_rsQuery rspec)
 
 tests_EntriesReport = tests "EntriesReport" [
   tests "entriesReport" [

hledger-lib/Hledger/Reports/MultiBalanceReport.hs

@@ -244,9 +244,9 @@ getPostings :: ReportSpec -> Journal -> PriceOracle -> [(Posting, Day)]
 getPostings rspec@ReportSpec{_rsQuery=query,_rsReportOpts=ropts} j priceoracle =
     map (\p -> (p, date p)) .
     journalPostings .
-    filterJournalAmounts symq .      -- remove amount parts excluded by cur:
+    valueJournal .
-    filterJournalPostings reportq $  -- remove postings not matched by (adjusted) query
+    filterJournalAmounts symq $      -- remove amount parts excluded by cur:
-    valuedJournal
+    filterJournalPostings reportq j  -- remove postings not matched by (adjusted) query
   where
     symq = dbg3 "symq" . filterQuery queryIsSym $ dbg3 "requested q" query
     -- The user's query with no depth limit, and expanded to the report span
@@ -254,8 +254,8 @@ getPostings rspec@ReportSpec{_rsQuery=query,_rsReportOpts=ropts} j priceoracle =
     -- handles the hledger-ui+future txns case above).
     reportq = dbg3 "reportq" $ depthless query
     depthless = dbg3 "depthless" . filterQuery (not . queryIsDepth)
-    valuedJournal | isJust (valuationAfterSum ropts) = j
+    valueJournal j' | isJust (valuationAfterSum ropts) = j'
-                  | otherwise = journalApplyValuationFromOptsWith rspec j priceoracle
+                    | otherwise = journalApplyValuationFromOptsWith rspec j' priceoracle
 
     date = case whichDateFromOpts ropts of
         PrimaryDate   -> postingDate

hledger-lib/Hledger/Reports/PostingsReport.hs

@@ -121,11 +121,11 @@ matchedPostingsBeforeAndDuring rspec@ReportSpec{_rsReportOpts=ropts,_rsQuery=q}
     beforeandduringps =
       dbg5 "ps5" $ sortOn sortdate $                                             -- sort postings by date or date2
       dbg5 "ps4" $ (if invert_ ropts then map negatePostingAmount else id) $     -- with --invert, invert amounts
-      dbg5 "ps3" $ map (filterPostingAmount symq) $                              -- remove amount parts which the query's cur: terms would exclude
+      dbg5 "ps3" $ (if related_ ropts then concatMap relatedPostings else id) $  -- with -r, replace each with its sibling postings
-      dbg5 "ps2" $ (if related_ ropts then concatMap relatedPostings else id) $  -- with -r, replace each with its sibling postings
+                   journalPostings $
-      dbg5 "ps1" $ filter (beforeandduringq `matchesPosting`) $                  -- filter postings by the query, with no start date or depth limit
+                   journalApplyValuationFromOpts rspec $                          -- convert to cost and apply valuation
-                  journalPostings $
+      dbg5 "ps2" $ filterJournalAmounts symq $                                    -- remove amount parts which the query's cur: terms would exclude
-                  journalApplyValuationFromOpts rspec j                          -- convert to cost and apply valuation
+      dbg5 "ps1" $ filterJournalPostings beforeandduringq j                       -- filter postings by the query, with no start date or depth limit
       where
         beforeandduringq = dbg4 "beforeandduringq" $ And [depthless $ dateless q, beforeendq]
           where

hledger/test/journal/amounts-and-commodities.test

@@ -297,3 +297,44 @@ $ hledger -f- print cur: amt:1112
 
 >=
 
+<
+P 2021-01-01 A  1000 B
+2021-01-01
+  (a)    1 A
+
+2021-01-02
+  (a)   10 A
+
+2021-01-03
+  (a)   10 A @ 0.1 B
+
+2021-01-04
+  (a)    1 B @ 0.001 A
+
+2021-01-05
+  (a)    1 B
+
+# 22. balance report cur: and amt: query matches currency before valuation/cost
+$ hledger -f- balance -N -V -B cur:A "amt:<5"
+              1000 B  a
+>=
+
+# 23. register report cur: and amt: query matches currency before valuation/cost
+$ hledger -f- register -V -B cur:A "amt:<5"
+2021-01-01                      (a)                         1000 B        1000 B
+>=
+
+# 24. aregister report cur: and amt: query matches currency before
+# valuation/cost (note that aregister currently only tests queries other than
+# "cur:" and the account query when given the --txn-dates option. Who knows why.
+$ hledger -f- aregister a -V -B cur:A "amt:<5" --txn-dates
+Transactions in a and subaccounts:
+2021-01-01                      a                           1000 B        1000 B
+>=
+
+# 25. print report cur: and amt: query matches currency before valuation/cost
+$ hledger -f- print -V -B a cur:A "amt:<5"
+2021-01-01
+    (a)          1000 B
+
+>=