From a07a084fb74c6098d730193eaf467859510621d1 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Fri, 11 Oct 2019 23:56:40 +0100 Subject: [PATCH 01/50] test: tests for multi-posting CSV import --- tests/csv.test | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/csv.test b/tests/csv.test index 8486d55f9..1de51cd43 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -111,6 +111,25 @@ $ printf 'fields date, description, amount, balance1, balance2\ndate-format %%d >=0 +# 11. More than two postings +$ printf 'fields date, description, amount, balance1, balance2, amount3,comment3\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\naccount3 expenses:tax\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,321,123,0.234,VAT\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +2009/09/10 Flubber Co + assets:myacct $50 = $321 + expense:unknown = $123 + expenses:tax $0.234 ; VAT + +>=0 + +# 12. More than two postings and different currencies +$ printf 'fields date, description, amount, balance1, balance2, currency3, amount3,comment3\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\naccount3 expenses:tax\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,321,123,£,0.234,VAT\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +2009/09/10 Flubber Co + assets:myacct $50 = $321 + expense:unknown = $123 + expenses:tax £0.234 ; VAT + +>=0 + + # . TODO: without --separator gives obscure error # | # 1 | 10/2009/09;Flubber Co🎅;50; From 0e1ead74c3866fc1790b08b54e5fd24c27f3080a Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 00:00:19 +0100 Subject: [PATCH 02/50] test: update old csv tests for new output format --- tests/csv.test | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/csv.test b/tests/csv.test index 1de51cd43..4bdec2160 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -7,8 +7,8 @@ # 1. read CSV to hledger journal format $ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules 2009/09/10 Flubber Co - assets:myacct $50 - income:unknown $-50 + assets:myacct $50 + expense:unknown >=0 @@ -19,11 +19,11 @@ $ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $ $ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescription %%2\namount-in %%3\namount-out %%4\ncurrency $\n' >t.$$.csv.rules ; hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules 2009/09/10 Flubber Co🎅 Assets:MyAccount $50 - income:unknown $-50 + expense:unknown 2009/09/11 Flubber Co🎅 Assets:MyAccount $-50 - expenses:unknown $50 + income:unknown >=0 @@ -31,39 +31,39 @@ $ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescrip $ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\nif Flubber\n account2 acct\n comment cmt' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules 2009/09/10 Flubber Co ; cmt assets:myacct $50 - acct $-50 + acct >=0 # 4. read CSV with balance field $ printf 'fields date, description, amount, balance\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,123\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules 2009/09/10 Flubber Co - assets:myacct $50 = $123 - income:unknown $-50 + assets:myacct $50 = $123 + expense:unknown >=0 # 5. read CSV with empty balance field $ printf 'fields date, description, amount, balance\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,123\n11/2009/09,Blubber Co,60,\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules 2009/09/10 Flubber Co - assets:myacct $50 = $123 - income:unknown $-50 + assets:myacct $50 = $123 + expense:unknown 2009/09/11 Blubber Co - assets:myacct $60 - income:unknown $-60 + assets:myacct $60 + expense:unknown >=0 # 6. read CSV with only whitespace in balance field $ printf 'fields date, description, amount, balance\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,123\n11/2009/09,Blubber Co,60, \n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules 2009/09/10 Flubber Co - assets:myacct $50 = $123 - income:unknown $-50 + assets:myacct $50 = $123 + expense:unknown 2009/09/11 Blubber Co - assets:myacct $60 - income:unknown $-60 + assets:myacct $60 + expense:unknown >=0 @@ -71,11 +71,11 @@ $ printf 'fields date, description, amount, balance\ndate-format %%d/%%Y/%%m\nc $ printf 'skip 1\n\ncurrency $\n\nfields date, payee, payment\n\namount -%%payment\naccount1 liabilities:bank\naccount2 expense:other' >t.$$.csv.rules; printf 'date,payee,amount\n2009/10/9,Flubber Co,50\n2009/11/09,Merchant Credit,-60\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules 2009/10/09 liabilities:bank $-50 - expense:other $50 + expense:other 2009/11/09 liabilities:bank $60 - expense:other $-60 + expense:other >=0 @@ -86,27 +86,27 @@ $ printf 'skip 1\n\ncurrency $\n\nfields date, payee, payment\n\namount -%%paym $ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescription %%2\namount-in %%3\namount-out %%4\ncurrency $\n' >rules.$$ ; hledger --separator ';' -f csv:- --rules-file rules.$$ print && rm -rf rules.$$ 2009/09/10 Flubber Co🎅 Assets:MyAccount $50 - income:unknown $-50 + expense:unknown 2009/09/11 Flubber Co🎅 Assets:MyAccount $-50 - expenses:unknown $50 + income:unknown >=0 # 9. read CSV with balance2 field $ printf 'fields date, description, amount, balance2\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,123\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules 2009/09/10 Flubber Co - assets:myacct $50 - income:unknown $-50 = $123 + assets:myacct $50 + expense:unknown = $123 >=0 # 10. read CSV with balance1 and balance2 fields $ printf 'fields date, description, amount, balance1, balance2\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,321,123\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules 2009/09/10 Flubber Co - assets:myacct $50 = $321 - income:unknown $-50 = $123 + assets:myacct $50 = $321 + expense:unknown = $123 >=0 From c5bab0ae402b2afa3253a756930a1f49d4acf663 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 00:06:16 +0100 Subject: [PATCH 03/50] test: added test for #570 --- tests/csv.test | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/csv.test b/tests/csv.test index 4bdec2160..03ce22433 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -129,6 +129,20 @@ $ printf 'fields date, description, amount, balance1, balance2, currency3, amou >=0 +# 13. reading CSV with in-field and out-field, where one could be zero +< +10/2009/09,Flubber Co🎅,50,0 +11/2009/09,Flubber Co🎅,0.00,50 +$ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescription %%2\namount-in %%3\namount-out %%4\ncurrency $\n' >t.$$.csv.rules ; hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +2009/09/10 Flubber Co🎅 + Assets:MyAccount $50 + expense:unknown + +2009/09/11 Flubber Co🎅 + Assets:MyAccount $-50 + income:unknown + +>=0 # . TODO: without --separator gives obscure error # | From 9aab476d5317612aa268d650be30abe02f754dd0 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 00:36:17 +0100 Subject: [PATCH 04/50] lib: csv parser supports up to 9 postings. Fixes #570, #627 --- hledger-lib/Hledger/Read/CsvReader.hs | 186 ++++++++++++++++---------- 1 file changed, 117 insertions(+), 69 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index f62a02c99..0ab169f6e 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -566,14 +566,19 @@ journalfieldnamep = do -- Transaction fields and pseudo fields for CSV conversion. -- Names must precede any other name they contain, for the parser -- (amount-in before amount; date2 before date). TODO: fix -journalfieldnames = [ - "account1" - ,"account2" - ,"amount-in" +journalfieldnames = + concat [[ "account" ++ i + ,"amount" ++ i ++ "-in" + ,"amount" ++ i ++ "-out" + ,"amount" ++ i + ,"balance" ++ i + ,"comment" ++ i + ,"currency" ++ i + ] | x <- [1..9], let i = show x] + ++ + ["amount-in" ,"amount-out" ,"amount" - ,"balance1" - ,"balance2" ,"balance" ,"code" ,"comment" @@ -662,8 +667,9 @@ regexp = do type CsvRecord = [String] --- Convert a CSV record to a transaction using the rules, or raise an --- error if the data can not be parsed. +showRules rules record = + unlines $ catMaybes [ (("the "++fld++" rule is: ")++) <$> getEffectiveAssignment rules record fld | fld <- journalfieldnames] + transactionFromCsvRecord :: SourcePos -> CsvRules -> CsvRecord -> Transaction transactionFromCsvRecord sourcepos rules record = t where @@ -679,7 +685,7 @@ transactionFromCsvRecord sourcepos rules record = t mdateformat = mdirective "date-format" date = render $ fromMaybe "" $ mfieldtemplate "date" date' = fromMaybe (error' $ dateerror "date" date mdateformat) $ mparsedate date - mdate2 = maybe Nothing (Just . render) $ mfieldtemplate "date2" + mdate2 = render <$> mfieldtemplate "date2" mdate2' = maybe Nothing (maybe (error' $ dateerror "date2" (fromMaybe "" mdate2) mdateformat) Just . mparsedate) mdate2 dateerror datefield value mdateformat = unlines ["error: could not parse \""++value++"\" as a date using date format "++maybe "\"YYYY/M/D\", \"YYYY-M-D\" or \"YYYY.M.D\"" show mdateformat @@ -707,54 +713,79 @@ transactionFromCsvRecord sourcepos rules record = t description = maybe "" render $ mfieldtemplate "description" comment = maybe "" render $ mfieldtemplate "comment" precomment = maybe "" render $ mfieldtemplate "precomment" - currency = maybe (fromMaybe "" mdefaultcurrency) render $ mfieldtemplate "currency" - amountstr = (currency++) <$> simplifySign <$> getAmountStr rules record - maybeamount = either amounterror (Mixed . (:[])) <$> runParser (evalStateT (amountp <* eof) mempty) "" <$> T.pack <$> amountstr - amounterror err = error' $ unlines - ["error: could not parse \""++fromJust amountstr++"\" as an amount" - ,showRecord record - ,"the amount rule is: "++(fromMaybe "" $ mfieldtemplate "amount") - ,"the currency rule is: "++(fromMaybe "unspecified" $ mfieldtemplate "currency") - ,"the default-currency is: "++fromMaybe "unspecified" mdefaultcurrency - ,"the parse error is: "++customErrorBundlePretty err - ,"you may need to " - ++"change your amount or currency rules, " - ++"or "++maybe "add a" (const "change your") mskip++" skip rule" - ] - amount1 = case maybeamount of - Just a -> a - Nothing | balance1 /= Nothing || balance2 /= Nothing -> nullmixedamt - Nothing -> error' $ "amount and balance have no value\n"++showRecord record - -- convert balancing amount to cost like hledger print, so eg if - -- amount1 is "10 GBP @@ 15 USD", amount2 will be "-15 USD". - amount2 = costOfMixedAmount (-amount1) + s `or` def = if null s then def else s - defaccount1 = fromMaybe "unknown" $ mdirective "default-account1" - defaccount2 = case isNegativeMixedAmount amount2 of - Just True -> "income:unknown" - _ -> "expenses:unknown" - account1 = T.pack $ maybe "" render (mfieldtemplate "account1") `or` defaccount1 - account2 = T.pack $ maybe "" render (mfieldtemplate "account2") `or` defaccount2 - balance1template = - case (mfieldtemplate "balance", mfieldtemplate "balance1") of - (Nothing, Nothing) -> Nothing - (balance, Nothing) -> balance - (Nothing, balance1) -> balance1 - (Just _, Just _) -> error' "Please use either balance or balance1, but not both" - balance1 = maybe Nothing (parsebalance "1".render) $ balance1template - balance2 = maybe Nothing (parsebalance "2".render) $ mfieldtemplate "balance2" - parsebalance n str + parsebalance currency n str | all isSpace str = Nothing | otherwise = Just $ (either (balanceerror n str) id $ runParser (evalStateT (amountp <* eof) mempty) "" $ T.pack $ (currency++) $ simplifySign str, nullsourcepos) balanceerror n str err = error' $ unlines ["error: could not parse \""++str++"\" as balance"++n++" amount" ,showRecord record - ,"the balance"++n++" rule is: "++(fromMaybe "" $ mfieldtemplate ("balance"++n)) - ,"the currency rule is: "++(fromMaybe "unspecified" $ mfieldtemplate "currency") + ,showRules rules record ,"the default-currency is: "++fromMaybe "unspecified" mdefaultcurrency ,"the parse error is: "++customErrorBundlePretty err ] + unknownAccountForAmount amt = + case isNegativeMixedAmount amt of + Just True -> "income:unknown" + _ -> "expense:unknown" + + parsePosting' number accountFld amtForUnknownAccount amountFld amountInFld amountOutFld balanceFld commentFld = + let currency = maybe (fromMaybe "" mdefaultcurrency) render $ + (mfieldtemplate ("currency"++number) `or `mfieldtemplate "currency") + amount = chooseAmountStr rules record currency amountFld amountInFld amountOutFld + account = ((T.pack . render) <$> (mfieldtemplate accountFld + `or` mdirective ("default-account" ++ number))) + `or` (unknownAccountForAmount <$> amtForUnknownAccount) + balance = (parsebalance currency number.render) =<< mfieldtemplate balanceFld + comment = T.pack $ maybe "" render $ mfieldtemplate commentFld + in + case account of + Nothing -> Nothing + Just account -> + Just $ posting {paccount=account, pamount=fromMaybe nullmixedamt amount, ptransaction=Just t, pbalanceassertion=toAssertion <$> balance, pcomment = comment} + + parsePosting number = + parsePosting' number + ("account"++number) + Nothing + ("amount"++number) + ("amount"++number++"-in") + ("amount"++number++"-out") + ("balance"++number) + ("comment" ++ number) + + postingLegacy = parsePosting' "" "account1" Nothing "amount" "amount-in" "amount-out" "balance" "comment1" + posting1' = parsePosting "1" + posting1 = + case (postingLegacy,posting1') of + (Just legacy, Nothing) -> legacy + (Nothing, Just posting1) -> posting1 + (Just legacy, Just posting1) -> + -- Here we merge legacy fields such as "amount" with "amount1", etc + -- Account and Comment would be the same by construction + let balanceassertion = (pbalanceassertion legacy) `or` (pbalanceassertion posting1) + amount = + let al = pamount legacy + a1 = pamount posting1 + in if al == a1 then al + else if isZeroMixedAmount a1 then al + else error' $ unlines [ "amount/amount-in/amount-out and amount1/amount1-in/amount1-out produced conflicting values" + , showRecord record + , showRules rules record + , "amount/amount-in/amount-out is " ++ show al + , "amount1/amount1-in/amount1-out is" ++ show a1 + ] + in posting {paccount=paccount posting1, pamount=amount, ptransaction=Just t, pbalanceassertion=balanceassertion, pcomment = pcomment posting1} + (Nothing, Nothing) -> error' $ unlines [ "sadly, no posting was generated for account1" + , showRecord record + , showRules rules record + ] + -- Posting 2 is special -- we want account to be income:unknown or expense:unknown if it is not specified, + -- based on the amount from posting 1 + posting2 = parsePosting' "2" "account2" (Just $ pamount posting1) "amount2" "amount2-in" "amount2-out" "balance2" "comment2" + postings2to9 = catMaybes $ posting2:[ parsePosting i | x<-[3..9], let i = show x] -- build the transaction t = nulltransaction{ tsourcepos = genericSourcePos sourcepos, @@ -764,39 +795,56 @@ transactionFromCsvRecord sourcepos rules record = t tcode = T.pack code, tdescription = T.pack description, tcomment = T.pack comment, - tprecedingcomment = T.pack precomment, - tpostings = - [posting {paccount=account1, pamount=amount1, ptransaction=Just t, pbalanceassertion=toAssertion <$> balance1} - ,posting {paccount=account2, pamount=amount2, ptransaction=Just t, pbalanceassertion=toAssertion <$> balance2} - ] + tprecedingcomment = T.pack precomment, + tpostings = posting1:postings2to9 } toAssertion (a, b) = assertion{ baamount = a, baposition = b } -getAmountStr :: CsvRules -> CsvRecord -> Maybe String -getAmountStr rules record = +chooseAmountStr :: CsvRules -> CsvRecord -> String -> String -> String -> String -> Maybe MixedAmount +chooseAmountStr rules record currency amountFld amountInFld amountOutFld = let - mamount = getEffectiveAssignment rules record "amount" - mamountin = getEffectiveAssignment rules record "amount-in" - mamountout = getEffectiveAssignment rules record "amount-out" - render = fmap (strip . renderTemplate rules record) + mamount = getEffectiveAssignment rules record amountFld + mamountin = getEffectiveAssignment rules record amountInFld + mamountout = getEffectiveAssignment rules record amountOutFld + parse amt = notZero =<< (parseAmount currency <$> notEmpty =<< (strip . renderTemplate rules record) <$> amt) in - case (render mamount, render mamountin, render mamountout) of - (Just "", Nothing, Nothing) -> Nothing + case (parse mamount, parse mamountin, parse mamountout) of + (Nothing, Nothing, Nothing) -> Nothing (Just a, Nothing, Nothing) -> Just a - (Nothing, Just "", Just "") -> error' $ "neither amount-in or amount-out has a value\n" - ++ " record: " ++ showRecord record - (Nothing, Just i, Just "") -> Just i - (Nothing, Just "", Just o) -> Just $ negateStr o - (Nothing, Just i, Just o) -> error' $ "both amount-in and amount-out have a value\n" - ++ " amount-in: " ++ i ++ "\n" - ++ " amount-out: " ++ o ++ "\n" + (Nothing, Just i, Nothing) -> Just i + (Nothing, Nothing, Just o) -> Just $ negate o + (Nothing, Just i, Just o) -> error' $ "both "++amountInFld++" and "++amountOutFld++" have a value\n" + ++ " "++amountInFld++": " ++ show i ++ "\n" + ++ " "++amountOutFld++": " ++ show o ++ "\n" ++ " record: " ++ showRecord record - _ -> error' $ "found values for amount and for amount-in/amount-out\n" - ++ "please use either amount or amount-in/amount-out\n" + _ -> error' $ "found values for "++amountFld++" and for "++amountInFld++"/"++amountOutFld++"\n" + ++ "please use either "++amountFld++" or "++amountInFld++"/"++amountOutFld++"\n" ++ " record: " ++ showRecord record + where + notZero amt = if isZeroMixedAmount amt then Nothing else Just amt + notEmpty str = if str=="" then Nothing else Just str + + parseAmount currency amountstr = + either (amounterror amountstr) (Mixed . (:[])) + <$> runParser (evalStateT (amountp <* eof) mempty) "" + <$> T.pack + <$> (currency++) + <$> simplifySign + <$> amountstr + + amounterror amountstr err = error' $ unlines + ["error: could not parse \""++fromJust amountstr++"\" as an amount" + ,showRecord record + ,showRules rules record + ,"the default-currency is: "++fromMaybe "unspecified" (getDirective "default-currency" rules) + ,"the parse error is: "++customErrorBundlePretty err + ,"you may need to " + ++"change your amount or currency rules, " + ++"or add or change your skip rule" + ] type CsvAmountString = String From 725bee02f242b73457230d66e7c6b9b25eb001cb Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 00:49:42 +0100 Subject: [PATCH 05/50] test: csv file with multiline description (#841, #416) --- tests/csv.test | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/csv.test b/tests/csv.test index 03ce22433..a64a768d1 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -144,6 +144,14 @@ $ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescrip >=0 +# 14. multiline descriptions +$ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,"Flubber Co\nCo\nCo",50\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +2009/09/10 Flubber Co Co Co + assets:myacct $50 + expense:unknown + +>=0 + # . TODO: without --separator gives obscure error # | # 1 | 10/2009/09;Flubber Co🎅;50; From e4add6df83bbbdeca8e83caf7c8cd21687ee2490 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 00:50:06 +0100 Subject: [PATCH 06/50] lib: fix for multiline descriptions in csv (fixes #841, #416) --- hledger-lib/Hledger/Read/CsvReader.hs | 8 ++++---- hledger-lib/Hledger/Utils/String.hs | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 0ab169f6e..f0c2ac684 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -709,10 +709,10 @@ transactionFromCsvRecord sourcepos rules record = t ["error: could not parse \""++str++"\" as a cleared status (should be *, ! or empty)" ,"the parse error is: "++customErrorBundlePretty err ] - code = maybe "" render $ mfieldtemplate "code" - description = maybe "" render $ mfieldtemplate "description" - comment = maybe "" render $ mfieldtemplate "comment" - precomment = maybe "" render $ mfieldtemplate "precomment" + code = singleline $ maybe "" render $ mfieldtemplate "code" + description = singleline $ maybe "" render $ mfieldtemplate "description" + comment = singleline $ maybe "" render $ mfieldtemplate "comment" + precomment = singleline $ maybe "" render $ mfieldtemplate "precomment" s `or` def = if null s then def else s parsebalance currency n str diff --git a/hledger-lib/Hledger/Utils/String.hs b/hledger-lib/Hledger/Utils/String.hs index 23fb31638..2490be6b7 100644 --- a/hledger-lib/Hledger/Utils/String.hs +++ b/hledger-lib/Hledger/Utils/String.hs @@ -21,6 +21,7 @@ module Hledger.Utils.String ( lstrip, rstrip, chomp, + singleline, elideLeft, elideRight, formatString, @@ -76,6 +77,10 @@ rstrip = reverse . lstrip . reverse chomp :: String -> String chomp = reverse . dropWhile (`elem` "\r\n") . reverse +-- | Remove line breaks +singleline :: String -> String +singleline = map (\c -> if c `elem` "\r\n" then ' ' else c) + stripbrackets :: String -> String stripbrackets = dropWhile (`elem` "([") . reverse . dropWhile (`elem` "])") . reverse :: String -> String From 294fb4172c57eb167dd35fcc6c8c27fdab69d5d6 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 00:55:25 +0100 Subject: [PATCH 07/50] test: recursive interpolation in csv rules (#500) --- tests/csv.test | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/csv.test b/tests/csv.test index a64a768d1..2373c185f 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -152,6 +152,14 @@ $ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $ >=0 +# 15. recursive interpolation +$ printf 'fields account1, date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $\nif Flubber\n account1 assets:%%account1\n amount (%%amount)' >t.$$.csv.rules; printf 'myacct,10/2009/09,Flubber Co,50\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +2009/09/10 Flubber Co + assets:myacct $-50 + income:unknown + +>=0 + # . TODO: without --separator gives obscure error # | # 1 | 10/2009/09;Flubber Co🎅;50; From 28ca65b99a922c05ff85ae5d8fbdccff7a794f0f Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 09:21:24 +0100 Subject: [PATCH 08/50] lib: more explicit conditions for using legacy csv parser --- hledger-lib/Hledger/Read/CsvReader.hs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index f0c2ac684..11652bb45 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -782,10 +782,21 @@ transactionFromCsvRecord sourcepos rules record = t , showRecord record , showRules rules record ] - -- Posting 2 is special -- we want account to be income:unknown or expense:unknown if it is not specified, + -- Posting 2 is special -- if there are no postings 3-9, we want to preserve legacy behaviour and + -- we want account to be income:unknown or expense:unknown if it is not specified, -- based on the amount from posting 1 - posting2 = parsePosting' "2" "account2" (Just $ pamount posting1) "amount2" "amount2-in" "amount2-out" "balance2" "comment2" - postings2to9 = catMaybes $ posting2:[ parsePosting i | x<-[3..9], let i = show x] + postings3to9 = catMaybes $ [ parsePosting i | x<-[3..9], let i = show x] + postings = + if postings3to9 == [] + then [fromMaybe justOnePostingError $ parsePosting' "2" "account2" (Just $ pamount posting1) "amount2" "amount2-in" "amount2-out" "balance2" "comment2"] + else case parsePosting "2" of + Just posting2 -> posting2:postings3to9 + Nothing -> postings3to9 + + justOnePostingError = error' $ unlines [ "Found single posting, cannot generate transaction" + , showRecord record + , showRules rules record + ] -- build the transaction t = nulltransaction{ tsourcepos = genericSourcePos sourcepos, @@ -796,7 +807,7 @@ transactionFromCsvRecord sourcepos rules record = t tdescription = T.pack description, tcomment = T.pack comment, tprecedingcomment = T.pack precomment, - tpostings = posting1:postings2to9 + tpostings = posting1:postings } toAssertion (a, b) = assertion{ baamount = a, From e4476dd2f123059e96555d7050eb768d627e9c3b Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 09:30:25 +0100 Subject: [PATCH 09/50] lib: more robust "unknown" account assignment in csv parser --- hledger-lib/Hledger/Read/CsvReader.hs | 29 +++++++++++++++++++-------- tests/csv.test | 12 +++++------ 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 11652bb45..4edd6d682 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -729,22 +729,35 @@ transactionFromCsvRecord sourcepos rules record = t unknownAccountForAmount amt = case isNegativeMixedAmount amt of Just True -> "income:unknown" - _ -> "expense:unknown" + Just False -> "expense:unknown" + _ -> "unknown" parsePosting' number accountFld amtForUnknownAccount amountFld amountInFld amountOutFld balanceFld commentFld = - let currency = maybe (fromMaybe "" mdefaultcurrency) render $ + let currency = maybe (fromMaybe "" mdefaultcurrency) render $ (mfieldtemplate ("currency"++number) `or `mfieldtemplate "currency") amount = chooseAmountStr rules record currency amountFld amountInFld amountOutFld - account = ((T.pack . render) <$> (mfieldtemplate accountFld + account' = ((T.pack . render) <$> (mfieldtemplate accountFld `or` mdirective ("default-account" ++ number))) `or` (unknownAccountForAmount <$> amtForUnknownAccount) - balance = (parsebalance currency number.render) =<< mfieldtemplate balanceFld - comment = T.pack $ maybe "" render $ mfieldtemplate commentFld + balance = (parsebalance currency number.render) =<< mfieldtemplate balanceFld + comment = T.pack $ maybe "" render $ mfieldtemplate commentFld + account = + case account' of + Just account -> Just account + Nothing -> + -- If we have amount or balance assertion (which implies potential amount change), + -- but no account name, lets generate "unknown" account name. + -- If we can figure out whether this is income or expense based on amount, do that + -- otherwise stick to "unknown" + case (amount, balance) of + (Just amt, _ ) -> Just $ unknownAccountForAmount amt + (_, Just _) -> Just "unknown" + (Nothing, Nothing) -> Nothing in case account of - Nothing -> Nothing - Just account -> - Just $ posting {paccount=account, pamount=fromMaybe nullmixedamt amount, ptransaction=Just t, pbalanceassertion=toAssertion <$> balance, pcomment = comment} + Nothing -> Nothing + Just account -> + Just $ posting {paccount=account, pamount=fromMaybe nullmixedamt amount, ptransaction=Just t, pbalanceassertion=toAssertion <$> balance, pcomment = comment} parsePosting number = parsePosting' number diff --git a/tests/csv.test b/tests/csv.test index 2373c185f..da4c51808 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -114,18 +114,18 @@ $ printf 'fields date, description, amount, balance1, balance2\ndate-format %%d # 11. More than two postings $ printf 'fields date, description, amount, balance1, balance2, amount3,comment3\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\naccount3 expenses:tax\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,321,123,0.234,VAT\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules 2009/09/10 Flubber Co - assets:myacct $50 = $321 - expense:unknown = $123 - expenses:tax $0.234 ; VAT + assets:myacct $50 = $321 + unknown = $123 + expenses:tax $0.234 ; VAT >=0 # 12. More than two postings and different currencies $ printf 'fields date, description, amount, balance1, balance2, currency3, amount3,comment3\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\naccount3 expenses:tax\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,321,123,£,0.234,VAT\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules 2009/09/10 Flubber Co - assets:myacct $50 = $321 - expense:unknown = $123 - expenses:tax £0.234 ; VAT + assets:myacct $50 = $321 + unknown = $123 + expenses:tax £0.234 ; VAT >=0 From 09f8d624584219dd281e4fa43ae61830fe7f6823 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 09:33:30 +0100 Subject: [PATCH 10/50] test: example on which recursive interpolation in CSV parser fails --- tests/csv.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/csv.test b/tests/csv.test index da4c51808..08288cc37 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -153,7 +153,7 @@ $ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $ >=0 # 15. recursive interpolation -$ printf 'fields account1, date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $\nif Flubber\n account1 assets:%%account1\n amount (%%amount)' >t.$$.csv.rules; printf 'myacct,10/2009/09,Flubber Co,50\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +$ printf 'fields account1, date, description, amount-in, amount-out\ndate-format %%d/%%Y/%%m\ncurrency $\nif Flubber\n account1 assets:%%account1\n amount-in (%%amount-in)' >t.$$.csv.rules; printf 'myacct,10/2009/09,Flubber Co,50,\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules 2009/09/10 Flubber Co assets:myacct $-50 income:unknown From 881422cd0f63dcd3666361392142745e3f3a77ea Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 09:33:58 +0100 Subject: [PATCH 11/50] lib: fix recursive interpolation in CSV parser for fields with dashes --- hledger-lib/Hledger/Read/CsvReader.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 4edd6d682..7aa6670fc 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -933,7 +933,7 @@ getEffectiveAssignment rules record f = lastMay $ assignmentsFor f -- | Render a field assigment's template, possibly interpolating referenced -- CSV field values. Outer whitespace is removed from interpolated values. renderTemplate :: CsvRules -> CsvRecord -> FieldTemplate -> String -renderTemplate rules record t = regexReplaceBy "%[A-z0-9]+" replace t +renderTemplate rules record t = regexReplaceBy "%[A-z0-9-]+" replace t where replace ('%':pat) = maybe pat (\i -> strip $ atDef "" record (i-1)) mindex where From 9440664fc441cc5438ffb47f3bc51cdf35637e70 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 10:00:14 +0100 Subject: [PATCH 12/50] test: cleaner tests for CSV parser proposal --- tests/csv.test | 19 +++++++++++++++++++ tests/hledger-csv | 15 +++++++++++++++ 2 files changed, 34 insertions(+) create mode 100755 tests/hledger-csv diff --git a/tests/csv.test b/tests/csv.test index 08288cc37..d05de86bf 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -160,6 +160,25 @@ $ printf 'fields account1, date, description, amount-in, amount-out\ndate-forma >=0 +# 16. use a script for cleaner csv tests? +< +myacct,10/2009/09,Flubber Co,50, + +RULES + +fields account1, date, description, amount-in, amount-out +date-format %d/%Y/%m +currency $ +if Flubber + account1 assets:%account1 + amount-in (%amount-in) +$ ./hledger-csv +2009/09/10 Flubber Co + assets:myacct $-50 + income:unknown + +>=0 + # . TODO: without --separator gives obscure error # | # 1 | 10/2009/09;Flubber Co🎅;50; diff --git a/tests/hledger-csv b/tests/hledger-csv new file mode 100755 index 000000000..a7383cc75 --- /dev/null +++ b/tests/hledger-csv @@ -0,0 +1,15 @@ +#!/bin/bash +# +# This scripts expects stdin formatted like this: +# +# RULES +# +# +awk -vCSV="t.$$.csv" -vRULES="t.$$.csv.rules" ' +BEGIN{output=CSV} +/^RULES/{output=RULES} +!/^RULES/{print $0 >output}' + +trap "rm -f t.$$.csv t.$$.csv.rules" EXIT ERR + +hledger -f csv:t.$$.csv --rules-file t.$$.csv.rules print From 24bba96ea2321b5939dc4233ee2b6b20a54b3a4a Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 10:16:38 +0100 Subject: [PATCH 13/50] lib: more robust multi-line joining in csv parser --- hledger-lib/Hledger/Utils/String.hs | 4 ++-- tests/csv.test | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hledger-lib/Hledger/Utils/String.hs b/hledger-lib/Hledger/Utils/String.hs index 2490be6b7..03342317d 100644 --- a/hledger-lib/Hledger/Utils/String.hs +++ b/hledger-lib/Hledger/Utils/String.hs @@ -77,9 +77,9 @@ rstrip = reverse . lstrip . reverse chomp :: String -> String chomp = reverse . dropWhile (`elem` "\r\n") . reverse --- | Remove line breaks +-- | Remove consequtive line breaks, replacing them with single space singleline :: String -> String -singleline = map (\c -> if c `elem` "\r\n" then ' ' else c) +singleline = unwords . filter (/="") . (map strip) . lines stripbrackets :: String -> String stripbrackets = dropWhile (`elem` "([") . reverse . dropWhile (`elem` "])") . reverse :: String -> String diff --git a/tests/csv.test b/tests/csv.test index d05de86bf..9fcc29271 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -145,7 +145,7 @@ $ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescrip >=0 # 14. multiline descriptions -$ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,"Flubber Co\nCo\nCo",50\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +$ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,"Flubber Co\n\n\n\nCo\nCo\n\n\n\n\n",50\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules 2009/09/10 Flubber Co Co Co assets:myacct $50 expense:unknown From f40d045251adce5ae25f15d52e184c714981af9f Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 21:12:50 +0100 Subject: [PATCH 14/50] test: updated csv parser tests to nicer format --- tests/csv.test | 217 ++++++++++++++++++++++++++++++++++++++++------ tests/hledger-csv | 2 +- 2 files changed, 193 insertions(+), 26 deletions(-) diff --git a/tests/csv.test b/tests/csv.test index 9fcc29271..16d531ba9 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -1,11 +1,13 @@ -# These tests read CSV from stdin for convenience, so to ensure we get the CSV reader's -# error, the csv: prefix is used. -# -# The final cleanup command is chained with && so as not to mask hledger's exit code, -# but this means a temp file is left behind whenever hledger fails. What TODO ? - # 1. read CSV to hledger journal format -$ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +< +10/2009/09,Flubber Co,50 +RULES +fields date, description, amount +date-format %d/%Y/%m +currency $ +account1 assets:myacct + +$ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 expense:unknown @@ -16,7 +18,16 @@ $ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $ < 10/2009/09,Flubber Co🎅,50, 11/2009/09,Flubber Co🎅,,50 -$ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescription %%2\namount-in %%3\namount-out %%4\ncurrency $\n' >t.$$.csv.rules ; hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +RULES +account1 Assets:MyAccount +date %1 +date-format %d/%Y/%m +description %2 +amount-in %3 +amount-out %4 +currency $ + +$ ./hledger-csv 2009/09/10 Flubber Co🎅 Assets:MyAccount $50 expense:unknown @@ -28,7 +39,18 @@ $ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescrip >=0 # 3. handle conditions assigning multiple fields -$ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\nif Flubber\n account2 acct\n comment cmt' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +< +10/2009/09,Flubber Co,50 + +RULES +fields date, description, amount +date-format %d/%Y/%m +currency $ +account1 assets:myacct +if Flubber + account2 acct + comment cmt +$ ./hledger-csv 2009/09/10 Flubber Co ; cmt assets:myacct $50 acct @@ -36,7 +58,16 @@ $ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $ >=0 # 4. read CSV with balance field -$ printf 'fields date, description, amount, balance\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,123\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +< +10/2009/09,Flubber Co,50,123 + +RULES +fields date, description, amount, balance +date-format %d/%Y/%m +currency $ +account1 assets:myacct + +$ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $123 expense:unknown @@ -44,7 +75,17 @@ $ printf 'fields date, description, amount, balance\ndate-format %%d/%%Y/%%m\nc >=0 # 5. read CSV with empty balance field -$ printf 'fields date, description, amount, balance\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,123\n11/2009/09,Blubber Co,60,\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +< +10/2009/09,Flubber Co,50,123 +11/2009/09,Blubber Co,60, + +RULES +fields date, description, amount, balance +date-format %d/%Y/%m +currency $ +account1 assets:myacct + +$ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $123 expense:unknown @@ -56,7 +97,17 @@ $ printf 'fields date, description, amount, balance\ndate-format %%d/%%Y/%%m\nc >=0 # 6. read CSV with only whitespace in balance field -$ printf 'fields date, description, amount, balance\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,123\n11/2009/09,Blubber Co,60, \n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +< +10/2009/09,Flubber Co,50,123 +11/2009/09,Blubber Co,60, + +RULES +fields date, description, amount, balance +date-format %d/%Y/%m +currency $ +account1 assets:myacct + +$ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $123 expense:unknown @@ -68,7 +119,22 @@ $ printf 'fields date, description, amount, balance\ndate-format %%d/%%Y/%%m\nc >=0 # 7. read CSV with rule double-negating column -$ printf 'skip 1\n\ncurrency $\n\nfields date, payee, payment\n\namount -%%payment\naccount1 liabilities:bank\naccount2 expense:other' >t.$$.csv.rules; printf 'date,payee,amount\n2009/10/9,Flubber Co,50\n2009/11/09,Merchant Credit,-60\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +< +date,payee,amount +2009/10/9,Flubber Co,50 +2009/11/09,Merchant Credit,-60 + +RULES +skip 1 + +currency $ + +fields date, payee, payment + +amount -%payment +account1 liabilities:bank +account2 expense:other +$ ./hledger-csv 2009/10/09 liabilities:bank $-50 expense:other @@ -83,7 +149,16 @@ $ printf 'skip 1\n\ncurrency $\n\nfields date, payee, payment\n\namount -%%paym < 10/2009/09;Flubber Co🎅;50; 11/2009/09;Flubber Co🎅;;50 -$ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescription %%2\namount-in %%3\namount-out %%4\ncurrency $\n' >rules.$$ ; hledger --separator ';' -f csv:- --rules-file rules.$$ print && rm -rf rules.$$ +RULES +account1 Assets:MyAccount +date %1 +date-format %d/%Y/%m +description %2 +amount-in %3 +amount-out %4 +currency $ + +$ ./hledger-csv --separator ';' 2009/09/10 Flubber Co🎅 Assets:MyAccount $50 expense:unknown @@ -95,7 +170,16 @@ $ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescrip >=0 # 9. read CSV with balance2 field -$ printf 'fields date, description, amount, balance2\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,123\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +< +10/2009/09,Flubber Co,50,123 + +RULES +fields date, description, amount, balance2 +date-format %d/%Y/%m +currency $ +account1 assets:myacct + +$ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 expense:unknown = $123 @@ -103,7 +187,16 @@ $ printf 'fields date, description, amount, balance2\ndate-format %%d/%%Y/%%m\n >=0 # 10. read CSV with balance1 and balance2 fields -$ printf 'fields date, description, amount, balance1, balance2\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,321,123\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +< +10/2009/09,Flubber Co,50,321,123 + +RULES +fields date, description, amount, balance1, balance2 +date-format %d/%Y/%m +currency $ +account1 assets:myacct + +$ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $321 expense:unknown = $123 @@ -111,8 +204,18 @@ $ printf 'fields date, description, amount, balance1, balance2\ndate-format %%d >=0 -# 11. More than two postings -$ printf 'fields date, description, amount, balance1, balance2, amount3,comment3\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\naccount3 expenses:tax\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,321,123,0.234,VAT\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +# 11. More than two postings +< +10/2009/09,Flubber Co,50,321,123,0.234,VAT + +RULES +fields date, description, amount, balance1, balance2, amount3,comment3 +date-format %d/%Y/%m +currency $ +account1 assets:myacct +account3 expenses:tax + +$ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $321 unknown = $123 @@ -120,8 +223,18 @@ $ printf 'fields date, description, amount, balance1, balance2, amount3,comment >=0 -# 12. More than two postings and different currencies -$ printf 'fields date, description, amount, balance1, balance2, currency3, amount3,comment3\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\naccount3 expenses:tax\n' >t.$$.csv.rules; printf '10/2009/09,Flubber Co,50,321,123,£,0.234,VAT\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +# 12. More than two postings and different currencies +< +10/2009/09,Flubber Co,50,321,123,£,0.234,VAT + +RULES +fields date, description, amount, balance1, balance2, currency3, amount3,comment3 +date-format %d/%Y/%m +currency $ +account1 assets:myacct +account3 expenses:tax + +$ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $321 unknown = $123 @@ -133,7 +246,16 @@ $ printf 'fields date, description, amount, balance1, balance2, currency3, amou < 10/2009/09,Flubber Co🎅,50,0 11/2009/09,Flubber Co🎅,0.00,50 -$ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescription %%2\namount-in %%3\namount-out %%4\ncurrency $\n' >t.$$.csv.rules ; hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +RULES +account1 Assets:MyAccount +date %1 +date-format %d/%Y/%m +description %2 +amount-in %3 +amount-out %4 +currency $ + +$ ./hledger-csv 2009/09/10 Flubber Co🎅 Assets:MyAccount $50 expense:unknown @@ -145,7 +267,25 @@ $ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescrip >=0 # 14. multiline descriptions -$ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $\naccount1 assets:myacct\n' >t.$$.csv.rules; printf '10/2009/09,"Flubber Co\n\n\n\nCo\nCo\n\n\n\n\n",50\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +< +10/2009/09,"Flubber Co + + + +Co +Co + + + + +",50 + +RULES +fields date, description, amount +date-format %d/%Y/%m +currency $ +account1 assets:myacct +$ ./hledger-csv 2009/09/10 Flubber Co Co Co assets:myacct $50 expense:unknown @@ -153,7 +293,18 @@ $ printf 'fields date, description, amount\ndate-format %%d/%%Y/%%m\ncurrency $ >=0 # 15. recursive interpolation -$ printf 'fields account1, date, description, amount-in, amount-out\ndate-format %%d/%%Y/%%m\ncurrency $\nif Flubber\n account1 assets:%%account1\n amount-in (%%amount-in)' >t.$$.csv.rules; printf 'myacct,10/2009/09,Flubber Co,50,\n' | hledger -f csv:- --rules-file t.$$.csv.rules print && rm -rf t.$$.csv.rules +< +myacct,10/2009/09,Flubber Co,50, + +RULES + +fields account1, date, description, amount-in, amount-out +date-format %d/%Y/%m +currency $ +if Flubber + account1 assets:%account1 + amount-in (%amount-in) +$ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $-50 income:unknown @@ -187,7 +338,15 @@ $ ./hledger-csv # < # 10/2009/09;Flubber Co🎅;50; # 11/2009/09;Flubber Co🎅;;50 -# $ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescription %%2\namount-in %%3\namount-out %%4\ncurrency $\n' >rules.$$ ; hledger -f csv:- --rules-file rules.$$ print && rm -rf rules.$$ +# RULES +# account1 Assets:MyAccount +# date %1 +# date-format %d/%Y/%m +# description %2 +# amount-in %3 +# amount-out %4 +# currency $ +# $ ./hledger-csv # 2009/09/10 Flubber Co🎅 # Assets:MyAccount $50 # income:unknown $-50 @@ -202,7 +361,15 @@ $ ./hledger-csv # < # 10/2009/09 Flubber Co🎅 50 # 11/2009/09 Flubber Co🎅 50 -# $ printf 'account1 Assets:MyAccount\ndate %%1\ndate-format %%d/%%Y/%%m\ndescription %%2\namount-in %%3\namount-out %%4\ncurrency $\n' >rules.$$ ; hledger --separator "\t" -f csv:- --rules-file rules.$$ print && rm -rf rules.$$ +# RULES +# account1 Assets:MyAccount +# date %1 +# date-format %d/%Y/%m +# description %2 +# amount-in %3 +# amount-out %4 +# currency $ +# $ ./hledger-csv # 2009/09/10 Flubber Co🎅 # Assets:MyAccount $50 # income:unknown $-50 diff --git a/tests/hledger-csv b/tests/hledger-csv index a7383cc75..8ea96a06d 100755 --- a/tests/hledger-csv +++ b/tests/hledger-csv @@ -12,4 +12,4 @@ BEGIN{output=CSV} trap "rm -f t.$$.csv t.$$.csv.rules" EXIT ERR -hledger -f csv:t.$$.csv --rules-file t.$$.csv.rules print +hledger -f csv:t.$$.csv --rules-file t.$$.csv.rules print "$@" From 825b9ce5b342bf9cbf24c3003a43460e237c83b2 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 21:22:40 +0100 Subject: [PATCH 15/50] lib: fixed amount vs amount1 conflict detection in csv parser --- hledger-lib/Hledger/Read/CsvReader.hs | 21 ++++++++++++-------- tests/csv.test | 28 ++++++++++++++++----------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 7aa6670fc..f86e0cdc3 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -782,14 +782,19 @@ transactionFromCsvRecord sourcepos rules record = t amount = let al = pamount legacy a1 = pamount posting1 - in if al == a1 then al - else if isZeroMixedAmount a1 then al - else error' $ unlines [ "amount/amount-in/amount-out and amount1/amount1-in/amount1-out produced conflicting values" - , showRecord record - , showRules rules record - , "amount/amount-in/amount-out is " ++ show al - , "amount1/amount1-in/amount1-out is" ++ show a1 - ] + in + if al == a1 then al + else + case (isZeroMixedAmount al, isZeroMixedAmount a1) of + (True, _) -> a1 + (False, True) -> al + (False, False) -> + error' $ unlines [ "amount/amount-in/amount-out and amount1/amount1-in/amount1-out produced conflicting values" + , showRecord record + , showRules rules record + , "amount/amount-in/amount-out is " ++ showMixedAmount al + , "amount1/amount1-in/amount1-out is " ++ showMixedAmount a1 + ] in posting {paccount=paccount posting1, pamount=amount, ptransaction=Just t, pbalanceassertion=balanceassertion, pcomment = pcomment posting1} (Nothing, Nothing) -> error' $ unlines [ "sadly, no posting was generated for account1" , showRecord record diff --git a/tests/csv.test b/tests/csv.test index 16d531ba9..b3975578e 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -311,22 +311,28 @@ $ ./hledger-csv >=0 -# 16. use a script for cleaner csv tests? +# 16. Real life-ish paypal parsing example < -myacct,10/2009/09,Flubber Co,50, +"12/22/2018","06:22:50","PST","Someone","Subscription Payment","Completed","USD","10.00","-0.59","9.41","someone@some.where","simon@joyful.com","123456789","Joyful Systems","","9KCXZZZZZXXAX","","57.60","" RULES - -fields account1, date, description, amount-in, amount-out -date-format %d/%Y/%m +fields date, time, timezone, description, type, status_, currency, grossamount, feeamount, netamount, fromemail, toemail, code, itemtitle, itemid, referencetxnid, receiptid, balance, note +account1 sm:assets:online:paypal +amount1 %netamount +account2 sm:expenses:unknown +account3 JS:expenses:banking:paypal +amount3 %feeamount +balance %18 +code %13 currency $ -if Flubber - account1 assets:%account1 - amount-in (%amount-in) +date %1 +date-format %m/%d/%Y +description %description for %itemtitle $ ./hledger-csv -2009/09/10 Flubber Co - assets:myacct $-50 - income:unknown +2018/12/22 (123456789) Someone for Joyful Systems + sm:assets:online:paypal $9.41 = $57.60 + sm:expenses:unknown + JS:expenses:banking:paypal $-0.59 >=0 From 77fa81ea4dcbbd64654f52747ab83d3d4d4747ca Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 21:57:42 +0100 Subject: [PATCH 16/50] lib: fix error message formatting (header displayed twice) --- hledger-lib/Hledger/Read/CsvReader.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index f86e0cdc3..e697827fa 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -689,7 +689,7 @@ transactionFromCsvRecord sourcepos rules record = t mdate2' = maybe Nothing (maybe (error' $ dateerror "date2" (fromMaybe "" mdate2) mdateformat) Just . mparsedate) mdate2 dateerror datefield value mdateformat = unlines ["error: could not parse \""++value++"\" as a date using date format "++maybe "\"YYYY/M/D\", \"YYYY-M-D\" or \"YYYY.M.D\"" show mdateformat - ,"the CSV record is: "++showRecord record + , showRecord record ,"the "++datefield++" rule is: "++(fromMaybe "required, but missing" $ mfieldtemplate datefield) ,"the date-format is: "++fromMaybe "unspecified" mdateformat ,"you may need to " From f74df08e98041cd5930b29306562997fabd8d29a Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 21:58:11 +0100 Subject: [PATCH 17/50] lib: fix accidental switch of income:unknown and expence:unknown in csv parser --- hledger-lib/Hledger/Read/CsvReader.hs | 2 +- tests/csv.test | 72 ++++++++++++++++----------- 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index e697827fa..4354bb78b 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -806,7 +806,7 @@ transactionFromCsvRecord sourcepos rules record = t postings3to9 = catMaybes $ [ parsePosting i | x<-[3..9], let i = show x] postings = if postings3to9 == [] - then [fromMaybe justOnePostingError $ parsePosting' "2" "account2" (Just $ pamount posting1) "amount2" "amount2-in" "amount2-out" "balance2" "comment2"] + then [fromMaybe justOnePostingError $ parsePosting' "2" "account2" (Just $ negate $ pamount posting1) "amount2" "amount2-in" "amount2-out" "balance2" "comment2"] else case parsePosting "2" of Just posting2 -> posting2:postings3to9 Nothing -> postings3to9 diff --git a/tests/csv.test b/tests/csv.test index b3975578e..c67c6f00a 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -9,8 +9,8 @@ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co - assets:myacct $50 - expense:unknown + assets:myacct $50 + income:unknown >=0 @@ -30,11 +30,11 @@ currency $ $ ./hledger-csv 2009/09/10 Flubber Co🎅 Assets:MyAccount $50 - expense:unknown + income:unknown 2009/09/11 Flubber Co🎅 Assets:MyAccount $-50 - income:unknown + expense:unknown >=0 @@ -69,8 +69,8 @@ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co - assets:myacct $50 = $123 - expense:unknown + assets:myacct $50 = $123 + income:unknown >=0 @@ -87,12 +87,12 @@ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co - assets:myacct $50 = $123 - expense:unknown + assets:myacct $50 = $123 + income:unknown 2009/09/11 Blubber Co - assets:myacct $60 - expense:unknown + assets:myacct $60 + income:unknown >=0 @@ -109,12 +109,12 @@ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co - assets:myacct $50 = $123 - expense:unknown + assets:myacct $50 = $123 + income:unknown 2009/09/11 Blubber Co - assets:myacct $60 - expense:unknown + assets:myacct $60 + income:unknown >=0 @@ -161,11 +161,11 @@ currency $ $ ./hledger-csv --separator ';' 2009/09/10 Flubber Co🎅 Assets:MyAccount $50 - expense:unknown + income:unknown 2009/09/11 Flubber Co🎅 Assets:MyAccount $-50 - income:unknown + expense:unknown >=0 @@ -181,8 +181,8 @@ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co - assets:myacct $50 - expense:unknown = $123 + assets:myacct $50 + income:unknown = $123 >=0 @@ -198,8 +198,8 @@ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co - assets:myacct $50 = $321 - expense:unknown = $123 + assets:myacct $50 = $321 + income:unknown = $123 >=0 @@ -258,11 +258,11 @@ currency $ $ ./hledger-csv 2009/09/10 Flubber Co🎅 Assets:MyAccount $50 - expense:unknown + income:unknown 2009/09/11 Flubber Co🎅 Assets:MyAccount $-50 - income:unknown + expense:unknown >=0 @@ -287,8 +287,8 @@ currency $ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co Co Co - assets:myacct $50 - expense:unknown + assets:myacct $50 + income:unknown >=0 @@ -306,14 +306,14 @@ if Flubber amount-in (%amount-in) $ ./hledger-csv 2009/09/10 Flubber Co - assets:myacct $-50 - income:unknown + assets:myacct $-50 + expense:unknown >=0 # 16. Real life-ish paypal parsing example < -"12/22/2018","06:22:50","PST","Someone","Subscription Payment","Completed","USD","10.00","-0.59","9.41","someone@some.where","simon@joyful.com","123456789","Joyful Systems","","9KCXZZZZZXXAX","","57.60","" +"12/22/2018","06:22:50","PST","Someone","Subscription Payment","Completed","USD","10.00","-0.59","9.41","someone@some.where","simon@joyful.com","123456789","Joyful Systems","","9KCXINCOME:UNKNOWNZXXAX","","57.60","" RULES fields date, time, timezone, description, type, status_, currency, grossamount, feeamount, netamount, fromemail, toemail, code, itemtitle, itemid, referencetxnid, receiptid, balance, note @@ -336,6 +336,20 @@ $ ./hledger-csv >=0 +# 18. Show that #415 is fixed +< +"2016/01/01","$1" +"2016/02/02","$1,000.00" +RULES +account1 unknown +amount %2 +date %1 +date-format %Y/%m/%d +$ ./hledger-csv | hledger balance -f - --no-total + $-1,001.00 income:unknown + $1,001.00 unknown +>=0 + # . TODO: without --separator gives obscure error # | # 1 | 10/2009/09;Flubber Co🎅;50; @@ -355,7 +369,7 @@ $ ./hledger-csv # $ ./hledger-csv # 2009/09/10 Flubber Co🎅 # Assets:MyAccount $50 -# income:unknown $-50 +# expense:unknown $-50 # # 2009/09/11 Flubber Co🎅 # Assets:MyAccount $-50 @@ -378,7 +392,7 @@ $ ./hledger-csv # $ ./hledger-csv # 2009/09/10 Flubber Co🎅 # Assets:MyAccount $50 -# income:unknown $-50 +# expense:unknown $-50 # # 2009/09/11 Flubber Co🎅 # Assets:MyAccount $-50 From fa61fdbe1b4836d391a7c0d4602440cfe59ec218 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 22:29:39 +0100 Subject: [PATCH 18/50] test: add a test for #1076 (conditional line skips in csv) --- tests/csv.test | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/csv.test b/tests/csv.test index c67c6f00a..ef17342ba 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -336,7 +336,7 @@ $ ./hledger-csv >=0 -# 18. Show that #415 is fixed +# 17. Show that #415 is fixed < "2016/01/01","$1" "2016/02/02","$1,000.00" @@ -350,6 +350,35 @@ $ ./hledger-csv | hledger balance -f - --no-total $1,001.00 unknown >=0 +# 18. Conditional skips +< +HEADER +10/2009/09,Flubber Co,50 +MIDDLE +10/2009/09,Flubber Co,50 +*** END OF FILE *** +RULES +fields date, description, amount +date-format %d/%Y/%m +currency $ +account1 assets:myacct + +if HEADER +MIDDLE +END OF FILE + skip + +$ ./hledger-csv +2009/09/10 Flubber Co + assets:myacct $50 + income:unknown + +2009/09/10 Flubber Co + assets:myacct $50 + income:unknown + +>=0 + # . TODO: without --separator gives obscure error # | # 1 | 10/2009/09;Flubber Co🎅;50; From 5dac141a7e5dfe774244fa2a2a7997143f10eedc Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 22:30:04 +0100 Subject: [PATCH 19/50] lib: implement conditional line skips in csv (fixes #1076) --- hledger-lib/Hledger/Read/CsvReader.hs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 4354bb78b..71d51c6b0 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -140,7 +140,7 @@ readJournalFromCsv separator mrulesfile csvfile csvdata = -- parsec seems to fail if you pass it "-" here TODO: try again with megaparsec let parsecfilename = if csvfile == "-" then "(stdin)" else csvfile records <- (either throwerr id . - dbg2 "validateCsv" . validateCsv skiplines . + dbg2 "validateCsv" . validateCsv rules skiplines . dbg2 "parseCsv") `fmap` parseCsv separator parsecfilename csvdata dbg1IO "first 3 csv records" $ take 3 records @@ -216,11 +216,12 @@ printCSV records = unlined (printRecord `map` records) unlined = concat . intersperse "\n" -- | Return the cleaned up and validated CSV data (can be empty), or an error. -validateCsv :: Int -> Either String CSV -> Either String [CsvRecord] -validateCsv _ (Left err) = Left err -validateCsv numhdrlines (Right rs) = validate $ drop numhdrlines $ filternulls rs +validateCsv :: CsvRules -> Int -> Either String CSV -> Either String [CsvRecord] +validateCsv _ _ (Left err) = Left err +validateCsv rules numhdrlines (Right rs) = validate $ filter (not.shouldSkip) $ drop numhdrlines $ filternulls rs where filternulls = filter (/=[""]) + shouldSkip r = isJust $ getEffectiveAssignment rules r "skip" validate [] = Right [] validate rs@(_first:_) | isJust lessthan2 = let r = fromJust lessthan2 in @@ -587,6 +588,7 @@ journalfieldnames = ,"date" ,"description" ,"status" + ,"skip" -- skip is not really a field, but we list it here to allow conditional rules that skip records ] assignmentseparatorp :: CsvRulesParser () From 5bd407b3b98cf22106ab68e1be35757d62127cca Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 22:47:54 +0100 Subject: [PATCH 20/50] test: show that #1000 is fixed --- tests/csv.test | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/csv.test b/tests/csv.test index ef17342ba..a85506a61 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -379,6 +379,34 @@ $ ./hledger-csv >=0 +# 19. Lines with just balance, no amount (#1000) +< +2018-10-15,100 +2018-10-16,200 +2018-10-17,300 +RULES +fields date,bal + +balance EUR %bal +date-format %Y-%m-%d +description Assets Update +account1 assets +account2 income +$ ./hledger-csv +2018/10/15 Assets Update + assets = EUR 100 + income + +2018/10/16 Assets Update + assets = EUR 200 + income + +2018/10/17 Assets Update + assets = EUR 300 + income + +>=0 + # . TODO: without --separator gives obscure error # | # 1 | 10/2009/09;Flubber Co🎅;50; From 1acebe14936972601ca48e64560c24a17319b0f2 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 23:35:36 +0100 Subject: [PATCH 21/50] test: csv reader test for #1001 --- tests/csv.test | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/csv.test b/tests/csv.test index a85506a61..a84f3a38f 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -367,7 +367,7 @@ if HEADER MIDDLE END OF FILE skip - + $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 @@ -407,6 +407,39 @@ $ ./hledger-csv >=0 +# 20. Test for #1001 - empty assignment to amount show not eat next line +< +2018-10-15,100 +2018-10-16,200 +2018-10-17,300 +RULES +fields date,bal + +balance EUR %bal +date-format %Y-%m-%d +description Assets Update +account1 assets +account2 income +if 2018 + amount + comment Dont eat me + balance + comment Dont eat me +$ ./hledger-csv +2018/10/15 Assets Update ; Dont eat me + assets + income + +2018/10/16 Assets Update ; Dont eat me + assets + income + +2018/10/17 Assets Update ; Dont eat me + assets + income + +>=0 + # . TODO: without --separator gives obscure error # | # 1 | 10/2009/09;Flubber Co🎅;50; From f108b52171058354b4cbe41f46a8052c1d84b52b Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 23:35:57 +0100 Subject: [PATCH 22/50] lib: fix for #1001 - empty field assignment consumes next line --- hledger-lib/Hledger/Read/CsvReader.hs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 71d51c6b0..2cf03517d 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -554,8 +554,9 @@ fieldassignmentp :: CsvRulesParser (JournalFieldName, FieldTemplate) fieldassignmentp = do lift $ dbgparse 3 "trying fieldassignmentp" f <- journalfieldnamep - assignmentseparatorp - v <- fieldvalp + v <- choiceInState [ assignmentseparatorp >> fieldvalp + , lift eolof >> return "" + ] return (f,v) "field assignment" @@ -594,12 +595,9 @@ journalfieldnames = assignmentseparatorp :: CsvRulesParser () assignmentseparatorp = do lift $ dbgparse 3 "trying assignmentseparatorp" - choice [ - -- try (lift (skipMany spacenonewline) >> oneOf ":="), - try (lift (skipMany spacenonewline) >> char ':'), - spaceChar - ] - _ <- lift (skipMany spacenonewline) + _ <- choiceInState [ lift (skipMany spacenonewline) >> char ':' >> lift (skipMany spacenonewline) + , lift (skipSome spacenonewline) + ] return () fieldvalp :: CsvRulesParser String From 5d207ddd05fc8c4a8592398d136d0e4bb4b6be4e Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 23:48:51 +0100 Subject: [PATCH 23/50] doc: describe changes to csv parser --- hledger-lib/hledger_csv.m4.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index 03d0fd22c..944548775 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -92,7 +92,7 @@ You'll need this whenever your CSV data contains header lines. Eg: - +This can also be used in a conditional block (without numeric argument) to ignore certain CSV records. ```rules # ignore the first CSV line skip 1 @@ -145,6 +145,10 @@ Eg: fields date, description, , amount, , , somefield, anotherfield ``` +If you want one line of CSV file to produce more that two postings, you can use the following fields, substituting X with a number from 1 to 9: `accountX`, `amountX`, `amountX-in`, `amount-X-out`, `currencyX`, `balanceX`, `commentX`. + +Fields `amount`, `amount-in`, `amount-out`, `currency` and `balance` are treated as aliases for `amount1`, and so on. If your rules file leads to both aliased fields having different values, `hledger` will raise an error. + ## field assignment *`ENTRYFIELDNAME`* *`FIELDVALUE`* From a2bd1ceb6136fe7723ff7cfae01feb2864eb7768 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 12 Oct 2019 23:58:25 +0100 Subject: [PATCH 24/50] doc: close todos mentioned in csv parser description, more details --- hledger-lib/hledger_csv.m4.md | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index 944548775..d90dfd551 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -133,7 +133,15 @@ date-format %-m/%-d/%Y %l:%M %p This (a) names the CSV fields, in order (names may not contain whitespace; uninteresting names may be left blank), and (b) assigns them to journal entry fields if you use any of these standard field names: -`date`, `date2`, `status`, `code`, `description`, `comment`, `account1`, `account2`, `amount`, `amount-in`, `amount-out`, `currency`, `balance`, `balance1`, `balance2`. + +Fields `date`, `date2`, `status`, `code`, `description` will form transaction description. + +Fields `accountX`, `amountX`, `amountX-in`, `amount-X-out`, `currencyX`, `balanceX`, `commentX`, where X is a number from 1 to 9, will form up to 9 postings in the transaction. + +Fields `amount`, `amount-in`, `amount-out`, `currency`, `balance` and `comment` are treated as aliases for `amount1`, and so on. If your rules file leads to both aliased fields having different values, `hledger` will raise an error. + +You need to provide enough information to create at least two postings. + Eg: ```rules # use the 1st, 2nd and 4th CSV fields as the entry's date, description and amount, @@ -145,10 +153,6 @@ Eg: fields date, description, , amount, , , somefield, anotherfield ``` -If you want one line of CSV file to produce more that two postings, you can use the following fields, substituting X with a number from 1 to 9: `accountX`, `amountX`, `amountX-in`, `amount-X-out`, `currencyX`, `balanceX`, `commentX`. - -Fields `amount`, `amount-in`, `amount-out`, `currency` and `balance` are treated as aliases for `amount1`, and so on. If your rules file leads to both aliased fields having different values, `hledger` will raise an error. - ## field assignment *`ENTRYFIELDNAME`* *`FIELDVALUE`* @@ -174,12 +178,12 @@ Note, interpolation strips any outer whitespace, so a CSV value like ## conditional block `if` *`PATTERN`*\ -    *`FIELDASSIGNMENTS`*... +    *`FIELDASSIGNMENTS or skip`*... `if`\ *`PATTERN`*\ *`PATTERN`*...\ -    *`FIELDASSIGNMENTS`*... +    *`FIELDASSIGNMENTS or skip`*... This applies one or more field assignments, only to those CSV records matched by one of the PATTERNs. The patterns are case-insensitive regular expressions which match anywhere @@ -187,6 +191,9 @@ within the whole CSV record (it's not yet possible to match within a specific field). When there are multiple patterns they can be written on separate lines, unindented. The field assignments are on separate lines indented by at least one space. + +Instead of field assignments you can specify `skip` to skip the matching record. + Examples: ```rules # if the CSV record contains "groceries", set account2 to "expenses:groceries" @@ -243,10 +250,10 @@ It's conventional and recommended to use `account1` for the account whose CSV we A transaction [amount](journal.html#amounts) must be set, in one of these ways: -- with an `amount` field assignment, which sets the first posting's amount +- with an `amount` or `amount1` field assignment, which sets the first posting's amount - (When the CSV has debit and credit amounts in separate fields:)\ - with field assignments for the `amount-in` and `amount-out` pseudo + with field assignments for the `amount-in` and `amount-out` (or `amount1-in` and `amount1-out`) pseudo fields (both of them). Whichever one has a value will be used, with appropriate sign. If both contain a value, it might not work so well. @@ -270,12 +277,10 @@ amount %amount %currency ## CSV balance assertions/assignments If the CSV includes a running balance, you can assign that to one of the pseudo fields -`balance` (or `balance1`) or `balance2`. +`balance` (or `balance1`), `balance2`, ... up to `balance9`. This will generate a [balance assertion](journal.html#balance-assertions) (or if the amount is left empty, a [balance assignment](journal.html#balance-assignments)), -on the first or second posting, -whenever the running balance field is non-empty. -(TODO: [#1000](https://github.com/simonmichael/hledger/issues/1000)) +on the appropriate posting, whenever the running balance field is non-empty. ## Reading multiple CSV files From f877a7789f7fafcd21298b4907e61b446d4e635e Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sun, 13 Oct 2019 12:44:08 +0100 Subject: [PATCH 25/50] doc: further clarify posting generation --- hledger-lib/hledger_csv.m4.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index d90dfd551..a3b131cc3 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -136,7 +136,7 @@ and (b) assigns them to journal entry fields if you use any of these standard fi Fields `date`, `date2`, `status`, `code`, `description` will form transaction description. -Fields `accountX`, `amountX`, `amountX-in`, `amount-X-out`, `currencyX`, `balanceX`, `commentX`, where X is a number from 1 to 9, will form up to 9 postings in the transaction. +An assignment to any of `accountN`, `amountN`, `amountN-in`, `amountN-out`, `balanceN` or `currencyN` will generate a posting (though it's your responsibility to ensure it is a well formed one). Normally the `N`'s are consecutive starting from 1 but it's not required. One posting will be generated for each unique `N`. If you wish to supply a comment for the posting, use `commentN`, though comment on its own will not cause posting to be generated. Fields `amount`, `amount-in`, `amount-out`, `currency`, `balance` and `comment` are treated as aliases for `amount1`, and so on. If your rules file leads to both aliased fields having different values, `hledger` will raise an error. From f1ab1074001d440ff6d6bba522f6881ea8d8cf12 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Mon, 14 Oct 2019 21:40:52 +0100 Subject: [PATCH 26/50] lib, doc: extended "skip" in "if" body to "skip N" --- hledger-lib/Hledger/Read/CsvReader.hs | 13 +++++++++++-- hledger-lib/hledger_csv.m4.md | 8 ++++---- tests/csv.test | 8 ++++++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 2cf03517d..fccfbd8f7 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -218,10 +218,19 @@ printCSV records = unlined (printRecord `map` records) -- | Return the cleaned up and validated CSV data (can be empty), or an error. validateCsv :: CsvRules -> Int -> Either String CSV -> Either String [CsvRecord] validateCsv _ _ (Left err) = Left err -validateCsv rules numhdrlines (Right rs) = validate $ filter (not.shouldSkip) $ drop numhdrlines $ filternulls rs +validateCsv rules numhdrlines (Right rs) = validate $ applyConditionalSkips $ drop numhdrlines $ filternulls rs where filternulls = filter (/=[""]) - shouldSkip r = isJust $ getEffectiveAssignment rules r "skip" + skipCount r = + case getEffectiveAssignment rules r "skip" of + Nothing -> Nothing + Just "" -> Just 1 + Just x -> Just (read x) + applyConditionalSkips [] = [] + applyConditionalSkips (r:rest) = + case skipCount r of + Nothing -> r:(applyConditionalSkips rest) + Just cnt -> applyConditionalSkips (drop (cnt-1) rest) validate [] = Right [] validate rs@(_first:_) | isJust lessthan2 = let r = fromJust lessthan2 in diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index a3b131cc3..dee69ce7b 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -92,7 +92,7 @@ You'll need this whenever your CSV data contains header lines. Eg: -This can also be used in a conditional block (without numeric argument) to ignore certain CSV records. +This can also be used in a conditional block to ignore certain CSV records. ```rules # ignore the first CSV line skip 1 @@ -178,12 +178,12 @@ Note, interpolation strips any outer whitespace, so a CSV value like ## conditional block `if` *`PATTERN`*\ -    *`FIELDASSIGNMENTS or skip`*... +    *`FIELDASSIGNMENTS or skip N`*... `if`\ *`PATTERN`*\ *`PATTERN`*...\ -    *`FIELDASSIGNMENTS or skip`*... +    *`FIELDASSIGNMENTS or skip N`*... This applies one or more field assignments, only to those CSV records matched by one of the PATTERNs. The patterns are case-insensitive regular expressions which match anywhere @@ -192,7 +192,7 @@ specific field). When there are multiple patterns they can be written on separate lines, unindented. The field assignments are on separate lines indented by at least one space. -Instead of field assignments you can specify `skip` to skip the matching record. +Instead of field assignments you can specify `skip N` to skip the next N records (including the one that matchied). Examples: ```rules diff --git a/tests/csv.test b/tests/csv.test index a84f3a38f..b26df9f38 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -354,7 +354,9 @@ $ ./hledger-csv | hledger balance -f - --no-total < HEADER 10/2009/09,Flubber Co,50 -MIDDLE +MIDDLE SKIP THIS LINE +AND THIS +AND THIS ONE 10/2009/09,Flubber Co,50 *** END OF FILE *** RULES @@ -364,10 +366,12 @@ currency $ account1 assets:myacct if HEADER -MIDDLE END OF FILE skip +if MIDDLE + skip 3 + $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 From b2ba1086b6b970588da312a8a56a17b4dc53aaa5 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Mon, 14 Oct 2019 21:41:25 +0100 Subject: [PATCH 27/50] lib: fixed validation rules for minimul viable csv rules to include account1 etc --- hledger-lib/Hledger/Read/CsvReader.hs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index fccfbd8f7..f8c620df7 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -468,16 +468,18 @@ validateRules rules = do (not amount && (amountin && amountout)) || balance) $ Left $ unlines [ - "Please specify (as a top level CSV rule) either the amount field," - ,"both the amount-in and amount-out fields, or the balance field. Eg:" - ,"amount %2\n" + "Please specify (as a top level CSV rule) either the amount1 field," + ,"both the amount1-in and amount1-out fields, or the balance1 field. Eg:" + ,"amount1 %2\n" + ,"You can also use amount, or both amount-in and amount-out, or balance," + ,"though this syntax is considered legacy." ] Right rules where - amount = isAssigned "amount" - amountin = isAssigned "amount-in" - amountout = isAssigned "amount-out" - balance = isAssigned "balance" || isAssigned "balance1" || isAssigned "balance2" + amount = isAssigned "amount" || isAssigned "amount1" + amountin = isAssigned "amount-in" || isAssigned "amount1-in" + amountout = isAssigned "amount-out" || isAssigned "amount1-out" + balance = isAssigned "balance" || isAssigned "balance1" isAssigned f = isJust $ getEffectiveAssignment rules [] f -- parsers From 38db7eb24da2e1075272b170b8cffc6a15d2e5b8 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Mon, 14 Oct 2019 21:46:35 +0100 Subject: [PATCH 28/50] lib: implement `skip end` in csv reader --- hledger-lib/Hledger/Read/CsvReader.hs | 1 + hledger-lib/hledger_csv.m4.md | 6 +++--- tests/csv.test | 7 ++++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index f8c620df7..3b176ef77 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -225,6 +225,7 @@ validateCsv rules numhdrlines (Right rs) = validate $ applyConditionalSkips $ dr case getEffectiveAssignment rules r "skip" of Nothing -> Nothing Just "" -> Just 1 + Just "end" -> Just maxBound Just x -> Just (read x) applyConditionalSkips [] = [] applyConditionalSkips (r:rest) = diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index dee69ce7b..04a484fb6 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -178,12 +178,12 @@ Note, interpolation strips any outer whitespace, so a CSV value like ## conditional block `if` *`PATTERN`*\ -    *`FIELDASSIGNMENTS or skip N`*... +    *`FIELDASSIGNMENTS` or `skip N` or `skip end`*... `if`\ *`PATTERN`*\ *`PATTERN`*...\ -    *`FIELDASSIGNMENTS or skip N`*... +    *`FIELDASSIGNMENTS` or `skip N` or `skip end`*... This applies one or more field assignments, only to those CSV records matched by one of the PATTERNs. The patterns are case-insensitive regular expressions which match anywhere @@ -192,7 +192,7 @@ specific field). When there are multiple patterns they can be written on separate lines, unindented. The field assignments are on separate lines indented by at least one space. -Instead of field assignments you can specify `skip N` to skip the next N records (including the one that matchied). +Instead of field assignments you can specify `skip N` to skip the next N records (including the one that matchied). Special form `skip end` will cause the rest of the file to be skipped. Examples: ```rules diff --git a/tests/csv.test b/tests/csv.test index b26df9f38..696bfd6bc 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -359,6 +359,8 @@ AND THIS AND THIS ONE 10/2009/09,Flubber Co,50 *** END OF FILE *** +More lines of the trailer here +They all should be ignored RULES fields date, description, amount date-format %d/%Y/%m @@ -366,9 +368,12 @@ currency $ account1 assets:myacct if HEADER -END OF FILE skip +if +END OF FILE + skip end + if MIDDLE skip 3 From b5d4918c163b712e6fd89180b8d2219898d24bc1 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Tue, 15 Oct 2019 20:52:42 +0100 Subject: [PATCH 29/50] lib: allow recursive interpolation of fields with underscores in names --- hledger-lib/Hledger/Read/CsvReader.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 3b176ef77..803ac8357 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -950,7 +950,7 @@ getEffectiveAssignment rules record f = lastMay $ assignmentsFor f -- | Render a field assigment's template, possibly interpolating referenced -- CSV field values. Outer whitespace is removed from interpolated values. renderTemplate :: CsvRules -> CsvRecord -> FieldTemplate -> String -renderTemplate rules record t = regexReplaceBy "%[A-z0-9-]+" replace t +renderTemplate rules record t = regexReplaceBy "%[A-z0-9_-]+" replace t where replace ('%':pat) = maybe pat (\i -> strip $ atDef "" record (i-1)) mindex where From 32cbe4c7b3cf13f5234dd7d313a4351e1ad4cbe4 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Tue, 15 Oct 2019 22:41:17 +0100 Subject: [PATCH 30/50] lib: better inference for unknown account names in csv parser --- hledger-lib/Hledger/Read/CsvReader.hs | 67 +++++++++++++-------------- tests/csv.test | 8 ++-- 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 803ac8357..1d8a433b3 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -738,19 +738,12 @@ transactionFromCsvRecord sourcepos rules record = t ,"the parse error is: "++customErrorBundlePretty err ] - unknownAccountForAmount amt = - case isNegativeMixedAmount amt of - Just True -> "income:unknown" - Just False -> "expense:unknown" - _ -> "unknown" - - parsePosting' number accountFld amtForUnknownAccount amountFld amountInFld amountOutFld balanceFld commentFld = + parsePosting' number accountFld amountFld amountInFld amountOutFld balanceFld commentFld = let currency = maybe (fromMaybe "" mdefaultcurrency) render $ (mfieldtemplate ("currency"++number) `or `mfieldtemplate "currency") - amount = chooseAmountStr rules record currency amountFld amountInFld amountOutFld + amount = chooseAmount rules record currency amountFld amountInFld amountOutFld account' = ((T.pack . render) <$> (mfieldtemplate accountFld `or` mdirective ("default-account" ++ number))) - `or` (unknownAccountForAmount <$> amtForUnknownAccount) balance = (parsebalance currency number.render) =<< mfieldtemplate balanceFld comment = T.pack $ maybe "" render $ mfieldtemplate commentFld account = @@ -759,29 +752,26 @@ transactionFromCsvRecord sourcepos rules record = t Nothing -> -- If we have amount or balance assertion (which implies potential amount change), -- but no account name, lets generate "unknown" account name. - -- If we can figure out whether this is income or expense based on amount, do that - -- otherwise stick to "unknown" case (amount, balance) of - (Just amt, _ ) -> Just $ unknownAccountForAmount amt + (Just _, _ ) -> Just "unknown" (_, Just _) -> Just "unknown" (Nothing, Nothing) -> Nothing in case account of Nothing -> Nothing Just account -> - Just $ posting {paccount=account, pamount=fromMaybe nullmixedamt amount, ptransaction=Just t, pbalanceassertion=toAssertion <$> balance, pcomment = comment} + Just $ posting {paccount=account, pamount=fromMaybe missingmixedamt amount, ptransaction=Just t', pbalanceassertion=toAssertion <$> balance, pcomment = comment} parsePosting number = parsePosting' number ("account"++number) - Nothing ("amount"++number) ("amount"++number++"-in") ("amount"++number++"-out") ("balance"++number) ("comment" ++ number) - postingLegacy = parsePosting' "" "account1" Nothing "amount" "amount-in" "amount-out" "balance" "comment1" + postingLegacy = parsePosting' "" "account1" "amount" "amount-in" "amount-out" "balance" "comment1" posting1' = parsePosting "1" posting1 = case (postingLegacy,posting1') of @@ -807,28 +797,37 @@ transactionFromCsvRecord sourcepos rules record = t , "amount/amount-in/amount-out is " ++ showMixedAmount al , "amount1/amount1-in/amount1-out is " ++ showMixedAmount a1 ] - in posting {paccount=paccount posting1, pamount=amount, ptransaction=Just t, pbalanceassertion=balanceassertion, pcomment = pcomment posting1} - (Nothing, Nothing) -> error' $ unlines [ "sadly, no posting was generated for account1" + in posting {paccount=paccount posting1, pamount=amount, ptransaction=Just t', pbalanceassertion=balanceassertion, pcomment = pcomment posting1} + (Nothing, Nothing) -> error' $ unlines [ "sadly, no posting was generated for account1, cannot generate transaction" , showRecord record , showRules rules record ] - -- Posting 2 is special -- if there are no postings 3-9, we want to preserve legacy behaviour and - -- we want account to be income:unknown or expense:unknown if it is not specified, - -- based on the amount from posting 1 - postings3to9 = catMaybes $ [ parsePosting i | x<-[3..9], let i = show x] + postings2to9 = catMaybes $ [ parsePosting i | x<-[2..9], let i = show x] postings = - if postings3to9 == [] - then [fromMaybe justOnePostingError $ parsePosting' "2" "account2" (Just $ negate $ pamount posting1) "amount2" "amount2-in" "amount2-out" "balance2" "comment2"] - else case parsePosting "2" of - Just posting2 -> posting2:postings3to9 - Nothing -> postings3to9 + if postings2to9 == [] + then [posting1,posting{paccount="unknown", pamount=missingmixedamt, ptransaction=Just t'}] + else posting1:postings2to9 - justOnePostingError = error' $ unlines [ "Found single posting, cannot generate transaction" - , showRecord record - , showRules rules record - ] + balanced = balanceTransaction Nothing t' + t = + case balanced of + Left _ -> t' + Right balanced -> + -- If we managed to balance transaction, lets infer better names for all "unknown" accounts + t' {tpostings = + [ originalPosting {paccount=newAccount} + | (originalPosting,p) <- zip postings (tpostings balanced) + , let account = paccount p + , let newAccount = + if account/="unknown" + then account + else case isNegativeMixedAmount (pamount p) of + Just True -> "income:unknown" + Just False -> "expense:unknown" + _ -> "unknown" + ]} -- build the transaction - t = nulltransaction{ + t' = nulltransaction{ tsourcepos = genericSourcePos sourcepos, tdate = date', tdate2 = mdate2', @@ -837,15 +836,15 @@ transactionFromCsvRecord sourcepos rules record = t tdescription = T.pack description, tcomment = T.pack comment, tprecedingcomment = T.pack precomment, - tpostings = posting1:postings + tpostings = postings } toAssertion (a, b) = assertion{ baamount = a, baposition = b } -chooseAmountStr :: CsvRules -> CsvRecord -> String -> String -> String -> String -> Maybe MixedAmount -chooseAmountStr rules record currency amountFld amountInFld amountOutFld = +chooseAmount :: CsvRules -> CsvRecord -> String -> String -> String -> String -> Maybe MixedAmount +chooseAmount rules record currency amountFld amountInFld amountOutFld = let mamount = getEffectiveAssignment rules record amountFld mamountin = getEffectiveAssignment rules record amountInFld diff --git a/tests/csv.test b/tests/csv.test index 696bfd6bc..f9c2c2bb5 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -217,9 +217,9 @@ account3 expenses:tax $ ./hledger-csv 2009/09/10 Flubber Co - assets:myacct $50 = $321 - unknown = $123 - expenses:tax $0.234 ; VAT + assets:myacct $50 = $321 + income:unknown = $123 + expenses:tax $0.234 ; VAT >=0 @@ -346,8 +346,8 @@ amount %2 date %1 date-format %Y/%m/%d $ ./hledger-csv | hledger balance -f - --no-total + $1,001.00 expense:unknown $-1,001.00 income:unknown - $1,001.00 unknown >=0 # 18. Conditional skips From 1ab8631264474a23d96218bc055fe260b9602115 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Tue, 15 Oct 2019 22:42:55 +0100 Subject: [PATCH 31/50] lib: change default rules file text to mention "amount1" --- hledger-lib/Hledger/Read/CsvReader.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 1d8a433b3..9274abf64 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -277,7 +277,7 @@ defaultRulesText csvfile = T.pack $ unlines ,"" ,"account1 assets:bank:checking" ,"" - ,"fields date, description, amount" + ,"fields date, description, amount1" ,"" ,"#skip 1" ,"#newest-first" From cb6a5fc4a119194848bb5488d92956af7fa5ed40 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Tue, 15 Oct 2019 22:53:01 +0100 Subject: [PATCH 32/50] doc: proofreading csv parser docs --- hledger-lib/hledger_csv.m4.md | 46 +++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index 04a484fb6..e0969943b 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -38,7 +38,7 @@ At minimum, the rules file must identify the date and amount fields. It's often necessary to specify the date format, and the number of header lines to skip, also. Eg: ``` -fields date, _, _, amount +fields date, _, _, amount1 date-format %d/%m/%Y skip 1 ``` @@ -55,7 +55,7 @@ A more complete example: skip 1 # name the csv fields (and assign the transaction's date, amount and code) -fields date, _, toorfrom, name, amzstatus, amount, fees, code +fields date, _, toorfrom, name, amzstatus, amount1, fees, code # how to parse the date date-format %b %-d, %Y @@ -64,7 +64,7 @@ date-format %b %-d, %Y description %toorfrom %name # save these fields as tags -comment status:%amzstatus, fees:%fees +comment status:%amzstatus # set the base account for all transactions account1 assets:amazon @@ -72,6 +72,9 @@ account1 assets:amazon # flip the sign on the amount amount -%amount +# Put fees in a separate posting +amount3 %fees +comment3 fees ``` For more examples, see [Convert CSV files](https://github.com/simonmichael/hledger/wiki/Convert-CSV-files). @@ -140,7 +143,7 @@ An assignment to any of `accountN`, `amountN`, `amountN-in`, `amountN-out`, `bal Fields `amount`, `amount-in`, `amount-out`, `currency`, `balance` and `comment` are treated as aliases for `amount1`, and so on. If your rules file leads to both aliased fields having different values, `hledger` will raise an error. -You need to provide enough information to create at least two postings. +You need to provide enough information to create at least one posting. Eg: ```rules @@ -150,7 +153,7 @@ Eg: # CSV field: # 1 2 3 4 5 6 7 8 # entry field: -fields date, description, , amount, , , somefield, anotherfield +fields date, description, , amount1, , , somefield, anotherfield ``` ## field assignment @@ -178,12 +181,20 @@ Note, interpolation strips any outer whitespace, so a CSV value like ## conditional block `if` *`PATTERN`*\ -    *`FIELDASSIGNMENTS` or `skip N` or `skip end`*... +    *`FIELDASSIGNMENTS`*... `if`\ *`PATTERN`*\ *`PATTERN`*...\ -    *`FIELDASSIGNMENTS` or `skip N` or `skip end`*... +    *`FIELDASSIGNMENTS`*... + +`if` *`PATTERN`*\ +*`PATTERN`*...\ +    *`skip N`*... + +`if` *`PATTERN`*\ +*`PATTERN`*...\ +    *`skip end`*... This applies one or more field assignments, only to those CSV records matched by one of the PATTERNs. The patterns are case-insensitive regular expressions which match anywhere @@ -192,7 +203,7 @@ specific field). When there are multiple patterns they can be written on separate lines, unindented. The field assignments are on separate lines indented by at least one space. -Instead of field assignments you can specify `skip N` to skip the next N records (including the one that matchied). Special form `skip end` will cause the rest of the file to be skipped. +Instead of field assignments you can specify `skip N` to skip the next N records (including the one that matchied) or `skip end` to skip the rest of the file. Examples: ```rules @@ -242,22 +253,21 @@ The order of same-day entries will be preserved ## CSV accounts -Each journal entry will have two [postings](journal.html#postings), to `account1` and `account2` respectively. -It's not yet possible to generate entries with more than two postings. +Each journal entry will have at least two [postings](journal.html#postings), to `account1` and some other account (usually `account2`). It's conventional and recommended to use `account1` for the account whose CSV we are reading. ## CSV amounts -A transaction [amount](journal.html#amounts) must be set, in one of these ways: +A posting [amount](journal.html#amounts) could be set in one of these ways: -- with an `amount` or `amount1` field assignment, which sets the first posting's amount +- with an `amountN` field assignment, which sets the Nth posting's amount - (When the CSV has debit and credit amounts in separate fields:)\ - with field assignments for the `amount-in` and `amount-out` (or `amount1-in` and `amount1-out`) pseudo + with field assignments for the `amountN-in` and `amountN-out` pseudo fields (both of them). Whichever one has a value will be used, with appropriate sign. If both contain a value, it might not work so well. -- or implicitly by means of a [balance assignment](journal.html#balance-assignments) (see below). +- with `balanceN` field assignment that creates a [balance assignment](journal.html#balance-assignments) (see below). There is some special handling for sign in amounts: @@ -265,13 +275,13 @@ There is some special handling for sign in amounts: - If an amount value begins with a double minus sign, those will cancel out and be removed. If the currency/commodity symbol is provided as a separate CSV field, -assign it to the `currency` pseudo field; the symbol will be prepended +assign it to the `currency` pseudo field (applicable to the whole transaction) or `currencyN` (applicable to Nth posting only); the symbol will be prepended to the amount (TODO: when there is an amount). -Or, you can use an `amount` [field assignment](#field-assignment) for more control, eg: +Or, you can use an `amountN` [field assignment](#field-assignment) for more control, eg: ``` -fields date,description,currency,amount -amount %amount %currency +fields date,description,currency,amount_ +amount1 %amount_ %currency ``` ## CSV balance assertions/assignments From 26a4f5e5195818a9608bf80dfcc602a8db1f8773 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Tue, 15 Oct 2019 23:11:31 +0100 Subject: [PATCH 33/50] doc: less confusing example --- hledger-lib/hledger_csv.m4.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index e0969943b..8a19ac471 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -280,8 +280,8 @@ to the amount (TODO: when there is an amount). Or, you can use an `amountN` [field assignment](#field-assignment) for more control, eg: ``` -fields date,description,currency,amount_ -amount1 %amount_ %currency +fields date,description,currency,amount1 +amount1 %amount1 %currency ``` ## CSV balance assertions/assignments From f92590f92c1e2aa4cdd42b993868588e1c4f8069 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Tue, 15 Oct 2019 23:11:53 +0100 Subject: [PATCH 34/50] lib: allow zero postings in csv reader --- hledger-lib/Hledger/Read/CsvReader.hs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 9274abf64..ab56feb8d 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -775,8 +775,8 @@ transactionFromCsvRecord sourcepos rules record = t posting1' = parsePosting "1" posting1 = case (postingLegacy,posting1') of - (Just legacy, Nothing) -> legacy - (Nothing, Just posting1) -> posting1 + (Just legacy, Nothing) -> Just legacy + (Nothing, Just posting1) -> Just posting1 (Just legacy, Just posting1) -> -- Here we merge legacy fields such as "amount" with "amount1", etc -- Account and Comment would be the same by construction @@ -797,16 +797,15 @@ transactionFromCsvRecord sourcepos rules record = t , "amount/amount-in/amount-out is " ++ showMixedAmount al , "amount1/amount1-in/amount1-out is " ++ showMixedAmount a1 ] - in posting {paccount=paccount posting1, pamount=amount, ptransaction=Just t', pbalanceassertion=balanceassertion, pcomment = pcomment posting1} - (Nothing, Nothing) -> error' $ unlines [ "sadly, no posting was generated for account1, cannot generate transaction" - , showRecord record - , showRules rules record - ] - postings2to9 = catMaybes $ [ parsePosting i | x<-[2..9], let i = show x] + in Just $ posting {paccount=paccount posting1, pamount=amount, ptransaction=Just t', pbalanceassertion=balanceassertion, pcomment = pcomment posting1} + (Nothing, Nothing) -> Nothing + postings' = catMaybes $ posting1:[ parsePosting i | x<-[2..9], let i = show x] postings = - if postings2to9 == [] - then [posting1,posting{paccount="unknown", pamount=missingmixedamt, ptransaction=Just t'}] - else posting1:postings2to9 + case postings' of + -- To be compatible with the behavior of the old code which allowed two postings only, we enforce + -- second posting when rules generated just one of them. + [posting1] -> [posting1,posting{paccount="unknown", pamount=missingmixedamt, ptransaction=Just t'}] + _ -> postings' balanced = balanceTransaction Nothing t' t = From 3e5f0e8dd51f73af1226aec5ca33b8282e3a970d Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Tue, 15 Oct 2019 23:38:23 +0100 Subject: [PATCH 35/50] doc: clarify csv rules application order --- hledger-lib/hledger_csv.m4.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index 8a19ac471..18b5d6ca1 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -292,6 +292,37 @@ This will generate a [balance assertion](journal.html#balance-assertions) (or if the amount is left empty, a [balance assignment](journal.html#balance-assignments)), on the appropriate posting, whenever the running balance field is non-empty. +## References to other fields and evaluation order + +Field assignments could include references to other fields or even to the same field you are trying to assign: + +``` +fields date,description,currency,amount1 + +amount1 %amount1 USD +amount1 %amount1 EUR +amount1 %amount1 %currency + +if SOME_REGEXP + amount1 %amount1 GBP +``` +This is how this file would be evaluated. + +First, parts of CVS record are assigned according to `fields` directive. + +Then all other field assignments -- written at top level, or included in `if` blocks -- are considered to see if they should be applied. They are checked in the order they are written, with later assignment overwriting earlier ones. + +Once full set of field assignments that should be applied is known, their values are computed, and this is when all `%` references are evaluated. + +So for a particular row from CSV file, value from fourth column would be assigned to `amount1`. + +Then `hledger` will decide that `amount1` would have to be amended to `%amount1 USD`, but this will not happen immediately. This choice would be replaced by decision to rewrite `amount1` to `%amount EUR`, which will in turn be thrown away in favor of `%amount1 %currency`. If the `if` block condition will match the row, it will assign `amount1` to `%amount1 GBP`. + +Overall, we will end up with one of the two alternatives for `amount1` - either `%amount1 %currency` or `%amount1 GBP`. + +Now substitution of all referenced values will happen, using the current values for `%amount1` and `currency`, which were provided by the `fields` directive. + + ## Reading multiple CSV files You can read multiple CSV files at once using multiple `-f` arguments on the command line, From 3c7d5d466d0b61ff495a065ece75673e72a202d6 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Tue, 15 Oct 2019 23:47:19 +0100 Subject: [PATCH 36/50] lib, doc, test: csv parser gains "end" command for "if" block --- hledger-lib/Hledger/Read/CsvReader.hs | 13 +++++++------ hledger-lib/hledger_csv.m4.md | 4 ++-- tests/csv.test | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index ab56feb8d..e221be515 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -222,11 +222,11 @@ validateCsv rules numhdrlines (Right rs) = validate $ applyConditionalSkips $ dr where filternulls = filter (/=[""]) skipCount r = - case getEffectiveAssignment rules r "skip" of - Nothing -> Nothing - Just "" -> Just 1 - Just "end" -> Just maxBound - Just x -> Just (read x) + case (getEffectiveAssignment rules r "end", getEffectiveAssignment rules r "skip") of + (Nothing, Nothing) -> Nothing + (Just _, _) -> Just maxBound + (Nothing, Just "") -> Just 1 + (Nothing, Just x) -> Just (read x) applyConditionalSkips [] = [] applyConditionalSkips (r:rest) = case skipCount r of @@ -601,7 +601,8 @@ journalfieldnames = ,"date" ,"description" ,"status" - ,"skip" -- skip is not really a field, but we list it here to allow conditional rules that skip records + ,"skip" -- skip and end are not really fields, but we list it here to allow conditional rules that skip records + ,"end" ] assignmentseparatorp :: CsvRulesParser () diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index 18b5d6ca1..1659ded16 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -194,7 +194,7 @@ Note, interpolation strips any outer whitespace, so a CSV value like `if` *`PATTERN`*\ *`PATTERN`*...\ -    *`skip end`*... +    *`end`*... This applies one or more field assignments, only to those CSV records matched by one of the PATTERNs. The patterns are case-insensitive regular expressions which match anywhere @@ -203,7 +203,7 @@ specific field). When there are multiple patterns they can be written on separate lines, unindented. The field assignments are on separate lines indented by at least one space. -Instead of field assignments you can specify `skip N` to skip the next N records (including the one that matchied) or `skip end` to skip the rest of the file. +Instead of field assignments you can specify `skip` or `skip 1` to skip this record, `skip N` to skip the next N records (including the one that matchied) or `end` to skip the rest of the file. Examples: ```rules diff --git a/tests/csv.test b/tests/csv.test index f9c2c2bb5..831e48d79 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -372,7 +372,7 @@ if HEADER if END OF FILE - skip end + end if MIDDLE skip 3 From 977592e49c132449af719d5ae653e46442c1bddd Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Tue, 15 Oct 2019 23:52:38 +0100 Subject: [PATCH 37/50] lib: typo fix, "expense:unknown" -> "expenses:unknown" --- hledger-lib/Hledger/Read/CsvReader.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index e221be515..29525c162 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -823,7 +823,7 @@ transactionFromCsvRecord sourcepos rules record = t then account else case isNegativeMixedAmount (pamount p) of Just True -> "income:unknown" - Just False -> "expense:unknown" + Just False -> "expenses:unknown" _ -> "unknown" ]} -- build the transaction From 490d2407f4f2c916e54f6e5396201e42561d8e40 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Wed, 16 Oct 2019 00:48:53 +0100 Subject: [PATCH 38/50] test: expense:unknown -> expenses:unknown --- tests/csv.test | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/csv.test b/tests/csv.test index 831e48d79..4de6548ec 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -34,7 +34,7 @@ $ ./hledger-csv 2009/09/11 Flubber Co🎅 Assets:MyAccount $-50 - expense:unknown + expenses:unknown >=0 @@ -165,7 +165,7 @@ $ ./hledger-csv --separator ';' 2009/09/11 Flubber Co🎅 Assets:MyAccount $-50 - expense:unknown + expenses:unknown >=0 @@ -262,7 +262,7 @@ $ ./hledger-csv 2009/09/11 Flubber Co🎅 Assets:MyAccount $-50 - expense:unknown + expenses:unknown >=0 @@ -306,8 +306,8 @@ if Flubber amount-in (%amount-in) $ ./hledger-csv 2009/09/10 Flubber Co - assets:myacct $-50 - expense:unknown + assets:myacct $-50 + expenses:unknown >=0 @@ -346,7 +346,7 @@ amount %2 date %1 date-format %Y/%m/%d $ ./hledger-csv | hledger balance -f - --no-total - $1,001.00 expense:unknown + $1,001.00 expenses:unknown $-1,001.00 income:unknown >=0 @@ -468,7 +468,7 @@ $ ./hledger-csv # $ ./hledger-csv # 2009/09/10 Flubber Co🎅 # Assets:MyAccount $50 -# expense:unknown $-50 +# expenses:unknown $-50 # # 2009/09/11 Flubber Co🎅 # Assets:MyAccount $-50 @@ -491,7 +491,7 @@ $ ./hledger-csv # $ ./hledger-csv # 2009/09/10 Flubber Co🎅 # Assets:MyAccount $50 -# expense:unknown $-50 +# expenses:unknown $-50 # # 2009/09/11 Flubber Co🎅 # Assets:MyAccount $-50 From 637741a7551a007aa9d26b441af61445d5e2a5f9 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Wed, 16 Oct 2019 23:49:28 +0100 Subject: [PATCH 39/50] lib: amount1 is no longer magical --- hledger-lib/Hledger/Read/CsvReader.hs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 29525c162..11e4cc2f8 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -465,22 +465,8 @@ parseCsvRules rulesfile s = validateRules :: CsvRules -> Either String CsvRules validateRules rules = do unless (isAssigned "date") $ Left "Please specify (at top level) the date field. Eg: date %1\n" - unless ((amount && not (amountin || amountout)) || - (not amount && (amountin && amountout)) || - balance) - $ Left $ unlines [ - "Please specify (as a top level CSV rule) either the amount1 field," - ,"both the amount1-in and amount1-out fields, or the balance1 field. Eg:" - ,"amount1 %2\n" - ,"You can also use amount, or both amount-in and amount-out, or balance," - ,"though this syntax is considered legacy." - ] Right rules where - amount = isAssigned "amount" || isAssigned "amount1" - amountin = isAssigned "amount-in" || isAssigned "amount1-in" - amountout = isAssigned "amount-out" || isAssigned "amount1-out" - balance = isAssigned "balance" || isAssigned "balance1" isAssigned f = isJust $ getEffectiveAssignment rules [] f -- parsers From 8d24a401504b6aa0d5a125073c346c9243bc380f Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Thu, 17 Oct 2019 00:02:26 +0100 Subject: [PATCH 40/50] lib: csv parser fills out amounts on all postings, if possible --- hledger-lib/Hledger/Read/CsvReader.hs | 7 +++- tests/csv.test | 49 ++++++++++++++------------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 11e4cc2f8..fe7cf5469 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -801,7 +801,7 @@ transactionFromCsvRecord sourcepos rules record = t Right balanced -> -- If we managed to balance transaction, lets infer better names for all "unknown" accounts t' {tpostings = - [ originalPosting {paccount=newAccount} + [ originalPosting {paccount=newAccount, pamount=newAmount} | (originalPosting,p) <- zip postings (tpostings balanced) , let account = paccount p , let newAccount = @@ -811,6 +811,11 @@ transactionFromCsvRecord sourcepos rules record = t Just True -> "income:unknown" Just False -> "expenses:unknown" _ -> "unknown" + , let newAmount = + if pamount originalPosting == missingmixedamt && + pamount p /= missingmixedamt + then pamount p + else pamount originalPosting ]} -- build the transaction t' = nulltransaction{ diff --git a/tests/csv.test b/tests/csv.test index 4de6548ec..916e8c565 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -10,7 +10,7 @@ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 - income:unknown + income:unknown $-50 >=0 @@ -30,11 +30,11 @@ currency $ $ ./hledger-csv 2009/09/10 Flubber Co🎅 Assets:MyAccount $50 - income:unknown + income:unknown $-50 2009/09/11 Flubber Co🎅 Assets:MyAccount $-50 - expenses:unknown + expenses:unknown $50 >=0 @@ -53,7 +53,7 @@ if Flubber $ ./hledger-csv 2009/09/10 Flubber Co ; cmt assets:myacct $50 - acct + acct $-50 >=0 @@ -70,7 +70,7 @@ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $123 - income:unknown + income:unknown $-50 >=0 @@ -88,11 +88,11 @@ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $123 - income:unknown + income:unknown $-50 2009/09/11 Blubber Co assets:myacct $60 - income:unknown + income:unknown $-60 >=0 @@ -110,11 +110,11 @@ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $123 - income:unknown + income:unknown $-50 2009/09/11 Blubber Co assets:myacct $60 - income:unknown + income:unknown $-60 >=0 @@ -137,11 +137,11 @@ account2 expense:other $ ./hledger-csv 2009/10/09 liabilities:bank $-50 - expense:other + expense:other $50 2009/11/09 liabilities:bank $60 - expense:other + expense:other $-60 >=0 @@ -161,11 +161,11 @@ currency $ $ ./hledger-csv --separator ';' 2009/09/10 Flubber Co🎅 Assets:MyAccount $50 - income:unknown + income:unknown $-50 2009/09/11 Flubber Co🎅 Assets:MyAccount $-50 - expenses:unknown + expenses:unknown $50 >=0 @@ -182,7 +182,7 @@ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 - income:unknown = $123 + income:unknown $-50 = $123 >=0 @@ -199,7 +199,7 @@ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $321 - income:unknown = $123 + income:unknown $-50 = $123 >=0 @@ -218,7 +218,7 @@ account3 expenses:tax $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $321 - income:unknown = $123 + income:unknown $-50.234 = $123 expenses:tax $0.234 ; VAT >=0 @@ -237,7 +237,8 @@ account3 expenses:tax $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $321 - unknown = $123 + unknown $-50 = $123 + unknown £-0.234 = $123 expenses:tax £0.234 ; VAT >=0 @@ -258,11 +259,11 @@ currency $ $ ./hledger-csv 2009/09/10 Flubber Co🎅 Assets:MyAccount $50 - income:unknown + income:unknown $-50 2009/09/11 Flubber Co🎅 Assets:MyAccount $-50 - expenses:unknown + expenses:unknown $50 >=0 @@ -288,7 +289,7 @@ account1 assets:myacct $ ./hledger-csv 2009/09/10 Flubber Co Co Co assets:myacct $50 - income:unknown + income:unknown $-50 >=0 @@ -307,7 +308,7 @@ if Flubber $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $-50 - expenses:unknown + expenses:unknown $50 >=0 @@ -331,7 +332,7 @@ description %description for %itemtitle $ ./hledger-csv 2018/12/22 (123456789) Someone for Joyful Systems sm:assets:online:paypal $9.41 = $57.60 - sm:expenses:unknown + sm:expenses:unknown $-8.82 JS:expenses:banking:paypal $-0.59 >=0 @@ -380,11 +381,11 @@ if MIDDLE $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 - income:unknown + income:unknown $-50 2009/09/10 Flubber Co assets:myacct $50 - income:unknown + income:unknown $-50 >=0 From 4b30a70c720933c046077fc80556ad5579b4723a Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Thu, 17 Oct 2019 00:04:23 +0100 Subject: [PATCH 41/50] doc: posting1 is not magical in csv rules --- hledger-lib/hledger_csv.m4.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index 1659ded16..c31d640e3 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -143,7 +143,7 @@ An assignment to any of `accountN`, `amountN`, `amountN-in`, `amountN-out`, `bal Fields `amount`, `amount-in`, `amount-out`, `currency`, `balance` and `comment` are treated as aliases for `amount1`, and so on. If your rules file leads to both aliased fields having different values, `hledger` will raise an error. -You need to provide enough information to create at least one posting. +You need to provide enough information to create at least one posting (not necessary number 1). Eg: ```rules From 91eb899b82228f3f1f4bd765c34577bdc9c26f09 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Thu, 17 Oct 2019 21:47:55 +0100 Subject: [PATCH 42/50] doc: advise on the default account selection --- hledger-lib/hledger_csv.m4.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index c31d640e3..7bb33c2e5 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -253,8 +253,11 @@ The order of same-day entries will be preserved ## CSV accounts -Each journal entry will have at least two [postings](journal.html#postings), to `account1` and some other account (usually `account2`). -It's conventional and recommended to use `account1` for the account whose CSV we are reading. +Each journal entry will have at least two [postings](journal.html#postings), usually to `account1` and `account2`. + +It is entirely up to you which account field to assign to the account whose CSV we are reading, however you might +consider assigning it to the last posting (`account2` in the simplest case). This way, if the CSV contrains transactions in multiple currencies, +and you don't provide posting [prices](journal.html#transaction-prices) explicitly they will be priced in the currency of the last posting. ## CSV amounts From 44f01cbd946d0372a0477e66a5a065e7a4397af0 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Thu, 17 Oct 2019 23:25:22 +0100 Subject: [PATCH 43/50] docs: revert advise on account, postpone for now --- hledger-lib/hledger_csv.m4.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index 7bb33c2e5..c31d640e3 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -253,11 +253,8 @@ The order of same-day entries will be preserved ## CSV accounts -Each journal entry will have at least two [postings](journal.html#postings), usually to `account1` and `account2`. - -It is entirely up to you which account field to assign to the account whose CSV we are reading, however you might -consider assigning it to the last posting (`account2` in the simplest case). This way, if the CSV contrains transactions in multiple currencies, -and you don't provide posting [prices](journal.html#transaction-prices) explicitly they will be priced in the currency of the last posting. +Each journal entry will have at least two [postings](journal.html#postings), to `account1` and some other account (usually `account2`). +It's conventional and recommended to use `account1` for the account whose CSV we are reading. ## CSV amounts From a6d91da217ea5798bb1d9ee84db62f051fcdc16f Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Fri, 18 Oct 2019 23:08:16 +0100 Subject: [PATCH 44/50] doc: remove "you need to create at least 1 posting" bit --- hledger-lib/hledger_csv.m4.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index c31d640e3..e3ca6fc39 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -143,8 +143,6 @@ An assignment to any of `accountN`, `amountN`, `amountN-in`, `amountN-out`, `bal Fields `amount`, `amount-in`, `amount-out`, `currency`, `balance` and `comment` are treated as aliases for `amount1`, and so on. If your rules file leads to both aliased fields having different values, `hledger` will raise an error. -You need to provide enough information to create at least one posting (not necessary number 1). - Eg: ```rules # use the 1st, 2nd and 4th CSV fields as the entry's date, description and amount, From d62f84bec2b42efaad33a7f99442f6eef6cb489b Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 19 Oct 2019 00:32:14 +0100 Subject: [PATCH 45/50] lib: csv reader does not generate postings if account="" --- hledger-lib/Hledger/Read/CsvReader.hs | 3 ++ tests/csv.test | 40 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index fe7cf5469..915cd04c9 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -735,6 +735,9 @@ transactionFromCsvRecord sourcepos rules record = t comment = T.pack $ maybe "" render $ mfieldtemplate commentFld account = case account' of + -- If account is explicitly "unassigned", suppress posting + -- Otherwise, generate posting with "unknown" account if we have amount/balance information + Just "" -> Nothing Just account -> Just account Nothing -> -- If we have amount or balance assertion (which implies potential amount change), diff --git a/tests/csv.test b/tests/csv.test index 916e8c565..6c48b3c53 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -450,6 +450,46 @@ $ ./hledger-csv >=0 +# 21. Amountless postings and conditional third posting +< +"12/22/2018","06:22:50","PST","Someone","Subscription Payment","Completed","USD","10.00","-0.59","9.41","someone@some.where","simon@joyful.com","123456789","Joyful Systems","","9KCXINCOME:UNKNOWNZXXAX","","57.60","" +"12/22/2018","06:22:50","PST","Someone","Empty fee","Completed","USD","10.00","","6.66","someone@some.where","simon@joyful.com","987654321","Joyful Systems","","9KCXINCOME:UNKNOWNZXXAX","","99.60","" +"12/22/2018","06:22:50","PST","Someone","Conditional Empty fee","Completed","USD","10.00","-1.23","7.77","someone@some.where","simon@joyful.com","10101010101","Joyful Systems","","9KCXINCOME:UNKNOWNZXXAX","","88.66","" + +RULES +fields date, time, timezone, description, type, status_, currency, grossamount, feeamount, netamount, fromemail, toemail, code, itemtitle, itemid, referencetxnid, receiptid, balance, note +account1 sm:assets:online:paypal +amount1 %netamount +account2 sm:expenses:unknown +account3 JS:expenses:banking:paypal +amount3 %feeamount +balance %18 +code %13 +currency $ +date %1 +date-format %m/%d/%Y +description %description for %itemtitle +if Conditional Empty Fee + account3 + +$ ./hledger-csv +2018/12/22 (123456789) Someone for Joyful Systems + sm:assets:online:paypal $9.41 = $57.60 + sm:expenses:unknown $-8.82 + JS:expenses:banking:paypal $-0.59 + +2018/12/22 (987654321) Someone for Joyful Systems + sm:assets:online:paypal $6.66 = $99.60 + sm:expenses:unknown + JS:expenses:banking:paypal + +2018/12/22 (10101010101) Someone for Joyful Systems + sm:assets:online:paypal $7.77 = $88.66 + sm:expenses:unknown $-7.77 + +>=0 + + # . TODO: without --separator gives obscure error # | # 1 | 10/2009/09;Flubber Co🎅;50; From 95ec5715cca661e7746e6499f24f0d14fe6aa4dc Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 19 Oct 2019 00:49:55 +0100 Subject: [PATCH 46/50] lib: better compatibility code in csv reader --- hledger-lib/Hledger/Read/CsvReader.hs | 28 +++++++++++++-------------- tests/csv.test | 9 ++++----- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index 915cd04c9..f66e4ff17 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -750,7 +750,7 @@ transactionFromCsvRecord sourcepos rules record = t case account of Nothing -> Nothing Just account -> - Just $ posting {paccount=account, pamount=fromMaybe missingmixedamt amount, ptransaction=Just t', pbalanceassertion=toAssertion <$> balance, pcomment = comment} + Just $ (number, posting {paccount=account, pamount=fromMaybe missingmixedamt amount, ptransaction=Just t', pbalanceassertion=toAssertion <$> balance, pcomment = comment}) parsePosting number = parsePosting' number @@ -765,9 +765,9 @@ transactionFromCsvRecord sourcepos rules record = t posting1' = parsePosting "1" posting1 = case (postingLegacy,posting1') of - (Just legacy, Nothing) -> Just legacy - (Nothing, Just posting1) -> Just posting1 - (Just legacy, Just posting1) -> + (Just (_,legacy), Nothing) -> Just ("1", legacy) + (Nothing, Just (_,posting1)) -> Just ("1", posting1) + (Just (_,legacy), Just (_,posting1)) -> -- Here we merge legacy fields such as "amount" with "amount1", etc -- Account and Comment would be the same by construction let balanceassertion = (pbalanceassertion legacy) `or` (pbalanceassertion posting1) @@ -787,15 +787,20 @@ transactionFromCsvRecord sourcepos rules record = t , "amount/amount-in/amount-out is " ++ showMixedAmount al , "amount1/amount1-in/amount1-out is " ++ showMixedAmount a1 ] - in Just $ posting {paccount=paccount posting1, pamount=amount, ptransaction=Just t', pbalanceassertion=balanceassertion, pcomment = pcomment posting1} + in Just $ ("1", posting {paccount=paccount posting1, pamount=amount, ptransaction=Just t', pbalanceassertion=balanceassertion, pcomment = pcomment posting1}) (Nothing, Nothing) -> Nothing postings' = catMaybes $ posting1:[ parsePosting i | x<-[2..9], let i = show x] postings = case postings' of -- To be compatible with the behavior of the old code which allowed two postings only, we enforce - -- second posting when rules generated just one of them. - [posting1] -> [posting1,posting{paccount="unknown", pamount=missingmixedamt, ptransaction=Just t'}] - _ -> postings' + -- second posting when rules generated just first of them. + -- When we have srictly first and second posting, but second posting does not have amount, we fill it in. + [("1",posting1)] -> [posting1,posting{paccount="unknown", pamount=costOfMixedAmount(-(pamount posting1)), ptransaction=Just t'}] + [("1",posting1),("2",posting2)] -> + case (pamount posting1 == missingmixedamt , pamount posting2 == missingmixedamt) of + (False, True) -> [posting1, posting2{pamount=costOfMixedAmount(-(pamount posting1))}] + _ -> [posting1, posting2] + _ -> map snd postings' balanced = balanceTransaction Nothing t' t = @@ -804,7 +809,7 @@ transactionFromCsvRecord sourcepos rules record = t Right balanced -> -- If we managed to balance transaction, lets infer better names for all "unknown" accounts t' {tpostings = - [ originalPosting {paccount=newAccount, pamount=newAmount} + [ originalPosting {paccount=newAccount} | (originalPosting,p) <- zip postings (tpostings balanced) , let account = paccount p , let newAccount = @@ -814,11 +819,6 @@ transactionFromCsvRecord sourcepos rules record = t Just True -> "income:unknown" Just False -> "expenses:unknown" _ -> "unknown" - , let newAmount = - if pamount originalPosting == missingmixedamt && - pamount p /= missingmixedamt - then pamount p - else pamount originalPosting ]} -- build the transaction t' = nulltransaction{ diff --git a/tests/csv.test b/tests/csv.test index 6c48b3c53..ca99ac413 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -218,7 +218,7 @@ account3 expenses:tax $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $321 - income:unknown $-50.234 = $123 + income:unknown = $123 expenses:tax $0.234 ; VAT >=0 @@ -237,8 +237,7 @@ account3 expenses:tax $ ./hledger-csv 2009/09/10 Flubber Co assets:myacct $50 = $321 - unknown $-50 = $123 - unknown £-0.234 = $123 + unknown = $123 expenses:tax £0.234 ; VAT >=0 @@ -332,7 +331,7 @@ description %description for %itemtitle $ ./hledger-csv 2018/12/22 (123456789) Someone for Joyful Systems sm:assets:online:paypal $9.41 = $57.60 - sm:expenses:unknown $-8.82 + sm:expenses:unknown JS:expenses:banking:paypal $-0.59 >=0 @@ -475,7 +474,7 @@ if Conditional Empty Fee $ ./hledger-csv 2018/12/22 (123456789) Someone for Joyful Systems sm:assets:online:paypal $9.41 = $57.60 - sm:expenses:unknown $-8.82 + sm:expenses:unknown JS:expenses:banking:paypal $-0.59 2018/12/22 (987654321) Someone for Joyful Systems From c66ccc5cee27ea4348f6d0f03bc677b26be04077 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 19 Oct 2019 01:18:07 +0100 Subject: [PATCH 47/50] lib: do not try to balance transaction in csv reader --- hledger-lib/Hledger/Read/CsvReader.hs | 38 +++++++++++---------------- tests/csv.test | 8 +++--- 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index f66e4ff17..b4568c899 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -750,7 +750,7 @@ transactionFromCsvRecord sourcepos rules record = t case account of Nothing -> Nothing Just account -> - Just $ (number, posting {paccount=account, pamount=fromMaybe missingmixedamt amount, ptransaction=Just t', pbalanceassertion=toAssertion <$> balance, pcomment = comment}) + Just $ (number, posting {paccount=account, pamount=fromMaybe missingmixedamt amount, ptransaction=Just t, pbalanceassertion=toAssertion <$> balance, pcomment = comment}) parsePosting number = parsePosting' number @@ -787,41 +787,33 @@ transactionFromCsvRecord sourcepos rules record = t , "amount/amount-in/amount-out is " ++ showMixedAmount al , "amount1/amount1-in/amount1-out is " ++ showMixedAmount a1 ] - in Just $ ("1", posting {paccount=paccount posting1, pamount=amount, ptransaction=Just t', pbalanceassertion=balanceassertion, pcomment = pcomment posting1}) + in Just $ ("1", posting {paccount=paccount posting1, pamount=amount, ptransaction=Just t, pbalanceassertion=balanceassertion, pcomment = pcomment posting1}) (Nothing, Nothing) -> Nothing postings' = catMaybes $ posting1:[ parsePosting i | x<-[2..9], let i = show x] + + improveUnknownAccountName p = + if paccount p /="unknown" + then p + else case isNegativeMixedAmount (pamount p) of + Just True -> p{paccount = "income:unknown"} + Just False -> p{paccount = "expenses:unknown"} + _ -> p + postings = case postings' of -- To be compatible with the behavior of the old code which allowed two postings only, we enforce -- second posting when rules generated just first of them. -- When we have srictly first and second posting, but second posting does not have amount, we fill it in. - [("1",posting1)] -> [posting1,posting{paccount="unknown", pamount=costOfMixedAmount(-(pamount posting1)), ptransaction=Just t'}] + [("1",posting1)] -> + [posting1,improveUnknownAccountName (posting{paccount="unknown", pamount=costOfMixedAmount(-(pamount posting1)), ptransaction=Just t})] [("1",posting1),("2",posting2)] -> case (pamount posting1 == missingmixedamt , pamount posting2 == missingmixedamt) of - (False, True) -> [posting1, posting2{pamount=costOfMixedAmount(-(pamount posting1))}] + (False, True) -> [posting1, improveUnknownAccountName (posting2{pamount=costOfMixedAmount(-(pamount posting1))})] _ -> [posting1, posting2] _ -> map snd postings' - balanced = balanceTransaction Nothing t' - t = - case balanced of - Left _ -> t' - Right balanced -> - -- If we managed to balance transaction, lets infer better names for all "unknown" accounts - t' {tpostings = - [ originalPosting {paccount=newAccount} - | (originalPosting,p) <- zip postings (tpostings balanced) - , let account = paccount p - , let newAccount = - if account/="unknown" - then account - else case isNegativeMixedAmount (pamount p) of - Just True -> "income:unknown" - Just False -> "expenses:unknown" - _ -> "unknown" - ]} -- build the transaction - t' = nulltransaction{ + t = nulltransaction{ tsourcepos = genericSourcePos sourcepos, tdate = date', tdate2 = mdate2', diff --git a/tests/csv.test b/tests/csv.test index ca99ac413..dfb5f1e47 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -217,9 +217,9 @@ account3 expenses:tax $ ./hledger-csv 2009/09/10 Flubber Co - assets:myacct $50 = $321 - income:unknown = $123 - expenses:tax $0.234 ; VAT + assets:myacct $50 = $321 + unknown = $123 + expenses:tax $0.234 ; VAT >=0 @@ -346,8 +346,8 @@ amount %2 date %1 date-format %Y/%m/%d $ ./hledger-csv | hledger balance -f - --no-total - $1,001.00 expenses:unknown $-1,001.00 income:unknown + $1,001.00 unknown >=0 # 18. Conditional skips From 09f17f2b2b7e468308dd5fc6aa0b7682aa34b980 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 19 Oct 2019 01:30:18 +0100 Subject: [PATCH 48/50] doc: mention the magic sauce --- hledger-lib/hledger_csv.m4.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hledger-lib/hledger_csv.m4.md b/hledger-lib/hledger_csv.m4.md index e3ca6fc39..4055c857f 100644 --- a/hledger-lib/hledger_csv.m4.md +++ b/hledger-lib/hledger_csv.m4.md @@ -154,6 +154,8 @@ Eg: fields date, description, , amount1, , , somefield, anotherfield ``` +For backwards compatibility, we treat posting 1 specially. If your rules generated just posting 1, another posting would be added to your transaction to balance it. If your rules generated posting 1 and posting 2, but amount in the posting 2 is empty, hledger will fill it out with the opposite of posting 1. This special handling is needed to ensure smooth upgrade path from version 1.15. + ## field assignment *`ENTRYFIELDNAME`* *`FIELDVALUE`* From fc001da1efc3e9002015971e41877538b6e42bbb Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Sat, 19 Oct 2019 01:33:16 +0100 Subject: [PATCH 49/50] lib: expenses:unknown is a much better default account name --- hledger-lib/Hledger/Read/CsvReader.hs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hledger-lib/Hledger/Read/CsvReader.hs b/hledger-lib/Hledger/Read/CsvReader.hs index b4568c899..1885e8e89 100644 --- a/hledger-lib/Hledger/Read/CsvReader.hs +++ b/hledger-lib/Hledger/Read/CsvReader.hs @@ -736,15 +736,15 @@ transactionFromCsvRecord sourcepos rules record = t account = case account' of -- If account is explicitly "unassigned", suppress posting - -- Otherwise, generate posting with "unknown" account if we have amount/balance information + -- Otherwise, generate posting with "expenses:unknown" account if we have amount/balance information Just "" -> Nothing Just account -> Just account Nothing -> -- If we have amount or balance assertion (which implies potential amount change), - -- but no account name, lets generate "unknown" account name. + -- but no account name, lets generate "expenses:unknown" account name. case (amount, balance) of - (Just _, _ ) -> Just "unknown" - (_, Just _) -> Just "unknown" + (Just _, _ ) -> Just "expenses:unknown" + (_, Just _) -> Just "expenses:unknown" (Nothing, Nothing) -> Nothing in case account of @@ -792,7 +792,7 @@ transactionFromCsvRecord sourcepos rules record = t postings' = catMaybes $ posting1:[ parsePosting i | x<-[2..9], let i = show x] improveUnknownAccountName p = - if paccount p /="unknown" + if paccount p /="expenses:unknown" then p else case isNegativeMixedAmount (pamount p) of Just True -> p{paccount = "income:unknown"} @@ -805,7 +805,7 @@ transactionFromCsvRecord sourcepos rules record = t -- second posting when rules generated just first of them. -- When we have srictly first and second posting, but second posting does not have amount, we fill it in. [("1",posting1)] -> - [posting1,improveUnknownAccountName (posting{paccount="unknown", pamount=costOfMixedAmount(-(pamount posting1)), ptransaction=Just t})] + [posting1,improveUnknownAccountName (posting{paccount="expenses:unknown", pamount=costOfMixedAmount(-(pamount posting1)), ptransaction=Just t})] [("1",posting1),("2",posting2)] -> case (pamount posting1 == missingmixedamt , pamount posting2 == missingmixedamt) of (False, True) -> [posting1, improveUnknownAccountName (posting2{pamount=costOfMixedAmount(-(pamount posting1))})] From fb5bca0b46b9536f62955033ac20075af5aaf985 Mon Sep 17 00:00:00 2001 From: Dmitry Astapov Date: Tue, 5 Nov 2019 22:10:41 +0000 Subject: [PATCH 50/50] test: update csv tests for expenses:unknown being new default --- tests/csv.test | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/csv.test b/tests/csv.test index dfb5f1e47..174d34740 100644 --- a/tests/csv.test +++ b/tests/csv.test @@ -217,9 +217,9 @@ account3 expenses:tax $ ./hledger-csv 2009/09/10 Flubber Co - assets:myacct $50 = $321 - unknown = $123 - expenses:tax $0.234 ; VAT + assets:myacct $50 = $321 + expenses:unknown = $123 + expenses:tax $0.234 ; VAT >=0 @@ -236,9 +236,9 @@ account3 expenses:tax $ ./hledger-csv 2009/09/10 Flubber Co - assets:myacct $50 = $321 - unknown = $123 - expenses:tax £0.234 ; VAT + assets:myacct $50 = $321 + expenses:unknown = $123 + expenses:tax £0.234 ; VAT >=0