imp: bal: more predictable sort order with multiple commodities (#1563, #1564)

This change provides more predictable and intuitive behaviour when 
using -S/--sort-amount with multiple commodities. 
It implements a custom Ord (and Eq) instance for MixedAmount 
which substitutes zero for any missing commodities.

As a consequence, all the ways of representing zero with a MixedAmount ([],
[A 0], [A 0, B 0, ...]) are now Eq-ual (==), whereas before they were
not. We have not been able to find anything broken by this change.

* imp: lib: Compare MixedAmounts by substituting zero for any missing commodities. (#1563)
* ;doc: Update docs for new multicommodity sort by amount rules.

hledger-lib/Hledger/Data/Amount.hs

@@ -93,6 +93,7 @@ module Hledger.Data.Amount (
   -- * MixedAmount
   nullmixedamt,
   missingmixedamt,
+  isMissingMixedAmount,
   mixed,
   mixedAmount,
   maAddAmount,
@@ -540,6 +541,10 @@ nullmixedamt = Mixed mempty
 missingmixedamt :: MixedAmount
 missingmixedamt = mixedAmount missingamt
 
+-- | Whether a MixedAmount has a missing amount
+isMissingMixedAmount :: MixedAmount -> Bool
+isMissingMixedAmount (Mixed ma) = amountKey missingamt `M.member` ma
+
 -- | Convert amounts in various commodities into a mixed amount.
 mixed :: Foldable t => t Amount -> MixedAmount
 mixed = maAddAmounts nullmixedamt
@@ -637,13 +642,12 @@ maIsNonZero = not . mixedAmountIsZero
 --
 amounts :: MixedAmount -> [Amount]
 amounts (Mixed ma)
-  | missingkey `M.member` ma = [missingamt]  -- missingamt should always be alone, but detect it even if not
+  | isMissingMixedAmount (Mixed ma) = [missingamt]  -- missingamt should always be alone, but detect it even if not
-  | M.null nonzeros          = [newzero]
+  | M.null nonzeros                 = [newzero]
-  | otherwise                = toList nonzeros
+  | otherwise                       = toList nonzeros
   where
     newzero = fromMaybe nullamt $ find (not . T.null . acommodity) zeros
     (zeros, nonzeros) = M.partition amountIsZero ma
-    missingkey = amountKey missingamt
 
 -- | Get a mixed amount's component amounts without normalising zero and missing
 -- amounts. This is used for JSON serialisation, so the order is important. In
@@ -966,7 +970,14 @@ tests_Amount = tests "Amount" [
 
   ,tests "MixedAmount" [
 
-     test "adding mixed amounts to zero, the commodity and amount style are preserved" $
+     test "comparing mixed amounts compares based on quantities" $ do
+       let usdpos = mixed [usd 1]
+           usdneg = mixed [usd (-1)]
+           eurneg = mixed [eur (-12)]
+       compare usdneg usdpos @?= LT
+       compare eurneg usdpos @?= LT
+
+     ,test "adding mixed amounts to zero, the commodity and amount style are preserved" $
       maSum (map mixedAmount
         [usd 1.25
         ,usd (-1) `withPrecision` Precision 3

hledger-lib/Hledger/Data/Posting.hs

@@ -183,7 +183,7 @@ isBalancedVirtual :: Posting -> Bool
 isBalancedVirtual p = ptype p == BalancedVirtualPosting
 
 hasAmount :: Posting -> Bool
-hasAmount = (/= missingmixedamt) . pamount
+hasAmount = not . isMissingMixedAmount . pamount
 
 hasBalanceAssignment :: Posting -> Bool
 hasBalanceAssignment p = not (hasAmount p) && isJust (pbalanceassertion p)

hledger-lib/Hledger/Data/Types.hs

@@ -27,8 +27,8 @@ module Hledger.Data.Types
 where
 
 import GHC.Generics (Generic)
-import Data.Decimal
+import Data.Decimal (Decimal)
-import Data.Default
+import Data.Default (Default(..))
 import Data.Functor (($>))
 import Data.List (intercalate)
 import Text.Blaze (ToMarkup(..))
@@ -230,7 +230,27 @@ data Amount = Amount {
       aprice      :: !(Maybe AmountPrice)  -- ^ the (fixed, transaction-specific) price for this amount, if any
     } deriving (Eq,Ord,Generic,Show)
 
-newtype MixedAmount = Mixed (M.Map MixedAmountKey Amount) deriving (Eq,Ord,Generic,Show)
+newtype MixedAmount = Mixed (M.Map MixedAmountKey Amount) deriving (Generic,Show)
+
+instance Eq  MixedAmount where a == b  = maCompare a b == EQ
+instance Ord MixedAmount where compare = maCompare
+
+-- | Compare two MixedAmounts, substituting 0 for the quantity of any missing
+-- commodities in either.
+maCompare :: MixedAmount -> MixedAmount -> Ordering
+maCompare (Mixed a) (Mixed b) = go (M.toList a) (M.toList b)
+  where
+    go xss@((kx,x):xs) yss@((ky,y):ys) = case compare kx ky of
+                 EQ -> compareQuantities (Just x) (Just y) <> go xs ys
+                 LT -> compareQuantities (Just x) Nothing  <> go xs yss
+                 GT -> compareQuantities Nothing  (Just y) <> go xss ys
+    go ((_,x):xs) [] = compareQuantities (Just x) Nothing  <> go xs []
+    go [] ((_,y):ys) = compareQuantities Nothing  (Just y) <> go [] ys
+    go []         [] = EQ
+    compareQuantities = comparing (maybe 0 aquantity) <> comparing (maybe 0 totalprice)
+    totalprice x = case aprice x of
+                        Just (TotalPrice p) -> aquantity p
+                        _                   -> 0
 
 -- | Stores the CommoditySymbol of the Amount, along with the CommoditySymbol of
 -- the price, and its unit price if being used.

hledger/Hledger/Cli/Commands/Balance.md

@@ -259,6 +259,9 @@ Here are some ways to handle that:
 
 With `-S/--sort-amount`, accounts with the largest (most positive) balances are shown first.
 Eg: `hledger bal expenses -MAS` shows your biggest averaged monthly expenses first.
+When more than one commodity is present, they will be sorted by the alphabetically earliest
+commodity first, and then by subsequent commodities (if an amount is missing a commodity, it
+is treated as 0).
 
 Revenues and liability balances are typically negative, however, so `-S` shows these in reverse order.
 To work around this, you can add `--invert` to flip the signs.

hledger/test/balance/sorting.test

@@ -15,7 +15,7 @@
 2018/1/1
   (a:k)  1
 
-# In tree mode, rows are sorted alphabetically by account name, at each tree level. 
+# 1. In tree mode, rows are sorted alphabetically by account name, at each tree level. 
 # Missing parent accounts are added (b).
 $ hledger -f- bal -N --tree
                    1  a:k
@@ -24,7 +24,7 @@ $ hledger -f- bal -N --tree
                    1    j
                    1  c
 
-# In flat mode, unused parent accounts are not added (b).
+# 2. In flat mode, unused parent accounts are not added (b).
 $ hledger -f- bal -N --flat
                    1  a:k
                    1  b:i
@@ -56,7 +56,7 @@ account d
 2018/1/1
   (d)    1
 
-# With account directives, in tree mode, at each tree level
+# 3. With account directives, in tree mode, at each tree level
 # declared accounts are sorted first in declaration order, 
 # followed by undeclared accounts sorted alphabetically.
 # Missing parent accounts are added (b).
@@ -70,7 +70,7 @@ $ hledger -f- bal -N --tree
                    1    i
                    1  c
 
-# In flat mode, unused parent accounts are not added (b).
+# 4. In flat mode, unused parent accounts are not added (b).
 $ hledger -f- bal -N --flat
                    1  d
                    1  a:l
@@ -81,7 +81,7 @@ $ hledger -f- bal -N --flat
 
 # ** Sort by amount
 
-# In flat mode with -S, largest (most-positive) amounts are shown first:
+# 5. In flat mode with -S, largest (most-positive) amounts are shown first:
 <
 2018/1/1
   (b:j)  2
@@ -101,7 +101,7 @@ $ hledger -f- bal -N -S --flat
                    1  b:i
                    1  c
 
-# In tree mode with -S, rows are sorted by largest amount, and then by account name, at each tree level.
+# 6. In tree mode with -S, rows are sorted by largest amount, and then by account name, at each tree level.
 $ hledger -f- bal -N -S --tree
                    3  b
                    2    j
@@ -109,8 +109,8 @@ $ hledger -f- bal -N -S --tree
                    1  a:k
                    1  c
 
-# In hledger 1.4-1.10, when the larger amount was composed of differently-priced amounts,
+# 7. In hledger 1.4-1.10, when the larger amount was composed of differently-priced amounts,
-# it could get sorted as if smaller. Should work now:
+# it could get sorted as if smaller. Should work now. Test tree mode.
 <
 2018/1/1
   (a)  2X @ 1Y
@@ -119,11 +119,16 @@ $ hledger -f- bal -N -S --tree
 2018/1/1
   (b)  3X
 
-$ hledger -f- bal -N -S
+$ hledger -f- bal -N -S --tree
                   4X  a
                   3X  b
 
-# #1279 in hledger 1.11-1.18, bal -S did not respect the hierarchy. Should work now:
+# 8. Same in flat mode.
+$ hledger -f- bal -N -S --flat
+                  4X  a
+                  3X  b
+
+# 9. #1279 in hledger 1.11-1.18, bal -S did not respect the hierarchy. Should work now:
 <
 2020-01-01
     (a:aa)                                         1
@@ -136,19 +141,19 @@ $ hledger -f- bal -N -S --tree
                    1    aa
                    2  b
 
-# #1287 bal -S -H did not sort by amount, should work now:
+# 10. #1287 bal -S -H did not sort by amount, should work now:
 $ hledger -f- bal -N -S -H --flat
                    3  a:ab
                    2  b
                    1  a:aa
 
-# #1287 and bal -S --cumulative:
+# 11. #1287 and bal -S --cumulative:
 $ hledger -f- bal -N -S --cumulative --flat
                    3  a:ab
                    2  b
                    1  a:aa
 
-# #1283 most-negative amounts are sorted last, so eg largest revenues/liabilities are last:
+# 12. #1283 most-negative amounts are sorted last, so eg largest revenues/liabilities are last:
 <
 2020-01-01
   (revenues:a)  -1
@@ -160,13 +165,13 @@ $ hledger -f- bal -N -S
                   -2  revenues:c
                   -3  revenues:b
 
-# This can be worked around by using --invert (sorting happens after sign-flipping):
+# 13. This can be worked around by using --invert (sorting happens after sign-flipping):
 $ hledger -f- bal -N -S --invert
                    3  revenues:b
                    2  revenues:c
                    1  revenues:a
 
-# Or a sign-flipping command like incomestatement:
+# 14. Or a sign-flipping command like incomestatement:
 $ hledger -f- is -N -S
 Income Statement 2020-01-01
 
@@ -180,3 +185,46 @@ Income Statement 2020-01-01
 ============++============
  Expenses   ||            
 ------------++------------
+
+<
+2021-01-01 Post in X
+    (f)   1 X
+
+2021-01-01 Post in X
+    (b)  -1 X
+
+2021-01-01 Post in X
+    (e)   1 X
+    (e)  -1 Y
+
+2021-01-01 Post in Y
+    (d)  -1 Y
+
+2021-01-01 Post in X
+    (c)  -1 X
+
+2021-01-01 Post in X
+    (a)  -1 X
+
+# 15. When sorting by amount with different commodities, missing commodities are
+# treated as 0, so negative amounts go before positive amounts. (#1563)
+#
+# Explanation: We treats these amounts as
+#              1 X,  0 Y  f
+#              1 X, -1 Y  e
+#              0 X, -1 Y  d
+#             -1 X,  0 Y  a
+#             -1 X,  0 Y  b
+#             -1 X,  0 Y  c
+# X is alphabetically before Y, so the X amount determines the order.
+# If X amounts are equal, the Y amount will decide, and so on.
+# If all commodity amounts are equal (a, b, and c above), their sort order is
+# determined by their account name.
+$ hledger -f- bal -N -S
+                 1 X  f
+                 1 X
+                -1 Y  e
+                -1 Y  d
+                -1 X  a
+                -1 X  b
+                -1 X  c