From a5ec1cb88836fcfe1e7afe0b69fb4487dcb0a67b Mon Sep 17 00:00:00 2001 From: Simon Michael Date: Thu, 28 Aug 2025 04:57:17 +0100 Subject: [PATCH] ;bin: paypal*, simplefin*: rename, improve UX, update to 1.1 --- bin/README.md | 12 ++-- bin/{paypaljson2csv => paypalcsv} | 8 +-- bin/simplefincsv | 104 ++++++++++++++++++++++++++++++ bin/simplefinjson | 10 +-- bin/simplefinjson2csv | 77 ---------------------- 5 files changed, 120 insertions(+), 91 deletions(-) rename bin/{paypaljson2csv => paypalcsv} (97%) create mode 100755 bin/simplefincsv delete mode 100755 bin/simplefinjson2csv diff --git a/bin/README.md b/bin/README.md index 6f93e1297..c1ed91ea4 100644 --- a/bin/README.md +++ b/bin/README.md @@ -49,25 +49,25 @@ is a small script to make it show up in the hledger commands list. [`paypaljson`](https://github.com/simonmichael/hledger/blob/master/bin/paypaljson) downloads the last 30 days of Paypal transactions (requires a free developer account & API key). -### paypaljson2csv +### paypalcsv -[`paypaljson2csv`](https://github.com/simonmichael/hledger/blob/master/bin/paypaljson2csv) (python) +[`paypalcsv`](https://github.com/simonmichael/hledger/blob/master/bin/paypalcsv) (python) converts `paypaljson`'s output to CSV, with format similar to Paypal's manually-downloaded CSV. ### simplefinsetup [`simplefinsetup`](https://github.com/simonmichael/hledger/blob/master/bin/simplefinsetup) -helps set up SimpleFIN ([simplefin.org](https://simplefin.org)), a developer-friendly aggregator of US bank data. +helps set up access to SimpleFIN ([simplefin.org](https://simplefin.org)), a developer-friendly aggregator of US bank data. ### simplefinjson [`simplefinjson`](https://github.com/simonmichael/hledger/blob/master/bin/simplefinjson) downloads data for one or more bank accounts from SimpleFIN's API, as JSON. -### simplefinjson2csv +### simplefincsv -[`simplefinjson2csv`](https://github.com/simonmichael/hledger/blob/master/bin/simplefinjson2csv) -converts SimpleFIN's JSON data to CSV files, one for each bank account. +[`simplefincsv`](https://github.com/simonmichael/hledger/blob/master/bin/simplefincsv) +converts SimpleFIN's JSON data to CSV, for one or more bank accounts. ## hledger command line scripts diff --git a/bin/paypaljson2csv b/bin/paypalcsv similarity index 97% rename from bin/paypaljson2csv rename to bin/paypalcsv index 421222e6b..ee22df283 100755 --- a/bin/paypaljson2csv +++ b/bin/paypalcsv @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -__version__ = "1.0" +__version__ = "1.1" __author__ = "Simon Michael" VERSION = f"%prog {__version__}, by {__author__} 2021; part of the hledger project." @@ -83,9 +83,9 @@ links[].rel links[].method Examples: -paypaljson | paypaljson2csv | hledger -f csv:- --rules-file paypal.csv.rules print -paypaljson | paypaljson2csv | hledger -f csv:- --rules-file paypal.csv.rules activity --daily -paypaljson | paypaljson2csv paypal.csv && hledger import paypal.csv [--dry-run] # save as a file to remember transactions seen +paypaljson | paypalcsv | hledger -f csv:- --rules-file paypal.csv.rules print +paypaljson | paypalcsv | hledger -f csv:- --rules-file paypal.csv.rules activity --daily +paypaljson | paypalcsv paypal.csv && hledger import paypal.csv [--dry-run] # save as a file to remember transactions seen Sample output: diff --git a/bin/simplefincsv b/bin/simplefincsv new file mode 100755 index 000000000..b3fbf6ef6 --- /dev/null +++ b/bin/simplefincsv @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# simplefincsv 1.1 - (c) Simon Michael 2025 + +__version__ = "1.1" +__author__ = "Simon Michael" +versionmsg = f"%prog {__version__}, by {__author__} 2025; part of the hledger project." +usagemsg = """%prog [options] [JSONFILE|-] [REGEX] + +Read SimpleFIN /accounts JSON from JSONFILE or stdin, and for each account with transactions, +or just the ones where "ORGNAME ACCTNAME ACCTID" is case-insensitively infix-matched +by the given regular expression, print CSV records to stdout: + +1. an account info record: "account",ORGNAME,ACCTNAME,ACCTID,BALANCE,CURRENCY +2. a field headings record: "date","amount","description","payee","memo","id" +3. any transaction records, in date order. + +Also if the JSON includes error messages, they will be displayed on stderr, +and the exit code will be non-zero. + +Requirements: + python 3 + a SimpleFIN account with financial institution(s) and app connection configured. + +Examples: + +Download the JSON for one account and print as CSV: + +$ simplefinjson ACT-00b02b825-7cf-495f-8f49-b097d310dd4 | simplefincsv + +Download the JSON for all accounts, then print the CSV for one of them: + +$ simplefinjson >sf.json +$ simplefincsv sf.json 'chase.*card' + +""" + +from pprint import pprint as pp +import csv +import datetime +import decimal +import json +import optparse +import re +import sys + +def parse_options(): + parser = optparse.OptionParser(usage=usagemsg, version=versionmsg) + opts, args = parser.parse_args() + if len(args) > 2: + parser.print_help() + sys.exit() + return opts, args + +def main(): + opts, args = parse_options() + infile = args[0] if len(args) > 0 else '-' + r = re.compile(args[1], re.I) if len(args) > 1 else None + with open(infile,'r') if infile != '-' else sys.stdin as inp: + with sys.stdout as out: + j = json.load(inp) + w = csv.writer(out, quoting=csv.QUOTE_ALL) + + for a in j['accounts']: + oname = a['org']['name'] + aname = a['name'] + aid = a['id'] + if r and not r.search(f"{oname} {aname} {aid}"): continue + + ts = a['transactions'] + if ts: + w.writerow([ + "account", + oname, + aname, + aid, + a['balance'], + a['currency'] + ]) + w.writerow([ + "date", + "id", + "amount", + "description", + "payee", + "memo" + ]) + for t in reversed(a['transactions']): + dt = datetime.datetime.fromtimestamp(t['posted']) + # dtl = dt.astimezone() + w.writerow([ + dt.strftime('%Y-%m-%d'), # %H:%M:%S %Z'), + t['id'], + t['amount'], + t['description'], + t['payee'], + t['memo'] + ]) + + errors = j['errors'] + if errors: + for e in errors: print(f"simplefincsv: {e}", file=sys.stderr) + sys.exit(1) + +if __name__ == "__main__": main() diff --git a/bin/simplefinjson b/bin/simplefinjson index 7cdc9fdf5..6948904b3 100755 --- a/bin/simplefinjson +++ b/bin/simplefinjson @@ -1,9 +1,11 @@ #!/bin/bash -# simplefinjson 1.0 - (c) Simon Michael 2025 -# Download accounts and recent transactions history from SimpleFIN, as JSON. +# simplefinjson 1.1 - (c) Simon Michael 2025 +# simplefinjson [ACCTID] +# download the last 30 days' transaction history of one or all accounts from SimpleFIN, and print as tidy JSON. # # Requirements: # a SimpleFIN account with financial institution(s) and app connection configured +# curl # GNU date # jq to prettify @@ -11,12 +13,12 @@ # (Or use a secrets manager, like https://bitwarden.com/help/secrets-manager-quick-start.) SIMPLEFIN_ACCESS_URL='' -# Run GNU date, which is gdate on mac. date() { if hash gdate 2>/dev/null; then gdate "$@"; else date "$@"; fi } START=`date +%s -d '-30 days'` +ACCTPARAM=${1:+&account=$1} -curl -sL "$SIMPLEFIN_ACCESS_URL/accounts?start-date=$START" | jq +curl -sL "$SIMPLEFIN_ACCESS_URL/accounts?start-date=$START$ACCTPARAM" | jq # https://www.simplefin.org/protocol.html#http-endpoints # https://www.simplefin.org/protocol.html#get-accounts diff --git a/bin/simplefinjson2csv b/bin/simplefinjson2csv deleted file mode 100755 index c3fe143f5..000000000 --- a/bin/simplefinjson2csv +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -# simplefinjson2csv 1.0 - (c) Simon Michael 2025 - -__version__ = "1.0" -__author__ = "Simon Michael" -versionmsg = f"%prog {__version__}, by {__author__} 2025; part of the hledger project." -usagemsg = """%prog [options] [JSONFILE|-] [-] - -Read SimpleFIN /accounts JSON from a JSONFILE or stdin; -write each account's transactions as date-ordered CSV, -to separate CSV files or to stdout. - -Requirements: - python 3 - a SimpleFIN account with financial institution(s) and app connection configured. - -Examples: -$ simplefinjson | simplefinjson2csv -$ simplefinjson >sf.json; simplefinjson2csv sf.json - -""" - -from pprint import pprint as pp -import csv -import datetime -import decimal -import json -import optparse -import re -import sys - -def parse_options(): - parser = optparse.OptionParser(usage=usagemsg, version=versionmsg) - opts, args = parser.parse_args() - if len(args) > 2: - parser.print_help() - sys.exit() - return opts, args - -def main(): - opts, args = parse_options() - with open(args[0],'r') if len(args) > 0 and not args[0]=='-' else sys.stdin as inp: - - i = json.load(inp) - for a in i['accounts']: #[0:1]: - aid = a['id'] - aname = a['name'] - oname = a['org']['name'] - ts = a['transactions'] - if len(args) < 2: - fname = f'sf-{aid}.csv' - out = open(fname,'w') - print(f"writing {len(ts)} transactions to {fname}") - else: - out = sys.stdout - w = csv.writer(out, quoting=csv.QUOTE_ALL) - w.writerow([ - "date", - "id", - "amount", - "description", - "payee", - "memo" - ]) - - for t in reversed(a['transactions']): - dt = datetime.datetime.fromtimestamp(t['posted']) - # dtl = dt.astimezone() - w.writerow([ - dt.strftime('%Y-%m-%d'), # %H:%M:%S %Z'), - t['id'], - t['amount'], - t['description'], - t['payee'], - t['memo'] - ]) - -if __name__ == "__main__": main()