bin: paypaljson, paypaljson2csv - download txns from paypal API
This commit is contained in:
Simon Michael
parent
4b5ffcc073
commit
e8c934ab33
40
bin/paypaljson
Executable file
40
bin/paypaljson
Executable file
@ -0,0 +1,40 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Download recent transaction history from Paypal as JSON,
|
||||||
|
# and print on stdout.
|
||||||
|
#
|
||||||
|
# Requirements: a Paypal developer account, curl, jq (optional, just for pretty-printing)
|
||||||
|
#
|
||||||
|
# brew install jq
|
||||||
|
#
|
||||||
|
# Limitations:
|
||||||
|
# - sees only the last 30 days of history (within the current timezone)
|
||||||
|
# - gets only the first page of results (no more than 500 transactions)
|
||||||
|
#
|
||||||
|
# Paypal API doc: https://developer.paypal.com/docs/api/transaction-search/v1
|
||||||
|
|
||||||
|
# credentials for an API app which you have created in https://developer.paypal.com
|
||||||
|
CLIENT_ID=$PAYPAL_CLIENT_ID
|
||||||
|
SECRET=$PAYPAL_SECRET
|
||||||
|
|
||||||
|
# GNU date (gdate on mac)
|
||||||
|
date()
|
||||||
|
{
|
||||||
|
if hash gdate 2>/dev/null; then gdate "$@"; else date "$@"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# max date range is 31d
|
||||||
|
START=`date +%Y-%m-%dT00:00:00Z -d '-30 days'`
|
||||||
|
END=`date +%Y-%m-%dT00:00:00Z -d '+1 day'`
|
||||||
|
# for testing:
|
||||||
|
# START=2021-05-21T00:00:00-0000
|
||||||
|
# END=2021-05-22T00:00:00-0000
|
||||||
|
|
||||||
|
# get a token allowing temporary access to the API
|
||||||
|
TOKEN=`curl -s https://api-m.paypal.com/v1/oauth2/token -d "grant_type=client_credentials" -u "$CLIENT_ID:$SECRET" | jq .access_token -r`
|
||||||
|
|
||||||
|
# 1. fetch json from paypal
|
||||||
|
# 2. prettify it with jq
|
||||||
|
curl -s -H "Authorization: Bearer $TOKEN" "https://api.paypal.com/v1/reporting/transactions?start_date=$START&end_date=$END&fields=all&page_size=500&page=1" \
|
||||||
|
| jq
|
||||||
|
|
||||||
|
echo
|
||||||
263
bin/paypaljson2csv
Executable file
263
bin/paypaljson2csv
Executable file
@ -0,0 +1,263 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
__version__ = "1.0"
|
||||||
|
__author__ = "Simon Michael"
|
||||||
|
|
||||||
|
VERSION = f"%prog {__version__}, by {__author__} 2021; part of the hledger project."
|
||||||
|
USAGE = """%prog [options] [JSONFILE|-] [CSVFILE|-]
|
||||||
|
|
||||||
|
Reads Paypal API's transaction history JSON from the first file or
|
||||||
|
stdin, writes Paypal-ish CSV (similar to manually downloaded CSV) to
|
||||||
|
the second file or stdout.
|
||||||
|
|
||||||
|
The goal is that you can use the same hledger CSV rules file to
|
||||||
|
process either CSV. So if you get tired of downloading CSV manually,
|
||||||
|
you can set up Paypal API access, download transactions as JSON,
|
||||||
|
convert that to CSV with this script, and keep importing it with
|
||||||
|
hledger without having to redo your paypal CSV rules.
|
||||||
|
|
||||||
|
Limitations:
|
||||||
|
|
||||||
|
- the Type column will have the short T-Codes rather than the long Descriptions
|
||||||
|
from https://developer.paypal.com/docs/reports/reference/tcodes
|
||||||
|
|
||||||
|
- the Status column will have the single-letter codes from
|
||||||
|
https://developer.paypal.com/docs/api/transaction-search/v1/#transactions
|
||||||
|
rather than the human-friendly values from
|
||||||
|
https://developer.paypal.com/docs/reports/online-reports/activity-download
|
||||||
|
|
||||||
|
Requirements: python 3, paypaljson for downloading JSON.
|
||||||
|
|
||||||
|
Expects paypal transactions JSON, as from
|
||||||
|
https://developer.paypal.com/docs/api/transaction-search/v1/#transactions,
|
||||||
|
containing most of the following items:
|
||||||
|
|
||||||
|
transaction_details[].transaction_info.paypal_account_id
|
||||||
|
transaction_details[].transaction_info.transaction_id
|
||||||
|
transaction_details[].transaction_info.paypal_reference_id
|
||||||
|
transaction_details[].transaction_info.paypal_reference_id_type
|
||||||
|
transaction_details[].transaction_info.transaction_event_code
|
||||||
|
transaction_details[].transaction_info.transaction_initiation_date
|
||||||
|
transaction_details[].transaction_info.transaction_updated_date
|
||||||
|
transaction_details[].transaction_info.transaction_amount.currency_code
|
||||||
|
transaction_details[].transaction_info.transaction_amount.value
|
||||||
|
transaction_details[].transaction_info.transaction_status
|
||||||
|
transaction_details[].transaction_info.ending_balance.currency_code
|
||||||
|
transaction_details[].transaction_info.ending_balance.value
|
||||||
|
transaction_details[].transaction_info.available_balance.currency_code
|
||||||
|
transaction_details[].transaction_info.available_balance.value
|
||||||
|
transaction_details[].transaction_info.invoice_id
|
||||||
|
transaction_details[].transaction_info.protection_eligibility
|
||||||
|
transaction_details[].payer_info.account_id
|
||||||
|
transaction_details[].payer_info.email_address
|
||||||
|
transaction_details[].payer_info.address_status
|
||||||
|
transaction_details[].payer_info.payer_status
|
||||||
|
transaction_details[].payer_info.payer_name.alternate_full_name
|
||||||
|
transaction_details[].payer_info.country_code
|
||||||
|
transaction_details[].shipping_info.name
|
||||||
|
transaction_details[].cart_info.item_details[].item_name
|
||||||
|
transaction_details[].cart_info.item_details[].item_description
|
||||||
|
transaction_details[].cart_info.item_details[].item_quantity
|
||||||
|
transaction_details[].cart_info.item_details[].item_unit_price.currency_code
|
||||||
|
transaction_details[].cart_info.item_details[].item_unit_price.value
|
||||||
|
transaction_details[].cart_info.item_details[].item_amount.currency_code
|
||||||
|
transaction_details[].cart_info.item_details[].item_amount.value
|
||||||
|
transaction_details[].cart_info.item_details[].total_item_amount.currency_code
|
||||||
|
transaction_details[].cart_info.item_details[].total_item_amount.value
|
||||||
|
transaction_details[].cart_info.item_details[].invoice_number
|
||||||
|
transaction_details[].store_info
|
||||||
|
transaction_details[].auction_info
|
||||||
|
transaction_details[].incentive_info
|
||||||
|
|
||||||
|
These are also in paypal transactions JSON, but ignored by this script:
|
||||||
|
|
||||||
|
account_number
|
||||||
|
start_date
|
||||||
|
end_date
|
||||||
|
last_refreshed_datetime
|
||||||
|
page
|
||||||
|
total_items
|
||||||
|
total_pages
|
||||||
|
links[].href
|
||||||
|
links[].rel
|
||||||
|
links[].method
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
paypaljson | paypaljson2csv | hledger -f csv:- --rules-file paypal.csv.rules print
|
||||||
|
paypaljson | paypaljson2csv | hledger -f csv:- --rules-file paypal.csv.rules activity --daily
|
||||||
|
paypaljson | paypaljson2csv paypal.csv && hledger import paypal.csv [--dry-run] # save as a file to remember transactions seen
|
||||||
|
|
||||||
|
Sample output:
|
||||||
|
|
||||||
|
"Date","Time","TimeZone","Name","Type","Status","Currency","Gross","Fee","Net","From Email Address","To Email Address","Transaction ID","Item Title","Item ID","Reference Txn ID","Receipt ID","Balance","Note"
|
||||||
|
"05/21/2021","00:25:50","HST","JetBrains Americas, Inc.","T0003","S","USD","-53.00","0.00","-53.00","","paypal.us@jetbrains.com","6XG491707Y653863W","1xPhpStorm","","B-63G006809C718195T","","-53.00",""
|
||||||
|
"05/21/2021","00:25:50","HST","","T0300","S","USD","53.00","0.00","53.00","","","724308646Y347530R","1xPhpStorm","","6XG491707Y653863W","","0.00",""\
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pprint import pprint as pp
|
||||||
|
import csv
|
||||||
|
import datetime
|
||||||
|
import decimal
|
||||||
|
import json
|
||||||
|
import optparse
|
||||||
|
import re
|
||||||
|
# import subprocess
|
||||||
|
import sys
|
||||||
|
# import tempfile
|
||||||
|
# import warnings
|
||||||
|
|
||||||
|
def parse_options():
|
||||||
|
parser = optparse.OptionParser(usage=USAGE, version=VERSION)
|
||||||
|
opts, args = parser.parse_args()
|
||||||
|
if len(args) > 2:
|
||||||
|
parser.print_help()
|
||||||
|
sys.exit()
|
||||||
|
return opts, args
|
||||||
|
|
||||||
|
def main():
|
||||||
|
opts, args = parse_options()
|
||||||
|
out = open(args[1],'w') if len(args) > 1 and not args[1]=='-' else sys.stdout
|
||||||
|
csvwriter = csv.writer(out, quoting=csv.QUOTE_ALL)
|
||||||
|
with open(args[0],'r') if len(args) > 0 and not args[0]=='-' else sys.stdin as inp:
|
||||||
|
|
||||||
|
j = json.load(inp)
|
||||||
|
txns = j['transaction_details']
|
||||||
|
|
||||||
|
csvwriter.writerow([
|
||||||
|
"Date",
|
||||||
|
"Time",
|
||||||
|
"TimeZone",
|
||||||
|
"Name",
|
||||||
|
"Type",
|
||||||
|
"Status",
|
||||||
|
"Currency",
|
||||||
|
"Gross",
|
||||||
|
"Fee",
|
||||||
|
"Net",
|
||||||
|
"From Email Address",
|
||||||
|
"To Email Address",
|
||||||
|
"Transaction ID",
|
||||||
|
"Item Title",
|
||||||
|
"Item ID",
|
||||||
|
"Reference Txn ID",
|
||||||
|
"Receipt ID",
|
||||||
|
"Balance",
|
||||||
|
"Note"
|
||||||
|
])
|
||||||
|
|
||||||
|
for t in txns:
|
||||||
|
txninfo = t['transaction_info']
|
||||||
|
payerinfo = t['payer_info']
|
||||||
|
shippinginfo = t['shipping_info']
|
||||||
|
cartinfo = t['cart_info']
|
||||||
|
storeinfo = t['store_info']
|
||||||
|
auctioninfo = t['auction_info']
|
||||||
|
incentiveinfo = t['incentive_info']
|
||||||
|
|
||||||
|
localdt = datetime.datetime.strptime(txninfo['transaction_initiation_date'], "%Y-%m-%dT%H:%M:%S%z").astimezone()
|
||||||
|
|
||||||
|
grossamt = decimal.Decimal(txninfo.get('transaction_amount',{}).get('value','0.00'))
|
||||||
|
feeamt = decimal.Decimal(txninfo.get('fee_amount',{}).get('value','0.00'))
|
||||||
|
netamt = grossamt + feeamt
|
||||||
|
|
||||||
|
# for type, csv shows the long Description from
|
||||||
|
# https://developer.paypal.com/docs/reports/reference/tcodes,
|
||||||
|
# json shows the short T-Code. There are too many, keep the T-Code for now.
|
||||||
|
# csv rules files will have to be aware.
|
||||||
|
ttype = txninfo.get('transaction_event_code','')
|
||||||
|
|
||||||
|
# for status, csv shows the human-friendly values from
|
||||||
|
# https://developer.paypal.com/docs/reports/online-reports/activity-download/#download-data-fields -> Status:
|
||||||
|
# Possible values for all activity:
|
||||||
|
# Completed
|
||||||
|
# Denied
|
||||||
|
# Reversed
|
||||||
|
# Pending
|
||||||
|
# Active
|
||||||
|
# Expired
|
||||||
|
# Removed
|
||||||
|
# Unverified
|
||||||
|
# Voided
|
||||||
|
# Processing
|
||||||
|
# Created
|
||||||
|
# Canceled
|
||||||
|
# Possible additional values are supported for invoice activity:
|
||||||
|
# Error
|
||||||
|
# Draft
|
||||||
|
# Unpaid
|
||||||
|
# Paid
|
||||||
|
# Unpaid (sent)
|
||||||
|
# Marked as paid
|
||||||
|
# Marked as refunded
|
||||||
|
# Refunded
|
||||||
|
# Partially refunded
|
||||||
|
# Scheduled
|
||||||
|
# Partially paid
|
||||||
|
# Payment pending
|
||||||
|
# json shows the single-letter codes from
|
||||||
|
# https://developer.paypal.com/docs/api/transaction-search/v1/#transactions -> transaction_status:
|
||||||
|
# D - PayPal or merchant rules denied the transaction.
|
||||||
|
# P - The transaction is pending. The transaction was created but waits for another payment process to complete, such as an ACH transaction, before the status changes to S.
|
||||||
|
# S - The transaction successfully completed without a denial and after any pending statuses.
|
||||||
|
# V - A successful transaction was reversed and funds were refunded to the original sender.
|
||||||
|
# Keep the short codes for now, csv rules files will have to be aware.
|
||||||
|
tstatus = txninfo.get('transaction_status','')
|
||||||
|
|
||||||
|
# for Item Title, use transaction_subject if any, otherwise first item title if any
|
||||||
|
if 'transaction_subject' in txninfo:
|
||||||
|
item_title = txninfo.get('transaction_subject','')
|
||||||
|
item_id = ''
|
||||||
|
else:
|
||||||
|
items = cartinfo.get('item_details',[])
|
||||||
|
if len(items)>0:
|
||||||
|
item_title = items[0].get('item_name','')
|
||||||
|
item_id = items[0].get('item_id','')
|
||||||
|
else:
|
||||||
|
item_title = ''
|
||||||
|
item_id = ''
|
||||||
|
|
||||||
|
# in the notes below, "csv" refers to CSV downloaded manually, "json" refers to JSON downloaded from the API
|
||||||
|
csvwriter.writerow([
|
||||||
|
# Date
|
||||||
|
localdt.strftime('%m/%d/%Y'),
|
||||||
|
# Time
|
||||||
|
localdt.strftime('%H:%M:%S'),
|
||||||
|
# TimeZone (should be downloader's local timezone)
|
||||||
|
localdt.strftime('%Z'),
|
||||||
|
# Name
|
||||||
|
payerinfo.get('payer_name',{}).get('alternate_full_name',''),
|
||||||
|
# Type
|
||||||
|
ttype,
|
||||||
|
# Status
|
||||||
|
tstatus,
|
||||||
|
# Currency
|
||||||
|
txninfo.get('transaction_amount',{}).get('currency_code',''),
|
||||||
|
# Gross
|
||||||
|
str(grossamt),
|
||||||
|
# Fee
|
||||||
|
str(feeamt),
|
||||||
|
# Net
|
||||||
|
str(netamt),
|
||||||
|
# From Email Address
|
||||||
|
# csv often mentions account owner's email here, json does not
|
||||||
|
payerinfo.get('email_address','') if grossamt > 0 else '',
|
||||||
|
# To Email Address
|
||||||
|
# csv often mentions account owner's email here, json does not
|
||||||
|
payerinfo.get('email_address','') if grossamt <= 0 else '',
|
||||||
|
# Transaction ID
|
||||||
|
txninfo.get('transaction_id',''),
|
||||||
|
# Item Title (of first item)
|
||||||
|
item_title,
|
||||||
|
# Item ID (if any)
|
||||||
|
item_id,
|
||||||
|
# Reference Txn ID
|
||||||
|
txninfo.get('paypal_reference_id',''),
|
||||||
|
# Receipt ID
|
||||||
|
txninfo.get('receipt_id',''),
|
||||||
|
# Balance
|
||||||
|
txninfo.get('ending_balance',{}).get('value',''),
|
||||||
|
# Note
|
||||||
|
txninfo.get('transaction_note','')
|
||||||
|
])
|
||||||
|
|
||||||
|
if __name__ == "__main__": main()
|
||||||
Loading…
Reference in New Issue
Block a user