├── .gitignore ├── README.md ├── account-monitor └── monitor-accounts.py ├── account-snapshots ├── cluster_token_count.sh ├── create-snapshot-csv.sh ├── get_owned_accounts_info.sh ├── get_program_accounts.sh ├── write-all-stake-accounts.sh └── write-all-system-accounts.sh └── ledger-history-collection ├── addrs └── get-account-txn-history.sh /.gitignore: -------------------------------------------------------------------------------- 1 | *.txt 2 | *.csv 3 | *.json 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # token-ops 2 | Scripts and tools for token accounting and operation 3 | -------------------------------------------------------------------------------- /account-monitor/monitor-accounts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import requests 3 | import csv 4 | import time 5 | import logging 6 | import json 7 | import argparse 8 | import pathlib 9 | import os 10 | 11 | 12 | def write_balances_to_file(balances, filename): 13 | with open(filename, 'w') as csv_file: 14 | writer = csv.writer(csv_file) 15 | writer.writerow(['address', 'balance']) 16 | for key, value in balances.items(): 17 | writer.writerow([key, value]) 18 | 19 | 20 | def read_balances_from_file(filename): 21 | with open(filename) as infile: 22 | reader = csv.DictReader(infile) 23 | data = {} 24 | for row in reader: 25 | key = row["address"] 26 | val = float(row["balance"]) 27 | data[key] = val 28 | return data 29 | 30 | 31 | def get_latest_balances(addresses, rpc_url, webhook_url): 32 | try: 33 | balances = [] 34 | max_addrs_per_request = 100 35 | i = 0 36 | last_iter = False 37 | 38 | while True: 39 | if len(addresses) <= max_addrs_per_request * (i + 1): 40 | addr_list_slice = addresses[max_addrs_per_request * i:] 41 | last_iter = True 42 | else: 43 | addr_list_slice = addresses[max_addrs_per_request * i: 44 | max_addrs_per_request * (i + 1)] 45 | 46 | payload = { 47 | "method": "getMultipleAccounts", 48 | "params": [ 49 | addr_list_slice, 50 | { 51 | "encoding": "base64" 52 | }, 53 | ], 54 | "jsonrpc": "2.0", 55 | "id": 0, 56 | } 57 | 58 | response = requests.post(rpc_url, json=payload, timeout=60).json() 59 | 60 | for entry in response['result']['value']: 61 | if entry is None: 62 | balances.append(0) 63 | else: 64 | balances.append(entry['lamports']/1000000000) 65 | 66 | if last_iter: 67 | break 68 | else: 69 | i += 1 70 | 71 | return {k: v for k, v in zip(addresses, balances)} 72 | 73 | except Exception as e: 74 | logging.error(str(e)) 75 | send_message_to_slack(str(e), webhook_url) 76 | return None 77 | 78 | 79 | # Read a csv file and return a dict with the addresses as keys, 80 | # and dicts with any subsequent column headers:data as values 81 | def get_dict_from_csv(filename): 82 | with open(filename) as infile: 83 | reader = csv.DictReader(infile) 84 | data = {} 85 | for row in reader: 86 | key = row.pop('address') 87 | data[key] = dict(row) 88 | return data 89 | 90 | 91 | def compare_balances(old_balances, 92 | new_balances, 93 | account_info, 94 | webhook_url=None): 95 | for address, new_balance in new_balances.items(): 96 | send_message = False 97 | if address in old_balances: 98 | if new_balance < old_balances[address]: 99 | message = "Balance of %s has decreased from %d to %d SOL" % \ 100 | (address, old_balances[address], new_balance) 101 | logging.warning(message) 102 | send_message = True 103 | elif new_balance > old_balances[address]: 104 | message = "Balance of %s has increased from %d to %d SOL" % \ 105 | (address, old_balances[address], new_balance) 106 | logging.info(message) 107 | else: 108 | message = "Balance of %s has not changed" % address 109 | logging.debug(message) 110 | else: 111 | message = "%s not found in prior balance data" % address 112 | logging.info(message) 113 | 114 | if webhook_url is not None and send_message is True: 115 | send_message_to_slack(message, 116 | webhook_url) 117 | publish_account_info_to_slack(address, 118 | account_info, 119 | webhook_url) 120 | 121 | 122 | def send_message_to_slack(payload, webhook_url): 123 | slack_data = {"text": payload} 124 | 125 | response = requests.post( 126 | webhook_url, data=json.dumps(slack_data), 127 | headers={'Content-Type': 'application/json'} 128 | ) 129 | if response.status_code != 200: 130 | raise ValueError( 131 | 'Request to slack returned an error %s, the response is:\n%s' 132 | % (response.status_code, response.text) 133 | ) 134 | 135 | 136 | def publish_account_info_to_slack(address, account_info, webhook_url): 137 | payload = "```address: %s" % address 138 | for k, v in account_info[address].items(): 139 | payload += "\n%s: %s" % (k, v) 140 | payload += "```" 141 | logging.info(payload) 142 | send_message_to_slack(payload, webhook_url) 143 | 144 | 145 | def publish_all_balances_to_slack(balances, webhook_url): 146 | payload = "\n" 147 | for k, v in balances.items(): 148 | payload += "`%s: %.2f`\n" % (k, v) 149 | logging.info(payload) 150 | send_message_to_slack(payload, webhook_url) 151 | 152 | 153 | def main(): 154 | logging.basicConfig(level=logging.INFO) 155 | 156 | parser = argparse.ArgumentParser( 157 | formatter_class=argparse.ArgumentDefaultsHelpFormatter) 158 | parser.add_argument('-u', '--rpc-url', 159 | type=str, 160 | default="http://api.mainnet-beta.solana.com", 161 | help="RPC endpoint to target", 162 | dest="rpc_url") 163 | parser.add_argument('-w', '--slack-webhook-url', 164 | type=str, 165 | dest="slack_webhook_url", 166 | help="Webhook URL to receive monitoring alerts") 167 | parser.add_argument('-i', '--input-file', 168 | type=str, 169 | default="accounts.csv", 170 | dest="input_file", 171 | help="Input .csv file that contains at a minimum, " 172 | "a single column containing a list of addresses" 173 | " to monitor") 174 | parser.add_argument('-o', '--output-file', type=str, 175 | default="latest_balances.csv", 176 | dest="balances_file", 177 | help="Output .csv file that will contain columns of " 178 | "account addresses and their latest balances in" 179 | " SOL") 180 | parser.add_argument('--balance-check-interval', 181 | type=int, 182 | default=60, 183 | dest="balance_check_interval", 184 | help="Number of seconds between balance checks") 185 | parser.add_argument('--liveness-check-interval', 186 | type=int, 187 | default=3600, 188 | dest="liveness_check_interval", 189 | help="Number of seconds between liveness checks") 190 | 191 | args = parser.parse_args() 192 | 193 | account_info = get_dict_from_csv(args.input_file) 194 | addresses = list(account_info.keys()) 195 | 196 | old_balances = {} 197 | if os.path.isfile(args.balances_file): 198 | old_balances = read_balances_from_file(args.balances_file) 199 | 200 | if args.slack_webhook_url is not None: 201 | send_message_to_slack("Starting up account monitor", 202 | args.slack_webhook_url) 203 | 204 | liveness_time = 0 205 | while True: 206 | if time.time() - liveness_time >= args.liveness_check_interval: 207 | message = "Liveness Check Stats\n" \ 208 | "--------------------\n" \ 209 | "Liveness check interval: %d seconds\n" \ 210 | "Balance check interval: %d seconds\n" \ 211 | "Time since last balance update: %d seconds\n" \ 212 | "--------------------\n" \ 213 | "Number of accounts monitored: %d\n" \ 214 | "Total SOL balance of all accounts at last check: %d\n" \ 215 | "--------------------" % \ 216 | (args.liveness_check_interval, 217 | args.balance_check_interval, 218 | time.time() - 219 | pathlib.Path(args.balances_file).stat().st_mtime, 220 | len(addresses), 221 | sum(old_balances.values())) 222 | logging.info(message) 223 | 224 | if args.slack_webhook_url is not None: 225 | send_message_to_slack(message, args.slack_webhook_url) 226 | liveness_time = time.time() 227 | 228 | latest_balances = get_latest_balances(addresses, 229 | args.rpc_url, 230 | args.slack_webhook_url) 231 | if latest_balances is not None: 232 | compare_balances(old_balances, 233 | latest_balances, 234 | account_info, 235 | webhook_url=args.slack_webhook_url) 236 | 237 | write_balances_to_file(latest_balances, args.balances_file) 238 | old_balances = latest_balances 239 | else: 240 | message = "Unable to retrieve latest balances from RPC node" 241 | logging.warning(message) 242 | send_message_to_slack(message, args.slack_webhook_url) 243 | 244 | time.sleep(args.balance_check_interval) 245 | 246 | 247 | if __name__ == "__main__": 248 | main() 249 | -------------------------------------------------------------------------------- /account-snapshots/cluster_token_count.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source "$(dirname "$0")"/get_program_accounts.sh 4 | 5 | usage() { 6 | exitcode=0 7 | if [[ -n "$1" ]]; then 8 | exitcode=1 9 | echo "Error: $*" 10 | fi 11 | cat <> $csvfile 58 | fi 59 | } 60 | 61 | function display_results_summary { 62 | stake_account_balance_total=0 63 | num_stake_accounts=0 64 | { 65 | read 66 | while IFS=, read -r program account_pubkey lamports lockup_epoch; do 67 | case $program in 68 | SYSTEM) 69 | system_account_balance=$lamports 70 | ;; 71 | STAKE) 72 | stake_account_balance_total=$((stake_account_balance_total + $lamports)) 73 | num_stake_accounts=$((num_stake_accounts + 1)) 74 | ;; 75 | *) 76 | echo "Unknown program: $program" 77 | exit 1 78 | ;; 79 | esac 80 | done 81 | } < "$results_file" 82 | 83 | stake_account_balance_total_sol="$(bc <<< "scale=3; $stake_account_balance_total/$LAMPORTS_PER_SOL")" 84 | system_account_balance_sol="$(bc <<< "scale=3; $system_account_balance/$LAMPORTS_PER_SOL")" 85 | 86 | all_account_total_balance="$(bc <<< "scale=3; $system_account_balance+$stake_account_balance_total")" 87 | all_account_total_balance_sol="$(bc <<< "scale=3; ($system_account_balance+$stake_account_balance_total)/$LAMPORTS_PER_SOL")" 88 | 89 | echo "--------------------------------------------------------------------------------------" 90 | echo "Results written to: $results_file" 91 | echo "--------------------------------------------------------------------------------------" 92 | echo "Summary of accounts owned by $filter_pubkey" 93 | echo "" 94 | printf "Number of STAKE accounts: %'d\n" $num_stake_accounts 95 | printf "Balance of all STAKE accounts: %'d lamports\n" $stake_account_balance_total 96 | printf "Balance of all STAKE accounts: %'.3f SOL\n" $stake_account_balance_total_sol 97 | printf "\n" 98 | printf "Balance of SYSTEM account: %'d lamports\n" $system_account_balance 99 | printf "Balance of SYSTEM account: %'.3f SOL\n" $system_account_balance_sol 100 | printf "\n" 101 | printf "Total Balance of ALL accounts: %'d lamports\n" $all_account_total_balance 102 | printf "Total Balance of ALL accounts: %'.3f SOL\n" $all_account_total_balance_sol 103 | echo "--------------------------------------------------------------------------------------" 104 | } 105 | 106 | function display_results_details { 107 | cat $results_file | column -t -s, 108 | } 109 | 110 | [[ -n $filter_pubkey ]] || usage 111 | results_file=accounts_owned_by_${filter_pubkey}.csv 112 | system_account_json_file=system_account_${filter_pubkey}.json 113 | 114 | echo "Program,Account_Pubkey,Lamports,Lockup_Epoch" > $results_file 115 | 116 | echo "Getting system account data" 117 | get_account_info $filter_pubkey $RPC_URL $system_account_json_file 118 | system_account_balance="$(cat "$system_account_json_file" | jq -r '(.result | .value | .lamports)')" 119 | if [[ "$system_account_balance" == "null" ]]; then 120 | echo "The provided pubkey is not found in the system program: $filter_pubkey" 121 | exit 1 122 | fi 123 | echo SYSTEM,$filter_pubkey,$system_account_balance,N/A >> $results_file 124 | 125 | echo "Getting all stake program accounts" 126 | get_program_accounts STAKE $STAKE_PROGRAM_PUBKEY $RPC_URL $all_stake_accounts_json_file 127 | write_program_account_data_csv STAKE $all_stake_accounts_json_file $all_stake_accounts_csv_file 128 | 129 | echo "Querying cluster at $RPC_URL for stake accounts with authorized staker: $filter_pubkey" 130 | last_tick=$SECONDS 131 | { 132 | read 133 | while IFS=, read -r account_pubkey lamports; do 134 | parse_stake_account_data_to_file $account_pubkey $filter_pubkey $results_file & 135 | sleep 0.01 136 | if [[ $(($SECONDS - $last_tick)) == 1 ]]; then 137 | last_tick=$SECONDS 138 | printf "." 139 | fi 140 | done 141 | } < "$all_stake_accounts_csv_file" 142 | wait 143 | printf "\n" 144 | 145 | display_results_details 146 | display_results_summary 147 | -------------------------------------------------------------------------------- /account-snapshots/get_program_accounts.sh: -------------------------------------------------------------------------------- 1 | # | source | this file 2 | 3 | STAKE_PROGRAM_PUBKEY=Stake11111111111111111111111111111111111111 4 | SYSTEM_PROGRAM_PUBKEY=11111111111111111111111111111111 5 | VOTE_PROGRAM_PUBKEY=Vote111111111111111111111111111111111111111 6 | STORAGE_PROGRAM_PUBKEY=Storage111111111111111111111111111111111111 7 | CONFIG_PROGRAM_PUBKEY=Config1111111111111111111111111111111111111 8 | 9 | function get_program_accounts { 10 | PROGRAM_NAME="$1" 11 | PROGRAM_PUBKEY="$2" 12 | URL="$3" 13 | 14 | if [[ -n "$4" ]] ; then 15 | JSON_OUTFILE="$4" 16 | else 17 | JSON_OUTFILE="${PROGRAM_NAME}_account_data.json" 18 | fi 19 | curl -s -X POST -H "Content-Type: application/json" -d \ 20 | '{"jsonrpc":"2.0","id":1, "method":"getProgramAccounts", "params":["'$PROGRAM_PUBKEY'"]}' $URL | jq '.' \ 21 | > $JSON_OUTFILE 22 | } 23 | 24 | function write_program_account_data_csv { 25 | PROGRAM_NAME="$1" 26 | if [[ -n "$2" ]] ; then 27 | JSON_INFILE="$2" 28 | else 29 | JSON_INFILE="${PROGRAM_NAME}_account_data.json" 30 | fi 31 | if [[ -n "$3" ]] ; then 32 | CSV_OUTFILE="$3" 33 | else 34 | CSV_OUTFILE="${PROGRAM_NAME}_account_data.csv" 35 | fi 36 | 37 | echo "program,account_address,balance" > $CSV_OUTFILE 38 | cat "$JSON_INFILE" | jq -r --arg PROGRAM_NAME "$PROGRAM_NAME" '(.result | .[]) | [$PROGRAM_NAME, .pubkey, (.account | .lamports)/1000000000] | @csv' \ 39 | >> $CSV_OUTFILE 40 | } 41 | 42 | function get_account_info { 43 | ACCOUNT_PUBKEY="$1" 44 | URL="$2" 45 | 46 | if [[ -n "$3" ]] ; then 47 | JSON_OUTFILE="$3" 48 | else 49 | JSON_OUTFILE="${ACCOUNT_PUBKEY}_account_info.json" 50 | fi 51 | curl -s -X POST -H "Content-Type: application/json" -d \ 52 | '{"jsonrpc":"2.0","id":1, "method":"getAccountInfo", "params":["'$ACCOUNT_PUBKEY'"]}' $URL | jq '.' \ 53 | > $JSON_OUTFILE 54 | } 55 | -------------------------------------------------------------------------------- /account-snapshots/write-all-stake-accounts.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | RPC_URL=https://api.mainnet-beta.solana.com 6 | timestamp="$(date -u +"%Y-%m-%d_%H:%M:%S")" 7 | output_csv="stake_accounts-${timestamp}.csv" 8 | 9 | usage() { 10 | exitcode=0 11 | if [[ -n "$1" ]]; then 12 | exitcode=1 13 | echo "Error: $*" 14 | fi 15 | cat <> $output_csv 40 | } 41 | 42 | shortArgs=() 43 | while [[ -n $1 ]]; do 44 | if [[ ${1:0:2} = -- ]]; then 45 | if [[ $1 = --url ]]; then 46 | RPC_URL="$2" 47 | shift 2 48 | elif [[ $1 = --output-csv ]]; then 49 | output_csv="$2" 50 | shift 2 51 | else 52 | usage "Unknown option: $1" 53 | fi 54 | else 55 | shortArgs+=("$1") 56 | shift 57 | fi 58 | done 59 | 60 | while getopts "o:" opt "${shortArgs[@]}"; do 61 | case $opt in 62 | o) 63 | output_csv=$OPTARG 64 | ;; 65 | *) 66 | usage "Error: unhandled option: $opt" 67 | ;; 68 | esac 69 | done 70 | 71 | echo "Writing stake account data to $output_csv..." 72 | 73 | all_stake_accounts_file="all_stake_accounts-${timestamp}" 74 | solana stakes --url $RPC_URL > "$all_stake_accounts_file" 75 | slot="$(solana slot --url $RPC_URL)" 76 | 77 | echo "program,account_address,balance,stake_authority,withdraw_authority,delegated_to,lockup_timestamp,checked_at_slot" > $output_csv 78 | 79 | stake_account_info=() 80 | { 81 | read 82 | while IFS=, read -r line; do 83 | if [[ -n $line ]]; then 84 | stake_account_info+="${line} 85 | " 86 | else 87 | write_stake_info_to_csv "$stake_account_info" "$output_csv" 88 | stake_account_info=() 89 | fi 90 | done 91 | } < "$all_stake_accounts_file" 92 | 93 | echo "Finished" 94 | -------------------------------------------------------------------------------- /account-snapshots/write-all-system-accounts.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | source "$(dirname "$0")"/get_program_accounts.sh 6 | 7 | RPC_URL=http://api.mainnet-beta.solana.com 8 | timestamp="$(date -u +"%Y-%m-%d_%H:%M:%S")" 9 | 10 | json_file=system_account_data-${timestamp}.json 11 | output_csv=system_account_balances-${timestamp}.csv 12 | 13 | usage() { 14 | exitcode=0 15 | if [[ -n "$1" ]]; then 16 | exitcode=1 17 | echo "Error: $*" 18 | fi 19 | cat < $output_file 42 | total_txns=$(cat $output_file | wc -l | awk '{print $1}') 43 | echo "Found $total_txns transactions for $address" 44 | } 45 | 46 | write_transaction_details_to_file() { 47 | input_file="$1" 48 | output_csv="$2" 49 | 50 | header="slot,signature,err,target_addr,target_pre,target_post,target_change," 51 | header="${header}0_addr,0_pre,0_post,0_change,1_addr,1_pre,1_post,1_change," 52 | header="${header}2_addr,2_pre,2_post,2_change,3_addr,3_pre,3_post,3_change," 53 | header="${header}4_addr,4_pre,4_post,4_change,5_addr,5_pre,5_post,5_change," 54 | header="${header}6_addr,6_pre,6_post,6_change,7_addr,7_pre,7_post,7_change," 55 | 56 | echo "$header" > "$output_csv" 57 | { 58 | while IFS=, read -r signature; do 59 | [[ -n $signature ]] || break 60 | 61 | txn_details="$(curl -sX POST -H "Content-Type: application/json" -d '{"jsonrpc": "2.0","id":1,"method":"getConfirmedTransaction","params":["'$signature'", "json"]}' $RPC_URL)" 62 | slot=$(echo $txn_details | jq -r '.result | .slot') 63 | err=$(echo $txn_details | jq -r '.result | .meta | .err | @text') 64 | [[ $err = "null" ]] || err="error" 65 | 66 | accounts="$(echo $txn_details | jq -r '.result | .transaction | .message | .accountKeys | .[]')" 67 | pre_balances="$(echo $txn_details | jq -r '.result | .meta | .preBalances? | .[]?')" 68 | post_balances="$(echo $txn_details | jq -r '.result | .meta | .postBalances? | .[]?')" 69 | 70 | accounts_array=(${accounts}) 71 | pre_balances_array=(${pre_balances}) 72 | post_balances_array=(${post_balances}) 73 | 74 | output_string="$slot,$signature,$err,$address," 75 | additional_account_balances_string="" 76 | 77 | len=${#accounts_array[*]} 78 | for (( i=0; i<${len} ; i++ )); do 79 | if [[ "${accounts_array[$i]}" = "$address" ]]; then 80 | target_pre_balance_lamports=${pre_balances_array[$i]} 81 | target_post_balance_lamports=${post_balances_array[$i]} 82 | target_pre_balance=$((target_pre_balance_lamports / 1000000000)) 83 | target_post_balance=$((target_post_balance_lamports / 1000000000)) 84 | target_change=$((target_post_balance - target_pre_balance)) 85 | else 86 | other_pre_balance_lamports=${pre_balances_array[$i]} 87 | other_post_balance_lamports=${post_balances_array[$i]} 88 | other_pre_balance=$((other_pre_balance_lamports / 1000000000)) 89 | other_post_balance=$((other_post_balance_lamports / 1000000000)) 90 | other_change=$((other_post_balance - other_pre_balance)) 91 | additional_account_balances_string="${additional_account_balances_string}${accounts_array[$i]},${other_pre_balance},${other_post_balance},${other_change}," 92 | fi 93 | done 94 | 95 | if [[ len > 0 ]]; then 96 | output_string="${output_string}${target_pre_balance},${target_post_balance},${target_change},${additional_account_balances_string}" 97 | fi 98 | 99 | echo "$output_string" >> $output_csv 100 | done 101 | } < "$input_file" 102 | } 103 | 104 | get_account_txn_history() { 105 | out_dir="$1" 106 | addr="$2" 107 | 108 | signature_file=$out_dir/txn_sigs_$addr.csv 109 | account_history_csv=$out_dir/account_history_$addr.csv 110 | 111 | echo "Finding all transaction signatures for $address" 112 | get_all_transaction_signatures $addr $signature_file 113 | 114 | echo "Parsing all transaction details for $address" 115 | write_transaction_details_to_file $signature_file $account_history_csv 116 | } 117 | 118 | export GOOGLE_APPLICATION_CREDENTIALS=~/mainnet-beta-bigtable-ro.json 119 | RPC_URL=https://api.mainnet-beta.solana.com 120 | timestamp="$(date -u +"%Y-%m-%d_%H:%M:%S")" 121 | 122 | address= 123 | address_file= 124 | 125 | while [[ -n $1 ]]; do 126 | if [[ ${1:0:2} = -- ]]; then 127 | if [[ $1 = --url ]]; then 128 | RPC_URL="$2" 129 | shift 2 130 | elif [[ $1 = --address ]]; then 131 | address="$2" 132 | shift 2 133 | elif [[ $1 = --address-file ]]; then 134 | address_file="$2" 135 | shift 2 136 | else 137 | usage "Unknown option: $1" 138 | fi 139 | else 140 | usage "Unknown option: $1" 141 | fi 142 | done 143 | 144 | if [[ -n $address && -n $address_file ]]; then 145 | usage "Cannot provide both --address AND --address-file" 146 | elif [[ -z $address && -z $address_file ]]; then 147 | usage "Must provide --address OR --address-file" 148 | fi 149 | 150 | output_dir="account_histories-${timestamp}" 151 | mkdir -p $output_dir 152 | 153 | if [[ -n $address_file ]]; then 154 | { 155 | while IFS=, read -r address; do 156 | get_account_txn_history "$output_dir" "$address" 157 | done 158 | } < "$address_file" 159 | else 160 | get_account_txn_history "$output_dir" "$address" 161 | fi 162 | --------------------------------------------------------------------------------