├── .gitignore
├── requirements.txt
├── README.md
├── binance_trades_verifier.py
└── binance_trades_fetcher.py


/.gitignore:
--------------------------------------------------------------------------------
1 | env
2 | *.csv


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | certifi==2020.4.5.1
 2 | chardet==3.0.4
 3 | idna==2.9
 4 | numpy==1.22.0
 5 | pandas==1.0.3
 6 | python-dateutil==2.8.1
 7 | pytz==2020.1
 8 | requests==2.23.0
 9 | six==1.14.0
10 | urllib3==1.26.5
11 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Code for [How To Easily Fetch Binance Historical Trades Using Python]
 2 | 
 3 | ## Running the code (Ubuntu)
 4 | 
 5 | You can run by creating a virtual environment and installing `requirements.txt` packages
 6 | 
 7 | ### Create a virtual enviroment
 8 | - `python -m venv env`
 9 | 
10 | ### Activate virtual env
11 | - `source env/bin/activate`
12 | 
13 | ### Install dependencies
14 | - `pip install -r requirements.txt`
15 | 
16 | 
17 | And that's it!
18 | 


--------------------------------------------------------------------------------
/binance_trades_verifier.py:
--------------------------------------------------------------------------------
 1 | import sys
 2 | import pandas as pd
 3 | import numpy as np
 4 | 
 5 | def main(file_name):
 6 |     df = pd.read_csv(file_name)
 7 |     values = df.to_numpy()
 8 | 
 9 |     last_id = values[0][1]
10 | 
11 |     for row in values[1:]:
12 |         trade_id = row[1]
13 |         if last_id + 1 != trade_id:
14 |             print('last_id', last_id)
15 |             print('trade_id', trade_id)
16 |             print('inconsistent data')
17 |             exit()
18 |         last_id = trade_id
19 |     
20 |     print('data is OK!')
21 | 
22 | if __name__ == "__main__":
23 |     file_name = sys.argv[1]
24 |     main(file_name)


--------------------------------------------------------------------------------
/binance_trades_fetcher.py:
--------------------------------------------------------------------------------
 1 | import requests
 2 | import json
 3 | import time
 4 | import sys
 5 | import pandas as pd
 6 | import calendar
 7 | from datetime import datetime, timedelta
 8 | 
 9 | def get_unix_ms_from_date(date):
10 |     return int(calendar.timegm(date.timetuple()) * 1000 + date.microsecond/1000)
11 | 
12 | def get_first_trade_id_from_start_date(symbol, from_date):
13 |     new_end_date = from_date + timedelta(seconds=60)
14 |     r = requests.get('https://api.binance.com/api/v3/aggTrades', 
15 |         params = {
16 |             "symbol" : symbol,
17 |             "startTime": get_unix_ms_from_date(from_date),
18 |             "endTime": get_unix_ms_from_date(new_end_date)
19 |         })
20 |         
21 |     if r.status_code != 200:
22 |         print('somethings wrong!', r.status_code)
23 |         print('sleeping for 10s... will retry')
24 |         time.sleep(10)
25 |         get_first_trade_id_from_start_date(symbol, from_date)
26 |     
27 |     response = r.json()
28 |     if len(response) > 0:
29 |         return response[0]['a']
30 |     else:
31 |         raise Exception('no trades found')
32 | 
33 | def get_trades(symbol, from_id):
34 |     r = requests.get("https://api.binance.com/api/v3/aggTrades",
35 |         params = {
36 |             "symbol": symbol,
37 |             "limit": 1000,
38 |             "fromId": from_id
39 |             })
40 | 
41 |     if r.status_code != 200:
42 |         print('somethings wrong!', r.status_code)
43 |         print('sleeping for 10s... will retry')
44 |         time.sleep(10)
45 |         get_historical_trades(symbol, from_id)
46 | 
47 |     return r.json()
48 | 
49 | def trim(df, to_date):
50 |     return df[df['T'] <= get_unix_ms_from_date(to_date)]
51 | 
52 | def fetch_binance_trades(symbol, from_date, to_date):
53 |     from_id = get_first_trade_id_from_start_date(symbol, from_date)
54 |     current_time = 0
55 |     df = pd.DataFrame()
56 | 
57 |     while current_time < get_unix_ms_from_date(to_date):
58 |         try:
59 |             trades = get_trades(symbol, from_id)
60 |         
61 |             from_id = trades[-1]['a']
62 |             current_time = trades[-1]['T']
63 |             
64 |             print(f'fetched {len(trades)} trades from id {from_id} @ {datetime.utcfromtimestamp(current_time/1000.0)}')
65 |             
66 |             df = pd.concat([df, pd.DataFrame(trades)])
67 |         
68 |             #dont exceed request limits
69 |             time.sleep(0.5)
70 |         except Exception:
71 |             print('somethings wrong....... sleeping for 15s')
72 |             time.sleep(15)
73 | 
74 |     df.drop_duplicates(subset='a', inplace=True)
75 |     df = trim(df, to_date)
76 |     
77 |     filename = f'binance__{symbol}__trades__from__{sys.argv[2].replace("/", "_")}__to__{sys.argv[3].replace("/", "_")}.csv'
78 |     df.to_csv(filename)
79 | 
80 |     print(f'{filename} file created!')
81 | 
82 | if __name__ == "__main__":
83 |     if len(sys.argv) < 4:
84 |         raise Exception('arguments format: <symbol> <start_date> <end_date>')
85 |         exit()
86 | 
87 |     symbol = sys.argv[1]
88 | 
89 |     from_date = datetime.strptime(sys.argv[2], '%m/%d/%Y')
90 |     to_date = datetime.strptime(sys.argv[3], '%m/%d/%Y') + timedelta(days=1) - timedelta(microseconds=1)
91 | 
92 |     fetch_binance_trades(symbol, from_date, to_date)


--------------------------------------------------------------------------------