├── renovate.json ├── requirements.txt ├── install_locally.py ├── LICENSE ├── README.md ├── .gitignore └── crypto_history.py /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base" 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | beautifulsoup4==4.7.1 2 | bs4==0.0.1 3 | certifi==2018.11.29 4 | chardet==3.0.4 5 | idna==2.8 6 | requests==2.21.0 7 | urllib3==1.25 8 | -------------------------------------------------------------------------------- /install_locally.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os, site 3 | 4 | #package = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'packages') 5 | package = os.path.dirname(os.path.abspath(__file__)) 6 | print(package) 7 | 8 | pathspec = r""" 9 | # Generated by Crypto-History's installer (install_locally.py) 10 | # In the lines below, list the paths where Python should look for 11 | # supplied modules, one directory per line. 12 | # 13 | # If a directory does not exist when Python is started, it will be ignored. 14 | %s 15 | """ % package 16 | 17 | print("Adding path:", package) 18 | 19 | usp = site.getusersitepackages() 20 | if not os.path.exists(usp): 21 | os.makedirs(usp) 22 | uspfile = os.path.join(usp, 'crypto-history.pth') 23 | open(uspfile, 'w').write(pathspec) 24 | print('Wrote to ' + uspfile) 25 | print("Crypto-history package installed successfully!") 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 dylankilkenny 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CoinMarketCap-Historical-Prices 2 | This script scrapes data from the [historical data](https://coinmarketcap.com/currencies/ethereum/historical-data/) tab on coinmarketcap.com 3 | 4 | ## Install & Run 5 | 6 | #### Download and install 7 | ``` 8 | $ git clone https://github.com/dylankilkenny/CoinMarketCap-Historical-Prices.git 9 | $ cd CoinMarketCap-Historical-Prices 10 | $ pip3 install -r requirements.txt 11 | $ python install_locally.py 12 | ``` 13 | 14 | #### Running 15 | To run the script and gather data for all listed cryptocurrencys on coinmarketcap you need to pass the start date and end date in YYYYMMDD format 16 | 17 | ``` 18 | $ python3 crypto_history.py 20170101 20180201 19 | ``` 20 | you can also specify a cryptocurrency with a third argument 21 | 22 | ``` 23 | $ python3 crypto_history.py 20170101 20180201 ethereum 24 | ``` 25 | The data will be saved to a CSV file 26 | 27 | #### Importing 28 | 29 | ```bash 30 | Python 3.6.4 (default, Jan 5 2018, 02:35:40) 31 | [GCC 7.2.1 20171224] on linux 32 | Type "help", "copyright", "credits" or "license" for more information. 33 | >>> from crypto_history import gather 34 | >>> gather('20170101', '20170102', ['ethereum']) 35 | (['Coin', 'Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Market Cap'], [[], ['ethereum', 'Jan 02, 2017', '8.17', '8.44', '8.05', '8.38', '14,579,600', '714,900,000'], ['ethereum', 'Jan 01, 2017', '7.98', '8.47', '7.98', '8.17', '14,731,700', '698,149,000']]) 36 | ``` 37 | 38 | 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # custom 2 | bin/ 3 | include/ 4 | .DS_Store 5 | pip-selfcheck.json 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | env/ 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | 34 | # PyInstaller 35 | # Usually these files are written by a python script from a template 36 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 37 | *.manifest 38 | *.spec 39 | 40 | # Installer logs 41 | pip-log.txt 42 | pip-delete-this-directory.txt 43 | 44 | # Unit test / coverage reports 45 | htmlcov/ 46 | .tox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *.cover 53 | .hypothesis/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # pyenv 80 | .python-version 81 | 82 | # celery beat schedule file 83 | celerybeat-schedule 84 | 85 | # SageMath parsed files 86 | *.sage.py 87 | 88 | # dotenv 89 | .env 90 | 91 | # virtualenv 92 | .venv 93 | venv/ 94 | ENV/ 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | .spyproject 99 | 100 | # Rope project settings 101 | .ropeproject 102 | 103 | # mkdocs documentation 104 | /site 105 | 106 | # mypy 107 | .mypy_cache/ 108 | -------------------------------------------------------------------------------- /crypto_history.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """Script to gather historical cryptocurrency data from coinmarketcap.com (cmc) """ 4 | 5 | import json 6 | import requests 7 | from bs4 import BeautifulSoup 8 | import csv 9 | import sys 10 | from time import sleep 11 | 12 | 13 | def CoinNames(): 14 | """Gets ID's of all coins on cmc""" 15 | 16 | names = [] 17 | response = requests.get("https://api.coinmarketcap.com/v1/ticker/?limit=0") 18 | respJSON = json.loads(response.text) 19 | for i in respJSON: 20 | names.append(i['id']) 21 | return names 22 | 23 | def gather(startdate, enddate, names): 24 | historicaldata = [] 25 | counter = 1 26 | 27 | if len(names) == 0: 28 | names = CoinNames() 29 | 30 | for coin in names: 31 | sleep(10) 32 | r = requests.get("https://coinmarketcap.com/currencies/{0}/historical-data/?start={1}&end={2}".format(coin, startdate, enddate)) 33 | data = r.text 34 | soup = BeautifulSoup(data, "html.parser") 35 | table = soup.find('table', attrs={ "class" : "table"}) 36 | 37 | #Add table header to list 38 | if len(historicaldata) == 0: 39 | headers = [header.text for header in table.find_all('th')] 40 | headers.insert(0, "Coin") 41 | 42 | for row in table.find_all('tr'): 43 | currentrow = [val.text for val in row.find_all('td')] 44 | if(len(currentrow) != 0): 45 | currentrow.insert(0, coin) 46 | historicaldata.append(currentrow) 47 | 48 | print("Coin Counter -> " + str(counter), end='\r') 49 | counter += 1 50 | return headers, historicaldata 51 | 52 | def _gather(startdate, enddate): 53 | """ Scrape data off cmc""" 54 | 55 | if(len(sys.argv) == 3): 56 | names = CoinNames() 57 | else: 58 | names = [sys.argv[3]] 59 | 60 | headers, historicaldata = gather(startdate, enddate, names) 61 | 62 | Save(headers, historicaldata) 63 | 64 | def Save(headers, rows): 65 | 66 | if(len(sys.argv) == 3): 67 | FILE_NAME = "HistoricalCoinData.csv" 68 | else: 69 | FILE_NAME = sys.argv[3] + ".csv" 70 | 71 | with open(FILE_NAME, 'w') as f: 72 | writer = csv.writer(f) 73 | writer.writerow(headers) 74 | writer.writerows(row for row in rows if row) 75 | print("Finished!") 76 | 77 | if __name__ == "__main__": 78 | 79 | startdate = sys.argv[1] 80 | enddate = sys.argv[2] 81 | 82 | _gather(startdate, enddate) 83 | 84 | --------------------------------------------------------------------------------