├── .gitignore ├── LICENSE ├── README.md ├── ib-info.py └── stock-hist-data-download.py /.gitignore: -------------------------------------------------------------------------------- 1 | data 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Florian La Roche 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | TWS API examples for Interactive Brokers (IB) 2 | ============================================= 3 | 4 | [Interactive Brokers (IB)](https://www.interactivebrokers.com/en/home.php) allows to connect with their 5 | trading software [Trader Workstation (TWS)](https://www.interactivebrokers.com/en/trading/tws.php) 6 | through their [TWS API](https://interactivebrokers.github.io/tws-api/). 7 | 8 | 9 | Enable TWS API access for your local computer 10 | --------------------------------------------- 11 | 12 | You first need to start TWS on your computer and within the settings menue you have to 13 | enable TWS API access for your "localhost" network interface (127.0.0.1) on port 7496. 14 | For paper trading (demo/test account) this is port 7497 per default. 15 | This allows to run scripts on the same machine you run TWS on. 16 | 17 | Instead of IB TWS, you can also use IB Gateway. This uses port 4002 per default for 18 | paper trading (demo/test account) and 4001 for an active/real/live account. 19 | 20 | Check this: . 21 | 22 | Once you start scripts connecting to your TWS, you can see also a new tab "API" on your TWS. 23 | 24 | 25 | Official TWS API software from IB 26 | --------------------------------- 27 | 28 | Official TWS API software from Interactive Brokers can be found on 29 | . 30 | 31 | Discussions around the TWS API are best on . 32 | 33 | Please also check the FAQ at: . 34 | 35 | 36 | Install ib_async for python 37 | --------------------------- 38 | 39 | [ib_async](https://github.com/ib-api-reloaded/ib_async) is another python API to connect to your TWS 40 | with docu at and 41 | discussions at . 42 | (Original project at .) 43 | 44 | To install ib_async, first install python3 and then run: 45 | 
 46 | pip3 install ib_async
 47 |
48 | 49 | To update ib_async later on, run: 50 | 
 51 | pip3 install --upgrade ib_async
 52 |
53 | 54 | 55 | python library pandas 56 | --------------------- 57 | 58 | [pandas](https://pandas.pydata.org/) is a useful additional python library 59 | for data analysis and manipulation. 60 | 61 | Install on Debian or Ubuntu Linux with: 62 | 
 63 | sudo apt-get install python3-pandas
 64 |
65 | 66 | Or you can install via pip3: 67 | 
 68 | pip3 install pandas
 69 |
70 | 71 | For updates run: 72 | 
 73 | pip3 install --upgrade pandas
 74 |
75 | 76 | 77 | historic stock data download 78 | ---------------------------- 79 | 80 | Example script which downloads historic stock data for all 81 | companies of the DOW, SP500 and Nasdaq100 indices. 82 | 83 | The data is stored into the subdirectory "data" per default, 84 | so please create this directory before calling this script. 85 | 86 | See [stock-hist-data-download.py](stock-hist-data-download.py). 87 | 88 | How to update the index list of the SP500 and Nasdaq100: 89 | 
 90 | python3 stock-hist-data-download.py --list-index > TMPFILE
 91 | diff -u stock-hist-data-download.py TMPFILE
 92 |
93 | 94 | Also check out . 95 | 96 | 97 | links to similar / further projects 98 | ----------------------------------- 99 | 100 | - automate running IB TWS: 101 | - 102 | - Github topics to look at: 103 | - 104 | - 105 | - 106 | - 107 | - 108 | - IB ruby: 109 | - 110 | - 111 | - 112 | 113 | 114 | IB flex queries 115 | --------------- 116 | 117 | A bit different to the TWS API are flex queries and downloading/parsing of them. 118 | 119 | Here some projects around this: 120 | 121 | - 122 | 123 | 124 | -------------------------------------------------------------------------------- /ib-info.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright (C) 2023 Florian La Roche 4 | # 5 | # Tested on Debian-11 and Debian-12 (Should run fine on Ubuntu.): 6 | # sudo apt-get install python3-rich python3-pandas 7 | # python3 -m venv venv 8 | # . venv/bin/activate 9 | # pip3 install ib_async 10 | # 11 | 12 | import sys 13 | import locale 14 | import logging 15 | import ib_async 16 | 17 | from rich.console import Console 18 | from rich.panel import Panel 19 | from rich.table import Table 20 | 21 | # Turn off some of the more annoying logging output from ib_async 22 | #logging.getLogger("ib_async.ib").setLevel(logging.ERROR) 23 | #logging.getLogger("ib_async.wrapper").setLevel(logging.CRITICAL) 24 | 25 | # XXX How to detect base currency? 26 | BASE = '€' 27 | 28 | def print_data(value): 29 | if value >= 980000: 30 | return locale.format_string("%d", round(value / 1000), grouping=True) + 'T' 31 | return locale.format_string("%d", round(value), grouping=True) 32 | 33 | def show_account2(ib): 34 | #print([v for v in ib.accountValues() 35 | # if v.tag == 'NetLiquidationByCurrency' and v.currency == 'BASE']) 36 | portfolio = ib.portfolio() # account= 37 | if portfolio: 38 | print('Portfolio:') 39 | for p in portfolio: 40 | print(p) 41 | positions = ib.positions() 42 | if positions: 43 | print('Positions:') 44 | for p in positions: 45 | print(p) 46 | trades = ib.trades() 47 | if trades: 48 | print('Trades:') 49 | for t in trades: 50 | print(t) 51 | orders = ib.orders() 52 | if orders: 53 | print('Orders:') 54 | for o in orders: 55 | print(o) 56 | 57 | def show_account(ib, console, verbose): 58 | accounts = ib.managedAccounts() 59 | accountValues = ib.accountValues() 60 | if verbose >= 3 and accountValues: 61 | print('Account Values:') 62 | for p in accountValues: 63 | print(p) 64 | accountSummary = ib.accountSummary() 65 | if verbose >= 3 and accountSummary: 66 | print('Account Summary:') 67 | for p in accountSummary: 68 | print(p) 69 | nav = .0 70 | nav_str = '' 71 | cash = .0 72 | cash_str = '' 73 | cash_percent = '' 74 | margin = '' 75 | for p in accountSummary: 76 | if p.account == 'All' and p.tag == 'TotalCashBalance' and p.currency == 'BASE': 77 | cash = float(p.value) 78 | cash_str = print_data(cash) + BASE 79 | if p.tag == 'Cushion': 80 | margin = str(100 - round(float(p.value) * 100)) + '%' 81 | if p.tag == 'NetLiquidation': 82 | nav = float(p.value) 83 | nav_str = print_data(nav) + BASE 84 | if nav > .0: 85 | cash_percent = str(round(cash * 100 / nav)) + '%' 86 | if len(accounts) > 1: 87 | table = Table(title="Accounts: %s" % (",".join(accounts))) 88 | else: 89 | table = Table(title="Account: %s" % (",".join(accounts))) 90 | # XXX add info on time of last update 91 | if len(accounts) > 1: 92 | table.add_column("Accounts: %s" % (",".join(accounts))) 93 | else: 94 | table.add_column("Account: %s" % (",".join(accounts))) 95 | table.add_column(f"NetLiq: {nav_str}") 96 | table.add_column(f"Margin: {margin}") 97 | table.add_column(f"Cash: {cash_str} ({cash_percent})") 98 | #table.add_column("US-T: 120 T€ (7%)") 99 | #table.add_column("Sold Options: -100(12000) (0,05%)") 100 | #table.add_column("Stocks: 400 T€ (20%)") 101 | console.print(Panel(table)) 102 | 103 | if verbose >= 3: 104 | show_account2(ib) 105 | 106 | def usage(): 107 | print('ib-info.py ' + 108 | '[--host=127.0.0.1][--port=7496][--client-id=0]' + 109 | '[--help][--verbose][--debug][--quiet]') 110 | 111 | def main(argv): 112 | import getopt 113 | 114 | locale.setlocale(locale.LC_ALL, '') 115 | #locale.setlocale(locale.LC_ALL, 'de_DE') 116 | #print(locale.getlocale()) 117 | #for key, value in locale.localeconv().items(): 118 | # print("%s: %s" % (key, value)) 119 | #logger = logging.getLogger(__name__) 120 | 121 | verbose = 1 122 | 123 | # Connect params to your Interactive Brokers (IB) TWS or IB Gateway: 124 | host = '127.0.0.1' 125 | #port = 7497 # TWS paper account (demo/test) 126 | port = 7496 # TWS active/real/live account 127 | #port = 4002 # IB Gateway paper account (demo/test) 128 | #port = 4001 # IB Gateway active/real/live account 129 | client_id = 0 130 | 131 | try: 132 | opts, args = getopt.getopt(argv, 'dhqv', ['list-index', 'help', 133 | 'host=', 'port=', 'client-id=' 134 | 'data-dir=', 'quiet', 'verbose', 'debug']) 135 | except getopt.GetoptError: 136 | usage() 137 | sys.exit(2) 138 | for opt, arg in opts: 139 | if opt in ('-h', '--help'): 140 | usage() 141 | sys.exit() 142 | elif opt == '--host': 143 | host = arg 144 | elif opt == '--port': 145 | port = int(arg) 146 | elif opt == '--client-id': 147 | client_id = int(arg) 148 | elif opt in ('-v', '--verbose'): 149 | verbose += 1 150 | elif opt in ('-d', '--debug'): 151 | verbose = 3 152 | elif opt in ('-q', '--quiet'): 153 | verbose = 0 154 | #if len(args) == 0: 155 | # usage() 156 | # sys.exit() 157 | 158 | ib_async.util.allowCtrlC() 159 | 160 | if verbose == 0: 161 | ib_async.util.logToConsole(logging.ERROR) 162 | elif verbose == 1: 163 | ib_async.util.logToConsole(logging.WARNING) 164 | elif verbose == 2: 165 | ib_async.util.logToConsole(logging.INFO) 166 | elif verbose >= 3: 167 | ib_async.util.logToConsole(logging.DEBUG) 168 | #ib_async.util.logToFile("ib.log", logging.WARNING) 169 | 170 | ib = ib_async.IB() 171 | try: 172 | ib.connect(host, port, clientId=client_id) # account=, timeout= 173 | except ConnectionRefusedError: 174 | sys.exit(1) 175 | 176 | console = Console() 177 | 178 | if False: 179 | table = Table(title="Account summary") 180 | table.add_column("Item") 181 | table.add_column("Value", justify="right") 182 | table.add_row("Net liquidation", "0") 183 | table.add_row("Maintenance margin", "0") 184 | table.add_row("Total cash", "0") 185 | table.add_section() 186 | table.add_row("Total cash", "0") 187 | console.print(Panel(table)) 188 | 189 | 190 | show_account(ib, console, verbose) 191 | 192 | # ib.reqMarketDataType(self.config["account"]["market_data_type"]) 193 | 194 | ib.disconnect() 195 | 196 | if __name__ == '__main__': 197 | main(sys.argv[1:]) 198 | -------------------------------------------------------------------------------- /stock-hist-data-download.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3 2 | # 3 | # Download Interactive Brokers (IB) historical stock market data 4 | # by connecting to the local TWS software. 5 | # Data is stored per default into the directory "data". 6 | # 7 | # Required (tested on Debian 11 and 12): 8 | # sudo apt-get install python3-pandas 9 | # pip3 install ib_async 10 | # 11 | # Old requirement: 12 | # sudo apt-get install python3-sqlalchemy-utils 13 | # 14 | # TODO: 15 | # - Also support parquet fileformat in addition to csv: pd.read_parquet() pd.to_parquet(). 16 | # - Cache also empty data returns? 17 | # - Note date of data download and date of last check 18 | # - Add account id into sql filename? 19 | # - Check m_primaryExch as official exchange, also see m_validExchanges. 20 | # - Use also ARCA as exchange? 21 | # - List of exchanges? GLOBEX, CMECRYPTO, ECBOT, NYMEX 22 | # - Check: https://interactivebrokers.github.io/tws-api/contract_details.html 23 | # - Check time delta handling: hourly data for 2023 contains some hours from 2022. 24 | # - Does hourly data make sense? Often half hour data is part of RTH. ??? 25 | # - Configuration: allow env config settings for TWS connection and separate config file. 26 | # - Robust constantly running prog with re-connects to TWS to allow automatic trading. 27 | # 28 | # pylint: disable=C0103,C0114,C0116,C0413,C0415,W0603,W0614 29 | # 30 | 31 | import sys 32 | import os 33 | import time 34 | import logging 35 | 36 | import pandas 37 | import ib_async 38 | 39 | 40 | # https://en.wikipedia.org/wiki/List_of_S%26P_500_companies 41 | # also check: https://github.com/deltaray-io/US-Stock-Symbols 42 | SP500: tuple[str, ...] = ( 43 | 'A', 'AAL', 'AAPL', 'ABBV', 'ABNB', 'ABT', 'ACGL', 'ACN', 'ADBE', 'ADI', 44 | 'ADM', 'ADP', 'ADSK', 'AEE', 'AEP', 'AES', 'AFL', 'AIG', 'AIZ', 'AJG', 45 | 'AKAM', 'ALB', 'ALGN', 'ALL', 'ALLE', 'AMAT', 'AMCR', 'AMD', 'AME', 'AMGN', 46 | 'AMP', 'AMT', 'AMZN', 'ANET', 'ANSS', 'AON', 'AOS', 'APA', 'APD', 'APH', 47 | 'APTV', 'ARE', 'ATO', 'AVB', 'AVGO', 'AVY', 'AWK', 'AXON', 'AXP', 'AZO', 48 | 'BA', 'BAC', 'BALL', 'BAX', 'BBWI', 'BBY', 'BDX', 'BEN', 'BF.B', 'BG', 49 | 'BIIB', 'BIO', 'BK', 'BKNG', 'BKR', 'BLDR', 'BLK', 'BMY', 'BR', 'BRK.B', 50 | 'BRO', 'BSX', 'BWA', 'BX', 'BXP', 'C', 'CAG', 'CAH', 'CARR', 'CAT', 'CB', 51 | 'CBOE', 'CBRE', 'CCI', 'CCL', 'CDAY', 'CDNS', 'CDW', 'CE', 'CEG', 'CF', 52 | 'CFG', 'CHD', 'CHRW', 'CHTR', 'CI', 'CINF', 'CL', 'CLX', 'CMA', 'CMCSA', 53 | 'CME', 'CMG', 'CMI', 'CMS', 'CNC', 'CNP', 'COF', 'COO', 'COP', 'COR', 54 | 'COST', 'CPB', 'CPRT', 'CPT', 'CRL', 'CRM', 'CSCO', 'CSGP', 'CSX', 'CTAS', 55 | 'CTLT', 'CTRA', 'CTSH', 'CTVA', 'CVS', 'CVX', 'CZR', 'D', 'DAL', 'DD', 56 | 'DE', 'DFS', 'DG', 'DGX', 'DHI', 'DHR', 'DIS', 'DLR', 'DLTR', 'DOV', 'DOW', 57 | 'DPZ', 'DRI', 'DTE', 'DUK', 'DVA', 'DVN', 'DXCM', 'EA', 'EBAY', 'ECL', 58 | 'ED', 'EFX', 'EG', 'EIX', 'EL', 'ELV', 'EMN', 'EMR', 'ENPH', 'EOG', 'EPAM', 59 | 'EQIX', 'EQR', 'EQT', 'ES', 'ESS', 'ETN', 'ETR', 'ETSY', 'EVRG', 'EW', 60 | 'EXC', 'EXPD', 'EXPE', 'EXR', 'F', 'FANG', 'FAST', 'FCX', 'FDS', 'FDX', 61 | 'FE', 'FFIV', 'FI', 'FICO', 'FIS', 'FITB', 'FLT', 'FMC', 'FOX', 'FOXA', 62 | 'FRT', 'FSLR', 'FTNT', 'FTV', 'GD', 'GE', 'GEHC', 'GEN', 'GILD', 'GIS', 63 | 'GL', 'GLW', 'GM', 'GNRC', 'GOOG', 'GOOGL', 'GPC', 'GPN', 'GRMN', 'GS', 64 | 'GWW', 'HAL', 'HAS', 'HBAN', 'HCA', 'HD', 'HES', 'HIG', 'HII', 'HLT', 65 | 'HOLX', 'HON', 'HPE', 'HPQ', 'HRL', 'HSIC', 'HST', 'HSY', 'HUBB', 'HUM', 66 | 'HWM', 'IBM', 'ICE', 'IDXX', 'IEX', 'IFF', 'ILMN', 'INCY', 'INTC', 'INTU', 67 | 'INVH', 'IP', 'IPG', 'IQV', 'IR', 'IRM', 'ISRG', 'IT', 'ITW', 'IVZ', 'J', 68 | 'JBHT', 'JBL', 'JCI', 'JKHY', 'JNJ', 'JNPR', 'JPM', 'K', 'KDP', 'KEY', 69 | 'KEYS', 'KHC', 'KIM', 'KLAC', 'KMB', 'KMI', 'KMX', 'KO', 'KR', 'KVUE', 'L', 70 | 'LDOS', 'LEN', 'LH', 'LHX', 'LIN', 'LKQ', 'LLY', 'LMT', 'LNT', 'LOW', 71 | 'LRCX', 'LULU', 'LUV', 'LVS', 'LW', 'LYB', 'LYV', 'MA', 'MAA', 'MAR', 72 | 'MAS', 'MCD', 'MCHP', 'MCK', 'MCO', 'MDLZ', 'MDT', 'MET', 'META', 'MGM', 73 | 'MHK', 'MKC', 'MKTX', 'MLM', 'MMC', 'MMM', 'MNST', 'MO', 'MOH', 'MOS', 74 | 'MPC', 'MPWR', 'MRK', 'MRNA', 'MRO', 'MS', 'MSCI', 'MSFT', 'MSI', 'MTB', 75 | 'MTCH', 'MTD', 'MU', 'NCLH', 'NDAQ', 'NDSN', 'NEE', 'NEM', 'NFLX', 'NI', 76 | 'NKE', 'NOC', 'NOW', 'NRG', 'NSC', 'NTAP', 'NTRS', 'NUE', 'NVDA', 'NVR', 77 | 'NWS', 'NWSA', 'NXPI', 'O', 'ODFL', 'OKE', 'OMC', 'ON', 'ORCL', 'ORLY', 78 | 'OTIS', 'OXY', 'PANW', 'PARA', 'PAYC', 'PAYX', 'PCAR', 'PCG', 'PEAK', 79 | 'PEG', 'PEP', 'PFE', 'PFG', 'PG', 'PGR', 'PH', 'PHM', 'PKG', 'PLD', 'PM', 80 | 'PNC', 'PNR', 'PNW', 'PODD', 'POOL', 'PPG', 'PPL', 'PRU', 'PSA', 'PSX', 81 | 'PTC', 'PWR', 'PXD', 'PYPL', 'QCOM', 'QRVO', 'RCL', 'REG', 'REGN', 'RF', 82 | 'RHI', 'RJF', 'RL', 'RMD', 'ROK', 'ROL', 'ROP', 'ROST', 'RSG', 'RTX', 83 | 'RVTY', 'SBAC', 'SBUX', 'SCHW', 'SHW', 'SJM', 'SLB', 'SNA', 'SNPS', 'SO', 84 | 'SPG', 'SPGI', 'SRE', 'STE', 'STLD', 'STT', 'STX', 'STZ', 'SWK', 'SWKS', 85 | 'SYF', 'SYK', 'SYY', 'T', 'TAP', 'TDG', 'TDY', 'TECH', 'TEL', 'TER', 'TFC', 86 | 'TFX', 'TGT', 'TJX', 'TMO', 'TMUS', 'TPR', 'TRGP', 'TRMB', 'TROW', 'TRV', 87 | 'TSCO', 'TSLA', 'TSN', 'TT', 'TTWO', 'TXN', 'TXT', 'TYL', 'UAL', 'UBER', 88 | 'UDR', 'UHS', 'ULTA', 'UNH', 'UNP', 'UPS', 'URI', 'USB', 'V', 'VFC', 89 | 'VICI', 'VLO', 'VLTO', 'VMC', 'VRSK', 'VRSN', 'VRTX', 'VTR', 'VTRS', 'VZ', 90 | 'WAB', 'WAT', 'WBA', 'WBD', 'WDC', 'WEC', 'WELL', 'WFC', 'WHR', 'WM', 91 | 'WMB', 'WMT', 'WRB', 'WRK', 'WST', 'WTW', 'WY', 'WYNN', 'XEL', 'XOM', 92 | 'XRAY', 'XYL', 'YUM', 'ZBH', 'ZBRA', 'ZION', 'ZTS') 93 | 94 | # old stock symbols who got merged, renamed, removed: 95 | SP500old: tuple[str, ...] = ('FB', 'PVH') 96 | 97 | # https://en.wikipedia.org/wiki/NASDAQ-100 98 | NASDAQ100: tuple[str, ...] = ( 99 | 'ADBE', 'ADP', 'ABNB', 'GOOGL', 'GOOG', 'AMZN', 'AMD', 'AEP', 'AMGN', 100 | 'ADI', 'ANSS', 'AAPL', 'AMAT', 'ASML', 'AZN', 'TEAM', 'ADSK', 'BKR', 101 | 'BIIB', 'BKNG', 'AVGO', 'CDNS', 'CDW', 'CHTR', 'CTAS', 'CSCO', 'CCEP', 102 | 'CTSH', 'CMCSA', 'CEG', 'CPRT', 'CSGP', 'COST', 'CRWD', 'CSX', 'DDOG', 103 | 'DXCM', 'FANG', 'DLTR', 'DASH', 'EA', 'EXC', 'FAST', 'FTNT', 'GEHC', 104 | 'GILD', 'GFS', 'HON', 'IDXX', 'ILMN', 'INTC', 'INTU', 'ISRG', 'KDP', 105 | 'KLAC', 'KHC', 'LRCX', 'LULU', 'MAR', 'MRVL', 'MELI', 'META', 'MCHP', 'MU', 106 | 'MSFT', 'MRNA', 'MDLZ', 'MDB', 'MNST', 'NFLX', 'NVDA', 'NXPI', 'ORLY', 107 | 'ODFL', 'ON', 'PCAR', 'PANW', 'PAYX', 'PYPL', 'PDD', 'PEP', 'QCOM', 'REGN', 108 | 'ROP', 'ROST', 'SIRI', 'SPLK', 'SBUX', 'SNPS', 'TTWO', 'TMUS', 'TSLA', 109 | 'TXN', 'TTD', 'VRSK', 'VRTX', 'WBA', 'WBD', 'WDAY', 'XEL', 'ZS') 110 | 111 | REITS: tuple[str, ...] = ('ARE', 'AMT', 'AVB', 'BXP', 'CPT', 'CBRE', 'CCI', 112 | 'DLR', 'DRE', 'EQUIX', 'EQR', 'ESS', 'EXR', 'FRT', 'PEAK', 'HST', 'INVH', 113 | 'IRM', 'KIM', 'MAA', 'PLD', 'PSA', 'O', 'REG', 'SBAC', 'SPG', 'UDR', 114 | 'VTR', 'VICI', 'VNO', 'WELL', 'WY') 115 | 116 | # Read all companies of the SP500 from wikipedia. 117 | def read_sp500() -> pandas.DataFrame: 118 | table = pandas.read_html('https://en.wikipedia.org/wiki/List_of_S%26P_500_companies') 119 | df = table[0] 120 | #print(df.info()) 121 | #df.drop('SEC filings', axis=1, inplace=True) 122 | return df 123 | 124 | def print_sp500() -> None: 125 | import pprint 126 | df = read_sp500() 127 | #df['Symbol'] = df['Symbol'].str.replace('.', '/') 128 | symbols = df['Symbol'].values.tolist() 129 | symbols.sort() 130 | p = pprint.pformat(symbols, width=79, compact=True, indent=4) 131 | print(p) 132 | # XXX print REITS: df['GICS Sector'] == 'Real Estate' 133 | 134 | def read_nasdaq100() -> pandas.DataFrame: 135 | table = pandas.read_html('https://en.wikipedia.org/wiki/NASDAQ-100') 136 | df = table[4] 137 | return df 138 | 139 | def print_nasdaq100() -> None: 140 | import pprint 141 | df = read_nasdaq100() 142 | #df['Ticker'] = df['Ticker'].str.replace('.', '/') 143 | symbols = df['Ticker'].values.tolist() 144 | p = pprint.pformat(symbols, width=79, compact=True, indent=4) 145 | print(p) 146 | 147 | # CSV datafiles (and also used for sql database): 148 | #csv_dir = None 149 | csv_dir = 'data' 150 | 151 | sql_filename = 'IB.db' 152 | 153 | # database engine: 154 | engine = None 155 | 156 | def open_db(): 157 | global engine 158 | if not csv_dir: 159 | return True 160 | if not os.path.isdir(csv_dir): 161 | print('Directory %s does not exist, please create it.' % csv_dir) 162 | return False 163 | use_sqlalchemy = False 164 | if not use_sqlalchemy: 165 | import sqlite3 166 | #db_file = ':memory:' 167 | db_file = os.path.join(csv_dir, sql_filename) 168 | engine = sqlite3.connect(db_file) 169 | else: 170 | from sqlalchemy import create_engine 171 | #db_file = 'sqlite:///:memory:' 172 | #db_file = 'sqlite:///data/' + sql_filename 173 | db_file = os.path.join('sqlite:///' + csv_dir, sql_filename) 174 | engine = create_engine(db_file) 175 | return True 176 | 177 | def close_db(): 178 | global engine 179 | if not csv_dir: 180 | return 181 | engine.close() 182 | engine = None 183 | 184 | tables = [] 185 | 186 | # Get a list of available database tables. 187 | def getDbTables(): 188 | if not engine: 189 | return [] 190 | dbcurr = engine.cursor() 191 | dbcurr.execute("SELECT name FROM sqlite_master WHERE type='table';") 192 | return [table[0] for table in dbcurr.fetchall()] 193 | 194 | # weekly and daily data is in one big file, everything else is stored 195 | # on a per-year basis 196 | def getTableName(stock, exchange, year, timespan, onetable): 197 | if onetable: 198 | return '%s-%s-%s' % (stock, exchange, timespan) 199 | return '%s-%s-%d-%s' % (stock, exchange, year, timespan) 200 | 201 | # CSV filename, compressed with gzip 202 | def getCsvFilename(table_name): 203 | return os.path.join(csv_dir, table_name + '.csv.gz') 204 | 205 | # Convert IB data into pandas dataframe (df). 206 | def ConvertIB2Dataframe(bars): 207 | df = ib_async.util.df(bars) 208 | df.set_index(['date'], inplace=True) 209 | return df 210 | 211 | def writeIT2(ib, contract, stock, exchange, year, timespan, barSize, duration, 212 | onetable, check=False): 213 | table_name = getTableName(stock, exchange, year, timespan, onetable) 214 | exist = True 215 | if csv_dir: 216 | csv_file = getCsvFilename(table_name) 217 | if not os.path.exists(csv_file): 218 | exist = False 219 | if engine and table_name not in tables: 220 | exist = False 221 | if check: 222 | exist = False 223 | if exist: 224 | return 225 | if onetable: 226 | print(stock, timespan) 227 | else: 228 | print(stock, year, timespan) 229 | bars = ib.reqHistoricalData(contract, endDateTime='%d0101 00:00:00 UTC' % (year + 1), 230 | durationStr=duration, barSizeSetting=barSize, whatToShow='TRADES', # MIDPOINT 231 | useRTH=True) #, formatDate=1) 232 | if not bars: 233 | return 234 | df = ConvertIB2Dataframe(bars) 235 | # Save into CSV file and sql database: 236 | if csv_dir: 237 | df.to_csv(csv_file) 238 | if engine: # and table_name not in tables: 239 | df.to_sql(table_name, engine, if_exists='replace') 240 | if table_name not in tables: 241 | tables.append(table_name) 242 | 243 | def writeIT(ib, stock, exchange, currency, hourly=True): 244 | contract = ib_async.Stock(stock, exchange, currency) 245 | #details = ib.reqContractDetails(contract) 246 | #print(details) 247 | #if details.Contract.secType != 'STK': 248 | # raise 249 | #if details.symbol != stock or details.localSymbol != stock: 250 | # raise 251 | #if details.exchange != exchange: 252 | # raise 253 | # XXX: write down time of fetching/checking data 254 | cur_year = int(time.strftime('%Y')) 255 | writeIT2(ib, contract, stock, exchange, cur_year, 'weekly', '1 week', '40 Y', True) 256 | writeIT2(ib, contract, stock, exchange, cur_year, 'daily', '1 day', '40 Y', True) 257 | if not hourly: 258 | return 259 | # Find first year of data: 260 | startYear = 1980 261 | if csv_dir: 262 | table_name = getTableName(stock, exchange, cur_year, 'weekly', True) 263 | csv_file = getCsvFilename(table_name) 264 | wk = pandas.read_csv(csv_file) # index_col='Date') 265 | startYear = int(wk['date'][0][:4]) 266 | # Download yearly data: 267 | for year in range(cur_year, startYear - 1, -1): 268 | #writeIT2(ib, contract, stock, exchange, year, 'daily', '1 day', '1 Y', False) 269 | if year >= 2004 and hourly: 270 | writeIT2(ib, contract, stock, exchange, year, 'hourly', '1 hour', '1 Y', False) 271 | 272 | def write_some_stocks(ib): 273 | writeIT(ib, 'AAPL', 'SMART', 'USD') 274 | writeIT(ib, 'TSLA', 'SMART', 'USD') 275 | writeIT(ib, 'TSLA', 'NYSE', 'USD') 276 | writeIT(ib, 'TSLA', 'ISLAND', 'USD') 277 | 278 | def write_some_stocks2(ib): 279 | # CSCO FTRCQ IEP RDS.B 280 | stocks = ['APLE', 'BTI', 'CIM', 'D', 'DUK', 'ENB', 'EPD', 281 | 'EPR', 'ETRN', 'FAX', 'GE', 'GTY', 'HBI', 'JNJ', 'KHC', 'LMT', 282 | 'LTC', 'M', 'MA', 'MAIN', 'MMM', 'MMP', 'MO', 'MPW', 'OPI', 283 | 'OZK', 'PM', 'PPL', 'PRU', 'SKT', 'TEVA', 'TSN', 'UHS', 'V', 284 | 'VLO', 'WB', 'WSR'] 285 | for stock in stocks: 286 | writeIT(ib, stock, 'SMART', 'USD') 287 | 288 | # https://en.wikipedia.org/wiki/Dow_Jones_Industrial_Average 289 | def write_dow_stocks(ib): 290 | dow = ['NYSE:MMM', 'NYSE:AXP', 'AMGN', 'AAPL', 'NYSE:BA', 'NYSE:CAT', 'NYSE:CVX', 'CSCO', 291 | 'NYSE:KO', 'NYSE:DOW', 'NYSE:GS', 'NYSE:HD', 'HON', 'NYSE:IBM', 'INTC', 292 | 'NYSE:JNJ', 'NYSE:JPM', 'NYSE:MCD', 'NYSE:MRK', 'MSFT', 'NYSE:NKE', 'NYSE:PG', 293 | 'NYSE:CRM', 'NYSE:TRV', 'NYSE:UNH', 'NYSE:VZ', 'NYSE:V', 'WBA', 'NYSE:WMT', 'NYSE:DIS'] 294 | for stock in dow: 295 | #exchange = 'SMART' 296 | exchange = 'ISLAND' 297 | #exchange = 'NASDAQ' 298 | if stock[:5] == 'NYSE:': 299 | stock = stock[5:] 300 | exchange = 'NYSE' 301 | writeIT(ib, stock, exchange, 'USD', False) 302 | 303 | def write_sp500_stocks(ib): 304 | nyse = ['AAL', 'CSCO', 'KEYS', 'LIN', 'META', 'MNST', 'WELL'] 305 | disable = ['VICI',] 306 | for stock in SP500: 307 | stock = stock.replace('.', ' ') 308 | exchange = 'SMART' 309 | if stock in nyse: 310 | exchange = 'NYSE' 311 | if stock in disable: 312 | continue 313 | writeIT(ib, stock, exchange, 'USD', False) 314 | 315 | def write_nasdaq_stocks(ib): 316 | # https://en.wikipedia.org/wiki/NASDAQ-100#Components 317 | for stock in NASDAQ100: 318 | exchange = 'ISLAND' 319 | #exchange = 'NASDAQ' 320 | writeIT(ib, stock, exchange, 'USD', False) 321 | 322 | def getSPX(): 323 | return Index('SPX', 'CBOE', 'USD', description='SP500 Index') 324 | 325 | def getVIX(): 326 | return Index('VIX', 'CBOE', 'USD', description='CBOE Volatility Index') 327 | 328 | def getADNYSE(): 329 | return Index('AD-NYSE', 'NYSE', 'USD', description='NYSE Advance Decline Index') 330 | 331 | def getDAX(): 332 | return Index('DAX', 'EUREX', 'EUR', description='DAX Performance Index') 333 | 334 | def getVDAX(): 335 | return Index('VDAX', 'EUREX', 'EUR', description='German VDAX Volatility Index') 336 | 337 | def getSTOXX(): 338 | return Index('ESTX50', 'EUREX', 'EUR', description='Dow Jones Euro STOXX50') 339 | 340 | def getVSTOXX(): 341 | return Index('V2TX', 'EUREX', 'EUR', description='VSTOXX Volatility Index') 342 | 343 | def getHSI(): 344 | return Index('HSI', 'HKFE', 'HKD', description='Hang Seng Index') 345 | 346 | def getVHSI(): 347 | return Index('VHSI', 'HKFE', 'HKD', description='Hang Seng Volatility Index') 348 | 349 | def getMiniHSI(): 350 | return Index('MHI', 'HKFE', 'HKD', description='Mini Hang Seng Index') 351 | 352 | def getEURUSD(): 353 | return Forex('EURUSD') 354 | 355 | def usage(): 356 | print('stock-hist-data-download.py ' + 357 | '[--list-index][--data-dir=data]' + 358 | '[--host=127.0.0.1][--port=7496][--client-id=0]' + 359 | '[--help][--verbose][--debug][--quiet]') 360 | 361 | def show_account(ib): 362 | #print([v for v in ib.accountValues() if v.tag == 'NetLiquidationByCurrency' and v.currency == 'BASE']) 363 | if True: 364 | portfolio = ib.portfolio() 365 | if portfolio: 366 | print('Portfolio:') 367 | for p in portfolio: 368 | print(p) 369 | if True: 370 | positions = ib.positions() 371 | if positions: 372 | print('Positions:') 373 | for p in positions: 374 | print(p) 375 | if True: 376 | trades = ib.trades() 377 | if trades: 378 | print('Trades:') 379 | for t in trades: 380 | print(t) 381 | if True: 382 | orders = ib.orders() 383 | if orders: 384 | print('Orders:') 385 | for o in orders: 386 | print(o) 387 | 388 | def main(argv): 389 | global tables, csv_dir 390 | import getopt 391 | verbose = 1 392 | 393 | # Connect params to your Interactive Brokers (IB) TWS or IB Gateway: 394 | host = '127.0.0.1' 395 | #port = 7497 # TWS paper account (demo/test) 396 | port = 7496 # TWS active/real/live account 397 | #port = 4002 # IB Gateway paper account (demo/test) 398 | #port = 4001 # IB Gateway active/real/live account 399 | client_id = 0 400 | 401 | try: 402 | opts, args = getopt.getopt(argv, 'dhqv', ['list-index', 'help', 403 | 'host=', 'port=', 'client-id=' 404 | 'data-dir=', 'quiet', 'verbose', 'debug']) 405 | except getopt.GetoptError: 406 | usage() 407 | sys.exit(2) 408 | for opt, arg in opts: 409 | if opt in ('-h', '--help'): 410 | usage() 411 | sys.exit() 412 | elif opt == '--data-dir': 413 | if arg in ('', 'None'): 414 | csv_dir = None 415 | else: 416 | csv_dir = arg 417 | elif opt == '--list-index': 418 | print_sp500() 419 | print_nasdaq100() 420 | sys.exit(0) 421 | elif opt == '--host': 422 | host = arg 423 | elif opt == '--port': 424 | port = int(arg) 425 | elif opt == '--client-id': 426 | client_id = int(arg) 427 | elif opt in ('-v', '--verbose'): 428 | verbose += 1 429 | elif opt in ('-d', '--debug'): 430 | verbose = 3 431 | elif opt in ('-q', '--quiet'): 432 | verbose = 0 433 | #if len(args) == 0: 434 | # usage() 435 | # sys.exit() 436 | 437 | ib_async.util.allowCtrlC() 438 | 439 | if verbose == 0: 440 | ib_async.util.logToConsole(logging.ERROR) 441 | elif verbose == 1: 442 | ib_async.util.logToConsole(logging.WARNING) 443 | elif verbose == 2: 444 | ib_async.util.logToConsole(logging.INFO) 445 | elif verbose >= 3: 446 | ib_async.util.logToConsole(logging.DEBUG) 447 | #ib_async.util.logToFile("ib.log", logging.WARNING) 448 | 449 | ib = ib_async.IB() 450 | try: 451 | ib.connect(host, port, clientId=client_id) # account 452 | except ConnectionRefusedError: 453 | sys.exit(1) 454 | 455 | #show_account(ib) 456 | 457 | if not open_db(): 458 | sys.exit(3) 459 | tables = getDbTables() 460 | #print(tables) 461 | #trades = pd.read_sql(trades_query, self.dbconn) 462 | 463 | #write_some_stocks(ib) 464 | #write_some_stocks2(ib) 465 | write_dow_stocks(ib) 466 | write_sp500_stocks(ib) 467 | write_nasdaq_stocks(ib) 468 | 469 | #ib.sleep(10) 470 | ib.disconnect() 471 | close_db() 472 | 473 | if __name__ == '__main__': 474 | main(sys.argv[1:]) 475 | --------------------------------------------------------------------------------