├── README.md ├── stock_watcher.py └── blockspring_stock_watcher.py /README.md: -------------------------------------------------------------------------------- 1 | # StockWatcherAPI 2 | 3 | 4 | 5 | Given 1 or more stock symbols, this API returns a Python dictionary where the key is a stock symbol and the value is another Python dictionary with all of the information about the stock. This sub-dictionary contains the financial data, headlines, and list of related companies for the corresponding stock symbol. 6 | 7 | Note that this API will try to return as much information as possible - in other words, if there is an error in retrieving 1 bit of info, it will still try to return all others. If it can't retrieve anything at the moment, and the stock symbol is valid, then please try again; this means that the Yahoo servers weren't able to respond with the necessary info at that moment. 8 | 9 | Visit https://api.blockspring.com/SaiWebApps/cd5a999185baa03ed10bdd30c13f25af to see the API in action. 10 | -------------------------------------------------------------------------------- /stock_watcher.py: -------------------------------------------------------------------------------- 1 | import urllib2 2 | import json 3 | import re 4 | 5 | # BEGIN HELPER FUNCTIONS 6 | 7 | def _get_page_json_data(url): 8 | ''' 9 | Retrieve the contents at the specified "url," and convert them from 10 | JSON format to a Python dict. Note that this function assumes that the 11 | contents at "url" are in JSON format. 12 | 13 | Return None if something goes wrong. 14 | ''' 15 | try: 16 | response = urllib2.urlopen(url).read() 17 | return json.loads(response) 18 | except: 19 | return None 20 | 21 | 22 | def _get_url(symbol_list): 23 | ''' 24 | Helper method for _fetch_financials. Returns URL indicating where 25 | info about stocks in "symbol_list" can be found. 26 | ''' 27 | # YQL API for Yahoo Finance Quotes 28 | URL_HEAD = 'https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(' 29 | URL_TAIL = ')&format=json&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys&callback=' 30 | 31 | symbols_in_quotes = map(lambda elem: '\"' + elem + '\"', symbol_list) 32 | encoded = ','.join(symbols_in_quotes).strip(',') 33 | return ''.join([URL_HEAD, urllib2.quote(encoded), URL_TAIL]) 34 | 35 | 36 | def _remove_html_tags(json_data): 37 | ''' 38 | Helper method that filters out the HTML tags within the JSON financial 39 | data in _fetch_financials. 40 | ''' 41 | for data_dict in json_data: 42 | for key in data_dict: 43 | if not data_dict[key]: 44 | continue 45 | data_dict[key] = re.sub('<[^>]*>', '', data_dict[key]) 46 | 47 | 48 | def _fetch_financials(symbol_list): 49 | ''' 50 | Return a list of dictionaries containing financial data about 51 | the companies specified in "symbol_list." If there is an error, 52 | return None. 53 | ''' 54 | # Get the information. 55 | url = _get_url(symbol_list) 56 | all_info = _get_page_json_data(url) 57 | if not all_info or not all_info['query'] or not all_info['query']['results'] or \ 58 | not all_info['query']['results']['quote']: 59 | return None 60 | 61 | # json_stock_info['query']['results']['quote'] will be a single dict if client 62 | # requested data for 1 company alone or a list of dicts if client requested 63 | # data for multiple companies. 64 | json_stock_info = all_info['query']['results']['quote'] 65 | # So, if json_stock_info is a dict, add to a list, and then return the list. Otherwise, return as is. 66 | output = json_stock_info if len(symbol_list) > 1 else [json_stock_info] 67 | _remove_html_tags(output) 68 | return output 69 | 70 | 71 | def _fetch_headlines(symbol_list): 72 | ''' 73 | Fetch the headlines for each company in "symbol_list" from its respective 74 | Yahoo Finance profile page. Return a dict, where the key is the company symbol 75 | and the value is another dict containing the headlines' hyperlinks and contents 76 | (mapped to the key 'headlines_href' and 'headlines_content' respectively). 77 | ''' 78 | URL_HEAD = "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20html%20where%20url%3D'http%3A%2F%2Ffinance.yahoo.com%2Fq%3Fs%3D" 79 | URL_TAIL = "'%20and%20xpath%3D'%2F%2Fdiv%5B%40id%3D%22yfi_headlines%22%5D%2Fdiv%5B2%5D%2Ful%2Fli%2Fa'&format=json&callback=" 80 | 81 | headlines_map = {} # Key = company, value = list of headlines for that company 82 | 83 | # For each specified symbol, retrieve the headlines from Yahoo Finance, and add 84 | # a dict entry (company symbol, {'headlines_href': list of headlines' hyperlinks, 85 | # 'headlines_content': list of headlines' content}). 86 | for symbol in symbol_list: 87 | url = ''.join([URL_HEAD, symbol, URL_TAIL]) 88 | all_info = _get_page_json_data(url) 89 | if not all_info or not all_info.get('query', None) or not all_info['query'].get('results', None) \ 90 | or not all_info['query']['results'].get('a', None): 91 | continue 92 | 93 | # headlines = list of dicts with 2 keys: href (headlines' hyperlinks) and 94 | # content (headlines' textual contents) 95 | headlines = all_info['query']['results']['a'] 96 | headlines_href = map(lambda elem: elem['href'], headlines) 97 | headlines_contents = map(lambda elem: elem['content'], headlines) 98 | headlines_map[symbol] = {'headlines_href': headlines_href, 'headlines_content': headlines_contents} 99 | 100 | return headlines_map 101 | 102 | 103 | def _fetch_related_companies(symbol_list): 104 | ''' 105 | Return a dict, where the key is a company symbol from 'symbol_list' and 106 | the value is a list of related companies' symbols. 107 | ''' 108 | URL_HEAD = "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20html%20where\%20url%3D'http%3A%2F%2Ffinance.yahoo.com%2Fq%3Fs%3D" 109 | URL_TAIL = "'%20and%20xpath%3D'%2F%2Fdiv%5B%40id%3D%22yfi_related_tickers%22%5D%2Fp%2Fspan%2Fa%2Fstrong'&format=json&callback=" 110 | 111 | related_companies_map = {} 112 | for symbol in symbol_list: 113 | json_data = _get_page_json_data(''.join([URL_HEAD, symbol, URL_TAIL])) 114 | if not json_data or not json_data.get('query', None) or not json_data['query'].get('results', None) \ 115 | or not json_data['query']['results'].get('strong', None): 116 | continue 117 | related_companies_list = json_data['query']['results']['strong'] 118 | related_companies_map[symbol] = related_companies_list 119 | 120 | return related_companies_map 121 | 122 | 123 | def _add_related_companies_to_aggregate_data(aggregate_map, related_companies_map): 124 | ''' 125 | Helper method that synthesizes the data gathered by _fetch_headlines with 126 | _fetch_related_companies' data. 127 | aggregate_map: contains headlines data, key = symbol, value = dict w/ headlines info; 128 | will contain integrated results 129 | related_companies_map: key = symbol, value = list of related companies 130 | ''' 131 | if not related_companies_map: 132 | return 133 | for symbol in related_companies_map: 134 | aggregate_map[symbol]['related_companies'] = related_companies_map[symbol] 135 | 136 | 137 | def _add_financial_data_to_aggregate_data(aggregate_map, financials_map): 138 | ''' 139 | Helper method that synthesizes the data gathered by _fetch_financials with 140 | the data gathered from the other _fetch helpers. 141 | ''' 142 | if not financials_map: 143 | return None 144 | 145 | # data_dict = list of dicts, where each dict has financial stats about a company (symbol) 146 | # all_data = dict where key = company symbol, value = dict with headlines' info + related companies list 147 | # Merge financials dict with all_data[company symbol]. 148 | for data_dict in financials_map: 149 | # Pop 'symbol' & 'Symbol' since all_data already captures this info in the key. 150 | symbol = data_dict.pop('symbol', None) 151 | del data_dict['Symbol'] 152 | aggregate_map[symbol].update(data_dict) 153 | 154 | # END HELPER FUNCTIONS 155 | 156 | 157 | # BEGIN CLIENT-FACING (API) FUNCTIONS 158 | 159 | def get_current_data(symbol_list): 160 | ''' 161 | If successful, return a list of dicts, where each dict contains information about the 162 | companies specified in "symbol_list." If error, then return None. 163 | Company info gathered from Yahoo Finance, consists of headlines, financials, and related companies. 164 | ''' 165 | if not symbol_list: 166 | return None 167 | 168 | all_data = _fetch_headlines(symbol_list) 169 | if not all_data: 170 | all_data = dict((symbol, {}) for symbol in symbol_list) 171 | 172 | _add_related_companies_to_aggregate_data(all_data, _fetch_related_companies(symbol_list)) 173 | _add_financial_data_to_aggregate_data(all_data, _fetch_financials(symbol_list)) 174 | return all_data 175 | 176 | # END CLIENT-FACING (API) FUNCTIONS 177 | -------------------------------------------------------------------------------- /blockspring_stock_watcher.py: -------------------------------------------------------------------------------- 1 | import urllib2 2 | import json 3 | import re 4 | import blockspring 5 | 6 | # BEGIN HELPER FUNCTIONS 7 | 8 | def _get_page_json_data(url): 9 | ''' 10 | Retrieve the contents at the specified "url," and convert them from 11 | JSON format to a Python dict. Note that this function assumes that the 12 | contents at "url" are in JSON format. 13 | 14 | Return None if something goes wrong. 15 | ''' 16 | try: 17 | response = urllib2.urlopen(url).read() 18 | return json.loads(response) 19 | except: 20 | return None 21 | 22 | 23 | def _get_url(symbol_list): 24 | ''' 25 | Helper method for _fetch_financials. Returns URL indicating where 26 | info about stocks in "symbol_list" can be found. 27 | ''' 28 | # YQL API for Yahoo Finance Quotes 29 | URL_HEAD = 'https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20yahoo.finance.quotes%20where%20symbol%20in%20(' 30 | URL_TAIL = ')&format=json&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys&callback=' 31 | 32 | symbols_in_quotes = map(lambda elem: '\"' + elem + '\"', symbol_list) 33 | encoded = ','.join(symbols_in_quotes).strip(',') 34 | return ''.join([URL_HEAD, urllib2.quote(encoded), URL_TAIL]) 35 | 36 | 37 | def _remove_html_tags(json_data): 38 | ''' 39 | Helper method that filters out the HTML tags within the JSON financial 40 | data in _fetch_financials. 41 | ''' 42 | for data_dict in json_data: 43 | for key in data_dict: 44 | if not data_dict[key]: 45 | continue 46 | data_dict[key] = re.sub('<[^>]*>', '', data_dict[key]) 47 | 48 | 49 | def _fetch_financials(symbol_list): 50 | ''' 51 | Return a list of dictionaries containing financial data about 52 | the companies specified in "symbol_list." If there is an error, 53 | return None. 54 | ''' 55 | # Get the information. 56 | url = _get_url(symbol_list) 57 | all_info = _get_page_json_data(url) 58 | if not all_info or not all_info['query'] or not all_info['query']['results'] or \ 59 | not all_info['query']['results']['quote']: 60 | return None 61 | 62 | # json_stock_info['query']['results']['quote'] will be a single dict if client 63 | # requested data for 1 company alone or a list of dicts if client requested 64 | # data for multiple companies. 65 | json_stock_info = all_info['query']['results']['quote'] 66 | # So, if json_stock_info is a dict, add to a list, and then return the list. Otherwise, return as is. 67 | output = json_stock_info if len(symbol_list) > 1 else [json_stock_info] 68 | _remove_html_tags(output) 69 | return output 70 | 71 | 72 | def _fetch_headlines(symbol_list): 73 | ''' 74 | Fetch the headlines for each company in "symbol_list" from its respective 75 | Yahoo Finance profile page. Return a dict, where the key is the company symbol 76 | and the value is another dict containing the headlines' hyperlinks and contents 77 | (mapped to the key 'headlines_href' and 'headlines_content' respectively). 78 | ''' 79 | URL_HEAD = "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20html%20where%20url%3D'http%3A%2F%2Ffinance.yahoo.com%2Fq%3Fs%3D" 80 | URL_TAIL = "'%20and%20xpath%3D'%2F%2Fdiv%5B%40id%3D%22yfi_headlines%22%5D%2Fdiv%5B2%5D%2Ful%2Fli%2Fa'&format=json&callback=" 81 | 82 | headlines_map = {} # Key = company, value = list of headlines for that company 83 | 84 | # For each specified symbol, retrieve the headlines from Yahoo Finance, and add 85 | # a dict entry (company symbol, {'headlines_href': list of headlines' hyperlinks, 86 | # 'headlines_content': list of headlines' content}). 87 | for symbol in symbol_list: 88 | url = ''.join([URL_HEAD, symbol, URL_TAIL]) 89 | all_info = _get_page_json_data(url) 90 | if not all_info or not all_info.get('query', None) or not all_info['query'].get('results', None) \ 91 | or not all_info['query']['results'].get('a', None): 92 | continue 93 | 94 | # headlines = list of dicts with 2 keys: href (headlines' hyperlinks) and 95 | # content (headlines' textual contents) 96 | headlines = all_info['query']['results']['a'] 97 | headlines_href = map(lambda elem: elem['href'], headlines) 98 | headlines_contents = map(lambda elem: elem['content'], headlines) 99 | headlines_map[symbol] = {'headlines_href': headlines_href, 'headlines_content': headlines_contents} 100 | 101 | return headlines_map 102 | 103 | 104 | def _fetch_related_companies(symbol_list): 105 | ''' 106 | Return a dict, where the key is a company symbol from 'symbol_list' and 107 | the value is a list of related companies' symbols. 108 | ''' 109 | URL_HEAD = "https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20html%20where\%20url%3D'http%3A%2F%2Ffinance.yahoo.com%2Fq%3Fs%3D" 110 | URL_TAIL = "'%20and%20xpath%3D'%2F%2Fdiv%5B%40id%3D%22yfi_related_tickers%22%5D%2Fp%2Fspan%2Fa%2Fstrong'&format=json&callback=" 111 | 112 | related_companies_map = {} 113 | for symbol in symbol_list: 114 | json_data = _get_page_json_data(''.join([URL_HEAD, symbol, URL_TAIL])) 115 | if not json_data or not json_data.get('query', None) or not json_data['query'].get('results', None) \ 116 | or not json_data['query']['results'].get('strong', None): 117 | continue 118 | related_companies_list = json_data['query']['results']['strong'] 119 | related_companies_map[symbol] = related_companies_list 120 | 121 | return related_companies_map 122 | 123 | 124 | def _add_related_companies_to_aggregate_data(aggregate_map, related_companies_map): 125 | ''' 126 | Helper method that synthesizes the data gathered by _fetch_headlines with 127 | _fetch_related_companies' data. 128 | aggregate_map: contains headlines data, key = symbol, value = dict w/ headlines info; 129 | will contain integrated results 130 | related_companies_map: key = symbol, value = list of related companies 131 | ''' 132 | if not related_companies_map: 133 | return 134 | for symbol in related_companies_map: 135 | aggregate_map[symbol]['related_companies'] = related_companies_map[symbol] 136 | 137 | 138 | def _add_financial_data_to_aggregate_data(aggregate_map, financials_map): 139 | ''' 140 | Helper method that synthesizes the data gathered by _fetch_financials with 141 | the data gathered from the other _fetch helpers. 142 | ''' 143 | if not financials_map: 144 | return None 145 | 146 | # data_dict = list of dicts, where each dict has financial stats about a company (symbol) 147 | # all_data = dict where key = company symbol, value = dict with headlines' info + related companies list 148 | # Merge financials dict with all_data[company symbol]. 149 | for data_dict in financials_map: 150 | # Pop 'symbol' & 'Symbol' since all_data already captures this info in the key. 151 | symbol = data_dict.pop('symbol', None) 152 | del data_dict['Symbol'] 153 | if not symbol in aggregate_map: 154 | continue 155 | aggregate_map[symbol].update(data_dict) 156 | 157 | # END HELPER FUNCTIONS 158 | 159 | 160 | # BEGIN CLIENT-FACING (API) FUNCTIONS 161 | 162 | def get_current_data(symbol_list): 163 | ''' 164 | If successful, return a list of dicts, where each dict contains information about the 165 | companies specified in "symbol_list." If error, then return None. 166 | Company info gathered from Yahoo Finance, consists of headlines, financials, and related companies. 167 | ''' 168 | if not symbol_list: 169 | return None 170 | 171 | all_data = _fetch_headlines(symbol_list) 172 | if not all_data: 173 | all_data = dict((symbol, {}) for symbol in symbol_list) 174 | 175 | _add_related_companies_to_aggregate_data(all_data, _fetch_related_companies(symbol_list)) 176 | _add_financial_data_to_aggregate_data(all_data, _fetch_financials(symbol_list)) 177 | return all_data 178 | 179 | # END CLIENT-FACING (API) FUNCTIONS 180 | 181 | def block(request, response): 182 | symbols_string = request.params.get('symbols', None) 183 | if not symbols_string: 184 | response.addOutput('stock_info', 'Please enter at least 1 valid stock symbol.') 185 | response.end() 186 | return 187 | 188 | symbols_list = [symbol.strip() for symbol in symbols_string.split(',')] 189 | output = get_current_data(symbols_list) 190 | if not output: 191 | output = 'Oops! There was an error in retrieving and synthesizing the stock info. Please try again.' 192 | response.addOutput('stock_info', output) 193 | response.end() 194 | 195 | blockspring.define(block) 196 | --------------------------------------------------------------------------------