├── cron.py ├── leagues.py ├── reget.py ├── match.py ├── get.py ├── info.py └── parse.py /cron.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | os.environ['DJANGO_SETTINGS_MODULE'] = 'betmanager.settings' 4 | 5 | import get 6 | import parse 7 | import sys 8 | import json 9 | 10 | filename = get.get_matches(int(sys.argv[1])) 11 | print(filename) 12 | with open(filename) as f: 13 | lines = f.readlines() 14 | for line in lines: 15 | result = json.loads(line) 16 | parse.parse_json(result) 17 | -------------------------------------------------------------------------------- /leagues.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import re 3 | import json 4 | from info import leagues 5 | from lxml import html 6 | 7 | s = requests.Session() 8 | 9 | for league in leagues: 10 | page = s.get('http://www.oddsportal.com/soccer' + league[0]) 11 | tree = html.fromstring(page.text) 12 | i = 0 13 | ids = [] 14 | for a in tree.xpath('//ul[@class="main-filter"]//a'): 15 | if 'results' in a.attrib['href']: 16 | result = s.get('http://www.oddsportal.com' + a.attrib['href']) 17 | id = re.findall(r'id":"([^"]+)"', result.text)[0] 18 | t = html.fromstring(result.text) 19 | season = t.xpath('//div[@class="main-menu2 main-menu-gray"]//span[@class="active"]//a')[0].text_content() 20 | beka = [id, season, a.attrib['href'].replace('/results/', '')] 21 | if beka not in ids: 22 | ids.append(beka) 23 | print(a.attrib['href'].replace('/results/', '')) 24 | league.append(ids) 25 | print(json.dumps(leagues)) 26 | -------------------------------------------------------------------------------- /reget.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import re 3 | import json 4 | from lxml import html 5 | import match as match_dl 6 | import errno 7 | import os 8 | import sys 9 | from info import leagues 10 | import parse 11 | 12 | s = requests.Session() 13 | 14 | sid = 1 15 | bookie_hash = 'X0' 16 | use_premium = 1 # xD 17 | timezone_offset = 1 18 | 19 | reget_file = 'to_reget.dat' 20 | 21 | 22 | def mkdir_p(path): 23 | try: 24 | os.makedirs(path) 25 | except OSError as exc: # Python >2.5 26 | if exc.errno == errno.EEXIST and os.path.isdir(path): 27 | pass 28 | else: 29 | raise 30 | 31 | 32 | def get_league_info(link): 33 | for league in leagues: 34 | if league[0].replace('/results', '') in link: 35 | return league 36 | return [] 37 | 38 | 39 | def get_match(match_id): 40 | f = open("data/reget.json", "a+") 41 | r = s.get('http://www.oddsportal.com/a/b/c/d-%s/' % match_id) 42 | tree = html.fromstring(r.text) 43 | try: 44 | match = { 45 | 'match_id': match_id 46 | } 47 | print(match_id) 48 | match = match_dl.get_match(match) 49 | name = tree.xpath( 50 | '//div[@id="col-content"]/h1')[0].text_content().split(' - ') 51 | match['home'] = name[0] 52 | match['away'] = name[1] 53 | match['event'] = get_league_info(r.url)[1:] 54 | event_request = requests.get( 55 | 'http://www.soccer24.com/match/' + match['match_id']) 56 | event_tree = html.fromstring(event_request.text) 57 | phrases = event_tree.xpath( 58 | '//table[@class="detail"]//a/text()')[0].split(' - ')[1:] 59 | match['event'] += phrases[::-1] 60 | f.write(json.dumps(match) + '\n') 61 | except: 62 | fail = open("to_reget.dat", 'a+') 63 | fail.write(match_id + '\n') 64 | fail.close() 65 | f.close() 66 | 67 | 68 | if __name__ == "__main__": 69 | if os.path.exists(reget_file): 70 | os.rename(reget_file, reget_file + '.swp') 71 | with open(reget_file + '.swp') as f: 72 | for line in f.readlines(): 73 | get_match(line.replace('\n', '')) 74 | os.remove(reget_file + '.swp') 75 | with open('data/reget.json') as reget: 76 | for line in reget.readlines(): 77 | json_data = json.loads(line) 78 | parse.parse_json(json_data) 79 | os.remove('data/reget.json') 80 | -------------------------------------------------------------------------------- /match.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import requests 3 | import re 4 | import json 5 | from info import betting_types, bookmakers, leagues 6 | import time 7 | import sys 8 | import os 9 | 10 | 11 | def unhash(xhash): 12 | decoded = '' 13 | for i in xhash.split('%')[1:]: 14 | decoded += chr(int(i, 16)) 15 | return decoded 16 | 17 | 18 | version_id = 1 19 | sport_id = 1 20 | scope_ids = { 21 | 2, 3, 4, # full, 1st half, 2nd half 22 | } 23 | 24 | s = requests.Session() 25 | 26 | 27 | def get_match(match): 28 | page = s.get('http://www.oddsportal.com/a/b/c/d-%s/' % match['match_id']) 29 | # match_id = re.findall(r'id":"([^"]+)"', page.text)[0] 30 | xhash = unhash(re.findall(r'xhashf":"([^"]+)"', page.text)[0]) 31 | # xhashf = unhash(re.findall(r'xhashf":"([^"]+)"', page.text)[0]) 32 | score_result = s.get('http://fb.oddsportal.com/feed/postmatchscore/' + 33 | '%d-%s-%s.dat' % ( 34 | sport_id, 35 | match['match_id'], 36 | xhash), 37 | ).text 38 | # print(score_result) 39 | score_data = re.findall(r'[0-9]+:[0-9]+', score_result) 40 | 41 | match['odds'] = { 42 | 2: {}, 43 | 3: {}, 44 | 4: {}, 45 | } 46 | if score_data: 47 | match['score'] = score_data[0] 48 | match['status'] = 'finished' 49 | if len(score_data[:1]) > 0: 50 | match['partial'] = score_data[1:] 51 | else: 52 | try: 53 | match['status'] = re.findall(r'>([^>]+)<', score_result)[0] 54 | except: 55 | # jeżeli result = "", to mecz się jeszcze nie odbył 56 | pass 57 | 58 | for betting_type in betting_types: 59 | for scope_id in scope_ids: 60 | # try: 61 | r = s.get('http://fb.oddsportal.com/feed/match/' + 62 | '%d-%d-%s-%d-%d-%s.dat' % ( 63 | version_id, 64 | sport_id, 65 | match['match_id'], 66 | betting_type, 67 | scope_id, 68 | xhash, 69 | )) 70 | # regex = re.findall(r' ([^)]+)', r.text)[0] 71 | regex = re.findall(r' ([^$]+)\)', r.text)[0] 72 | data = json.loads(regex) 73 | bets = data['d']['oddsdata']['back'] 74 | if bets: 75 | for bet in bets: 76 | # for item in bet['odds']: 77 | for bookmaker in bookmakers: 78 | if bookmaker in bets[bet]['odds']: 79 | bets[bet]['odds'][bookmakers[bookmaker]] = bets[bet]['odds'].pop(bookmaker) 80 | # print(json.dumps(bets[bet])) 81 | if betting_types[betting_type] not in match['odds'][scope_id]: 82 | match['odds'][scope_id][betting_types[betting_type]] = {} 83 | if 'mixedParameterName' in bets[bet]: 84 | match['odds'][scope_id][betting_types[betting_type]][bets[bet]['mixedParameterName']] = \ 85 | bets[bet]['odds'] 86 | else: 87 | match['odds'][scope_id][betting_types[betting_type]][bets[bet]['handicapValue']] = \ 88 | bets[bet]['odds'] 89 | # except: 90 | # with open('to_reget.dat', 'a+') as f: 91 | # f.write(match['match_id'] + '\n') 92 | 93 | 94 | # na końcu przemianowujemy, bo fajnie jest wtedy 95 | match['odds']['full_time'] = match['odds'].pop(2) 96 | match['odds']['first_half'] = match['odds'].pop(3) 97 | match['odds']['second_half'] = match['odds'].pop(4) 98 | return match 99 | 100 | 101 | if __name__ == "__main__": 102 | with open(sys.argv[1]) as f: 103 | # match = json.load(data_file) 104 | matches = f.readlines() 105 | swp = open(sys.argv[1] + ".new", 'a+') 106 | for match in matches: 107 | match = get_match(json.loads(match)) 108 | print(json.dumps(match)) 109 | swp.write(json.dumps(match)) 110 | swp.close() 111 | -------------------------------------------------------------------------------- /get.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import re 3 | import json 4 | from lxml import html 5 | import match as match_dl 6 | import errno 7 | import os 8 | import sys 9 | import datetime 10 | import parse 11 | 12 | s = requests.Session() 13 | 14 | timezone_offset = 1 15 | sport = 'soccer' 16 | sport_id = 1 17 | table_type = 2 # 1 - kick off time, 2 - events 18 | 19 | reget_file = 'to_reget.dat' 20 | 21 | 22 | def utilize_link(link): 23 | link = link.split('/') 24 | del link[-2] 25 | return '/'.join(link) 26 | 27 | 28 | def mkdir_p(path): 29 | try: 30 | os.makedirs(path) 31 | except OSError as exc: # Python >2.5 32 | if exc.errno == errno.EEXIST and os.path.isdir(path): 33 | pass 34 | else: 35 | raise 36 | 37 | 38 | def unhash(xhash): 39 | decoded = '' 40 | for i in xhash.split('%')[1:]: 41 | decoded += chr(int(i, 16)) 42 | return decoded 43 | 44 | 45 | def build_match(tr): 46 | match = { 47 | 'match_id': tr.attrib['xeid'] 48 | } 49 | match = match_dl.get_match(match) 50 | date = tr.xpath('td[1]')[0] 51 | match['date'] = re.findall(r' t([^-]+)', date.attrib['class'])[0] 52 | name = tr.xpath('.//a[not(@id)]')[0].text_content().split(' - ') 53 | match['home'] = name[0] 54 | match['away'] = name[1] 55 | # match['event'] = leagues[i][1:-1] 56 | event_request = requests.get('http://www.soccer24.com/match/' + match['match_id']) 57 | event_tree = html.fromstring(event_request.text) 58 | phrases = event_tree.xpath('//table[@class="detail"]//a/text()')[0].split(' - ')[1:] 59 | match['event'] = phrases[::-1] 60 | return match 61 | 62 | 63 | def get_xhash(date): 64 | page = s.get('http://www.oddsportal.com/matches/%s/%s' % 65 | (sport, date)) 66 | return unhash(re.findall(r'%s":"([^"]+)"' % date, page.text)[1]) 67 | 68 | 69 | def get_league_info(link): 70 | with open('leagues.json') as f: 71 | leagues = json.load(f) 72 | for league in leagues: 73 | if utilize_link(link).endswith(league[0].replace('results', '')): 74 | return league 75 | return [] 76 | 77 | 78 | def get_season(league, link): 79 | for item in league[4]: 80 | if item[2] in utilize_link(link): 81 | return item[1] 82 | 83 | 84 | def get_matches(offset=0): 85 | date = datetime.datetime.today() + datetime.timedelta(days=offset) 86 | date = date.strftime("%Y%m%d") 87 | f = open('data/%s.json' % date, 'w+') 88 | xhash = get_xhash(date) 89 | r = s.get('http://fb.oddsportal.com/ajax-next-games/%d/%d/%d/%s/%s.dat' % ( 90 | sport_id, 91 | timezone_offset, 92 | table_type, 93 | date, 94 | xhash 95 | )) 96 | data = json.loads(re.findall(r' ([^$]+)\)', r.text)[0]) 97 | tree = html.fromstring(data['d']) 98 | for tr in tree.xpath('//tr[@xeid]'): 99 | link = tr.xpath('.//a[not(@id)]')[0] 100 | league = get_league_info(link.attrib['href']) 101 | if league: 102 | try: 103 | match = build_match(tr) 104 | match['event'] = league[1:-1] + \ 105 | [get_season(league, link.attrib['href'])] + \ 106 | match['event'] 107 | f.write(json.dumps(match) + '\n') 108 | # print(json.dumps(match) + '\n') 109 | except: 110 | fail = open("to_reget.dat", 'a+') 111 | fail.write(tr.attrib['xeid'] + '\n') 112 | fail.close() 113 | f.close() 114 | return 'data/%s.json' % date 115 | 116 | 117 | def get_match(match_id): 118 | f = open("data/reget.json", "a+") 119 | r = s.get('http://www.oddsportal.com/a/b/c/d-%s/' % match_id) 120 | tree = html.fromstring(r.text) 121 | try: 122 | match = { 123 | 'match_id': match_id 124 | } 125 | print(match_id) 126 | match = match_dl.get_match(match) 127 | name = tree.xpath( 128 | '//div[@id="col-content"]/h1')[0].text_content().split(' - ') 129 | match['home'] = name[0] 130 | match['away'] = name[1] 131 | 132 | league = get_league_info(r.url) 133 | 134 | # match['event'] = get_league_info(r.url)[1:] 135 | event_request = requests.get( 136 | 'http://www.soccer24.com/match/' + match['match_id']) 137 | event_tree = html.fromstring(event_request.text) 138 | phrases = event_tree.xpath( 139 | '//table[@class="detail"]//a/text()')[0].split(' - ')[1:] 140 | # match['event'] += phrases[::-1] 141 | 142 | match['event'] = league[1:-1] + \ 143 | [get_season(league, r.url)] + \ 144 | phrases[::-1] 145 | 146 | f.write(json.dumps(match) + '\n') 147 | # print(json.dumps(match) + '\n') 148 | except: 149 | fail = open("to_reget.dat", 'a+') 150 | fail.write(match_id + '\n') 151 | fail.close() 152 | f.close() 153 | 154 | 155 | if __name__ == "__main__": 156 | mkdir_p('data') 157 | if sys.argv[1] == 'retry': 158 | if os.path.exists(reget_file): 159 | os.rename(reget_file, reget_file + '.swp') 160 | with open(reget_file + '.swp') as f: 161 | for line in f.readlines(): 162 | get_match(line.replace('\n', '')) 163 | os.remove(reget_file + '.swp') 164 | with open('data/reget.json') as reget: 165 | for line in reget.readlines(): 166 | json_data = json.loads(line) 167 | parse.parse_json(json_data) 168 | os.remove('data/reget.json') 169 | else: 170 | get_matches(int(sys.argv[1])) 171 | 172 | -------------------------------------------------------------------------------- /info.py: -------------------------------------------------------------------------------- 1 | betting_types = { 2 | 1: '1x2', 3 | 2: 'over under', 4 | 3: 'home/away', 5 | 4: 'dc', 6 | 5: 'asian', 7 | 6: 'draw no bet', 8 | 7: 'to qualify', 9 | 8: 'correct score', 10 | 9: 'ht/ft', 11 | 10: 'odd/even', 12 | 12: 'euro handicap', 13 | 13: 'both teams to score', 14 | } 15 | 16 | bookmakers = { 17 | '1': 'Interwetten', 18 | '2': 'bwin', 19 | '3': 'bet-at-home', 20 | '5': 'Unibet', 21 | '8': 'Stan James', 22 | '9': 'Expekt', 23 | '14': '10Bet', 24 | '15': 'William Hill', 25 | '16': 'bet365', 26 | '18': 'Pinnacle Sports', 27 | '20': '5Dimes', 28 | '21': 'Betfred', 29 | '23': 'DOXXbet', 30 | '24': 'Betsafe', 31 | '26': 'Betway', 32 | '27': '888sport', 33 | '28': 'Ladbrokes', 34 | '30': 'Boylesports', 35 | '31': 'Intertops', 36 | '32': 'Betclic', 37 | '33': 'NordicBet', 38 | '38': 'Sportingbet', 39 | '41': 'myBet', 40 | '43': 'Betsson', 41 | '45': 'Chance.cz', 42 | '46': 'iFortuna.cz', 43 | '49': 'Tipsport.cz', 44 | '53': 'bwin.it', 45 | '55': 'Noxwin', 46 | '56': '188BET', 47 | '57': 'Jetbull', 48 | '70': 'Tipico', 49 | '71': 'Coral', 50 | '73': 'LEON Bets', 51 | '75': 'SBOBET', 52 | '76': 'BetVictor', 53 | '78': 'BetOnline', 54 | '80': '12BET', 55 | '90': 'Betgun', 56 | '92': 'Island Casino', 57 | '98': 'Superbahis', 58 | '101': 'Paf', 59 | '103': 'Redbet', 60 | '121': 'Titanbet', 61 | '124': 'Luxbet', 62 | '125': 'Bestbet', 63 | '128': 'youwin', 64 | '129': 'bwin.fr', 65 | '139': 'France Pari', 66 | '140': 'Betclic.it', 67 | '147': 'Dafabet', 68 | '149': 'Interwetten.es', 69 | '157': 'Unibet.it', 70 | '160': 'Unibet.fr', 71 | '161': 'NetBet', 72 | '164': 'iFortuna.sk', 73 | '372': 'WilliamHill.it', 74 | '377': 'Bovada', 75 | '378': 'TonyBet', 76 | '379': 'PaddyPower.it', 77 | '381': 'MarathonBet', 78 | '384': 'bet-at-home.it', 79 | '390': 'Matchbook', 80 | '392': 'bwin.es', 81 | '393': 'Oddsring', 82 | '394': 'Tipico.it', 83 | '395': 'Tempobet', 84 | '397': 'Winlinebet', 85 | '399': 'Winner', 86 | '403': 'Betrally', 87 | '406': 'Sportium', 88 | '411': 'Tipsport.sk', 89 | '416': '18bet', 90 | '417': '1xbet', 91 | '418': 'Vernons', 92 | '419': 'bet365.it', 93 | '422': '5plusbet', 94 | '424': '138', 95 | '425': 'Realdealbet', 96 | '426': 'Otobet', 97 | '427': 'VitalBet', 98 | } 99 | 100 | leagues = [ 101 | ["/albania/super-league/results", "Albania", "Super league", 1], 102 | ["/algeria/division-1/results", "Algeria", "Division 1", 1], 103 | ["/andorra/primera-divisio/results", "Andorra", "Primera Division", 1], 104 | ["/argentina/primera-division/results", "Argentina", "Primera Division", 1], 105 | ["/argentina/primera-b-nacional/results", "Argentina", "Primera B Nacional", 2], 106 | ["/argentina/primera-b-metropolitana/results", "Argentina", "Primera B Metropolitana", 3], 107 | ["/armenia/premier-league/results", "Armenia", "Premier League", 1], 108 | ["/australia/a-league/results", "Australia", "A league", 1], 109 | ["/austria/tipico-bundesliga/results", "Austria", "Tipico Bundesliga", 1], 110 | ["/austria/erste-liga/results", "Austria", "Erste Liga", 2], 111 | ["/azerbaijan/premier-league/results", "Azerbaijan", "Premier League", 1], 112 | ["/belarus/vysshaya-liga/results", "Belarus", "Vysshaya Liga", 1], 113 | ["/belgium/jupiler-league/results", "Belgium", "Jupiler League", 1], 114 | ["/belgium/proximus-league/results", "Belgium", "Proximus League", 2], 115 | ["/bosnia-and-herzegovina/premier-league/results", "Bosnia and Herzegovina", "Premier League", 1], 116 | ["/brazil/serie-a/results", "Brazil", "Serie A", 1], 117 | ["/brazil/serie-b/results", "Brazil", "Serie B", 2], 118 | ["/brazil/serie-c/results", "Brazil", "Serie C", 3], 119 | ["/brazil/serie-d/results", "Brazil", "Serie D", 4], 120 | ["/bulgaria/a-pfg/results", "Bulgaria", "A PFG", 1], 121 | ["/bulgaria/b-pfg/results", "Bulgaria", "B PFG", 2], 122 | ["/cameroon/elite-one/results", "Cameroon", "Elite One", 1], 123 | ["/canada/csl/results", "Canada", "CSL", 1], 124 | ["/chile/primera-division/results", "Chile", "Primera Division", 1], 125 | ["/chile/primera-b/results", "Chile", "Primera Division", 2], 126 | ["/china/super-league/results", "China", "Super League", 1], 127 | ["/china/jia-league/results", "China", "Jia League", 2], 128 | ["/colombia/liga-aguila/results", "Colombia", "Liga Aguila", 1], 129 | ["/colombia/torneo-aguila/results", "Colombia", "Torneo Aguila", 2], 130 | ["/costa-rica/primera-division/results", "Costa Rica", "Primera Division", 1], 131 | ["/croatia/1-hnl/results", "Croatia", "1 HNL", 1], 132 | ["/croatia/2-hnl/results", "Croatia", "2 NHL", 2], 133 | ["/czech-republic/synot-liga/results", "Czech Republic", "Synot Liga", 1], 134 | ["/czech-republic/division-2/results", "Czech Republic", "Division 2", 2], 135 | ["/denmark/superliga/results", "Denmark", "Superliga", 1], 136 | ["/denmark/bet25-liga/results", "Denmark", "Bet25 Liga", 2], 137 | ["/ecuador/serie-a/results", "Ecuador", "Serie A", 1], 138 | ["/egypt/premier-league/results", "Egypt", "Premier League", 1], 139 | ["/england/premier-league/results", "England", "Premier League", 1], 140 | ["/england/championship/results", "England", "Championship", 2], 141 | ["/england/league-one/results", "England", "League One", 3], 142 | ["/england/league-two/results", "England", "League Two", 4], 143 | ["/england/vanarama-national-league/results", "England", "Vanarama National League", 5], 144 | ["/estonia/meistriliiga/results", "Estonia", "Mestriliiga", 1], 145 | ["/estonia/esi-liiga/results", "Estonia", "Esi Liga", 2], 146 | ["/finland/veikkausliiga/results", "Finland", "Veikkausliiga", 1], 147 | ["/finland/ykkonen/results", "Finland", "Ykkonen", 2], 148 | ["/france/ligue-1/results", "France", "Ligue 1", 1], 149 | ["/france/ligue-2/results", "France", "Ligue 2", 2], 150 | ["/france/national/results", "France", "National", 3], 151 | ["/fyr-of-macedonia/first-league/results", "FYR of Macedonia", "First League", 1], 152 | ["/fyr-of-macedonia/division-1/results", "FYR of Macedonia", "Division 1", 2], 153 | ["/georgia/umaglesi-liga/results", "Georgia", "Umaglesi Liga", 1], 154 | ["/germany/bundesliga/results", "Germany", "Bundesliga", 1], 155 | ["/germany/2-bundesliga/results", "Germany", "2 Bundesliga", 2], 156 | ["/germany/3-liga/results", "Germany", "3 Bundesliga", 3], 157 | ["/ghana/premier-league/results", "Ghana", "Premier League", 1], 158 | ["/greece/super-league/results", "Greece", "Super League", 1], 159 | ["/honduras/liga-nacional/results", "Honduras", "Liga Nacional", 1], 160 | ["/hungary/otp-bank-liga/results", "Hungary", "OTP Bank Liga", 1], 161 | ["/hungary/merkantil-bank-liga/results", "Hungary", "Merkantil Bank Liga", 2], 162 | ["/iceland/pepsideild/results", "Iceland", "Pepsideild", 1], 163 | ["/iceland/division-1/results", "Iceland", "Division 1", 2], 164 | ["/iceland/division-2/results", "Iceland", "Division 2", 3], 165 | ["/iran/persian-gulf-pro-league/results", "Iran", "Persian Gulf Pro League", 1], 166 | ["/iran/division-1/results", "Iran", "Division 1", 2], 167 | ["/ireland/premier-league/results", "Ireland", "Premier League", 1], 168 | ["/ireland/division-1/results", "Ireland", "Division 1", 2], 169 | ["/italy/serie-a/results", "Italy", "Serie A", 1], 170 | ["/italy/serie-b/results", "Italy", "Serie B", 2], 171 | ["/italy/lega-pro-group-a/results", "Italy", "Lega Pro Group A", 3], 172 | ["/italy/lega-pro-group-b/results", "Italy", "Lega Pro Group B", 3], 173 | ["/italy/lega-pro-group-c/results", "Italy", "Lega Pro Group C", 3], 174 | ["/israel/ligat-ha-al/results", "Israel", "Ligat Ha al", 1], 175 | ["/israel/leumit-league/results", "Israel", "Leumit League", 2], 176 | ["/japan/j-league/results", "Japan", "J-League", 1], 177 | ["/japan/j-league-division-2/results", "Japan", "J-League Division 2", 2], 178 | ["/latvia/virsliga/results", "Latvia", "Virsliga", 1], 179 | ["/lithuania/a-lyga/results", "Lithuania", "A Lyga", 1], 180 | ["/mexico/primera-division/results", "Mexico", "Primera Division", 1], 181 | ["/mexico/liga-de-ascenso/results", "Mexico", "Liga de Ascenso", 2], 182 | ["/morocco/botola-pro/results", "Morroco", "Botola Pro", 1], 183 | ["/morocco/elite-2/results", "Morroco", "Elite 2", 2], 184 | ["/netherlands/eredivisie/results", "Netherlands", "Eredivisie", 1], 185 | ["/netherlands/eerste-divisie/results", "Netherlands", "Eerste Divisie", 2], 186 | ["/northern-ireland/nifl-premiership/results", "Northern Ireland", "NIFL Premiership", 1], 187 | ["/northern-ireland/ifa-championship-1/results", "Northern Ireland", "IFA Championship 1", 2], 188 | ["/norway/tippeligaen/results", "Norway", "Tippeligaen", 1], 189 | ["/norway/obos-ligaen/results", "Norway", "OBOS-ligaen", 2], 190 | ["/paraguay/primera-division/results", "Paraguay", "Primera Division", 1], 191 | ["/peru/primera-division/results", "Peru", "Primera Division", 1], 192 | ["/poland/ekstraklasa/results", "Poland", "Ekstraklasa", 1], 193 | ["/poland/division-1/results", "Poland", "Division 1", 2], 194 | ["/poland/division-2/results", "Poland", "Division 2", 3], 195 | ["/portugal/primeira-liga/results", "Portugal", "Primeira Liga", 1], 196 | ["/portugal/segunda-liga/results", "Portugal", "Segunda Liga", 2], 197 | ["/portugal/campeonato-nacional/results", "Portugal", "Compeonato Nacional", 3], 198 | ["/qatar/premier-league/results", "Quatar", "Premier League", 1], 199 | ["/romania/liga-1/results", "Romania", "Liga 1", 1], 200 | ["/russia/premier-league/results", "Russia", "Premier League", 1], 201 | ["/russia/division-1/results", "Russia", "Division 1", 2], 202 | ["/saudi-arabia/saudi-professional-league/results", "Saudi Arabia", "Saudi Professional League", 1], 203 | ["/saudi-arabia/division-1/results", "Saudi Arabia", "Division 1", 2], 204 | ["/scotland/premiership/results", "Scotland", "Premiership", 1], 205 | ["/scotland/championship/results", "Scotland", "Championship", 2], 206 | ["/scotland/league-one/results", "Scotland", "League One", 3], 207 | ["/scotland/league-two/results", "Scotland", "League Two", 4], 208 | ["/slovakia/fortuna-liga/results", "Slovakia", "Fortuna Liga", 1], 209 | ["/slovenia/prva-liga/results", "Slovenia", "Prva Liga", 1], 210 | ["/slovenia/2-snl/results", "Slovenia", "2 SNL", 2], 211 | ["/south-africa/premier-league/results", "South Africa", "Premier League", 1], 212 | ["/south-africa/first-division/results", "South Africa", "First Division", 2], 213 | ["/south-korea/k-league-classic/results", "South Korea", "K League Classic", 1], 214 | ["/south-korea/k-league-challenge/results", "South Korea", "K League Challenge", 2], 215 | ["/spain/primera-division/results", "Spain", "Primera Division", 1], 216 | ["/spain/segunda-division/results", "Spain", "Segunda Division", 2], 217 | ["/spain/segunda-division-b-group-1/results", "Spain", "Segunda Division B1", 3], 218 | ["/spain/segunda-division-b-group-2/results", "Spain", "Segunda Division B2", 3], 219 | ["/spain/segunda-division-b-group-3/results", "Spain", "Segunda Division B3", 3], 220 | ["/spain/segunda-division-b-group-4/results", "Spain", "Segunda Division B4", 3], 221 | ["/sweden/allsvenskan/results", "Sweden", "Allsvenskan", 1], 222 | ["/sweden/superettan/results", "Sweden", "Superettan", 2], 223 | ["/switzerland/super-league/results", "Switzerland", "Super League", 1], 224 | ["/switzerland/challenge-league/results", "Switzerland", "Challenge League", 2], 225 | ["/turkey/super-lig/results", "Turkey", "Super Lig", 1], 226 | ["/turkey/ptt-1-lig/results", "Turkey", "PTT 1 Lig", 2], 227 | ["/turkey/tff-2-lig-white-group/results", "Turkey", "TFF 2 Lig White", 3], 228 | ["/turkey/tff-2-lig-red-group/results", "Turkey", "TFF 2 Lig Red", 3], 229 | ["/ukraine/pari-match-league/results", "Ukraine", "Pari Match League", 1], 230 | ["/ukraine/division-2/results", "Ukraine", "Division 2", 2], 231 | ["/united-arab-emirates/uae-league/results", "United Arab Emirates", "UAE League", 1], 232 | ["/uruguay/primera-division/results", "Uruguay", "Primera Division", 1], 233 | ["/usa/mls/results", "USA", "MLS", 1], 234 | ["/venezuela/primera-division/results", "Venezuela", "Primera Division", 1], 235 | ["/wales/premier-league/results", "Wales", "Premier League", 1], 236 | ["/wales/division-1/results", "Wales", "Premier League", 2], 237 | ] 238 | -------------------------------------------------------------------------------- /parse.py: -------------------------------------------------------------------------------- 1 | import django 2 | from django.db import transaction 3 | import re 4 | import sys 5 | from datetime import datetime 6 | import pytz 7 | from info import bookmakers 8 | # sys.path.append('../') 9 | 10 | from football.models import * 11 | import json 12 | 13 | django.setup() 14 | 15 | states = { 16 | "finished": Match.OK, 17 | "The match has already started.": Match.INPLAY, 18 | "Awarded": Match.WO, 19 | "Canceled": Match.CANCELED, 20 | "postponed": Match.POSTPONED 21 | } 22 | 23 | 24 | def get_status(json_status): 25 | if json_status in states: 26 | return states[json_status] 27 | else: 28 | return Match.WARNING 29 | 30 | 31 | @transaction.atomic 32 | def parse_json(result): 33 | match_id = result['match_id'] 34 | if Match.objects.filter(external_id=match_id): 35 | match = Match.objects.get(external_id=match_id) 36 | else: 37 | match = Match() 38 | # ja pierdole, czemu external_id 39 | # ale i tak nie zmieniam, nie chce mi się migracji robić xD 40 | print("%s - %s (%s)" % (result['home'], result['away'], result['match_id'])) 41 | country = Country.objects.get_or_create(name=result['event'][0])[0] 42 | home = Team.objects.get_or_create(name=result['home'].replace(u'\xa0', u' ').strip(), 43 | country=country)[0] 44 | away = Team.objects.get_or_create(name=result['away'].replace(u'\xa0', u' ').strip(), 45 | country=country)[0] 46 | if 'status' in result: 47 | status = get_status(result['status']) 48 | else: 49 | status = Match.FUTURE 50 | 51 | name = result['event'][1] 52 | try: 53 | season = result['event'][3] 54 | except: 55 | season = '?' 56 | stage = '' 57 | importance = result['event'][2] 58 | league = League.objects.get_or_create(name=name, 59 | country=country, 60 | importance=importance)[0] 61 | 62 | match_round = 0 63 | 64 | try: 65 | if result['event'][3]: 66 | if 'Round' in result['event'][4]: 67 | match_round = int(re.search('\d+', result['event'][3]).group(0)) 68 | else: 69 | stage = result['event'][4] 70 | except: 71 | pass 72 | 73 | match.home = home 74 | match.away = away 75 | if 'date' in result: 76 | date = pytz.timezone('Europe/Warsaw').localize(datetime.fromtimestamp(int(result['date']))) 77 | match.date = date 78 | match.external_id = match_id 79 | match.league = league 80 | match.stage = stage 81 | match.round = match_round 82 | match.season = season 83 | match.status = status 84 | 85 | try: 86 | match.save() 87 | except: 88 | return 89 | 90 | def get_score(score): 91 | try: 92 | return re.findall('\d+', score) 93 | except: 94 | return None 95 | 96 | if 'score' in result: 97 | score = get_score(result['score']) 98 | if score: 99 | match.score = Score.objects.create(home=score[0], 100 | away=score[1]) 101 | 102 | if 'partial' in result and result['partial']: 103 | half_score = get_score(result['partial'][0]) 104 | match.half_score = Score.objects.create(home=half_score[0], 105 | away=half_score[1]) 106 | if len(result['partial']) < 2: 107 | for score in result['partial'][2:]: 108 | partial = get_score(score) 109 | match.additional_scores.all().delete() 110 | if partial: 111 | match.additional_scores.add(Score.objects.create(home=partial[0], 112 | away=partial[1])) 113 | 114 | match.save() 115 | 116 | simple_bets = ['1x2', 'dc', 'draw no bet', 'odd/even', 'both teams to score'] 117 | 118 | betting_types = [ 119 | '1x2', 120 | 'over under', 121 | 'odd/even', 122 | 'draw no bet', 123 | 'dc', 124 | 'both teams to score', 125 | 'asian', 126 | 'correct score', 127 | 'ht/ft', 128 | 'euro handicap' 129 | ] 130 | 131 | for bookmaker in bookmakers.values(): 132 | name = Bookmaker.objects.get_or_create(name=bookmaker)[0] 133 | football_bet = FootballBet.objects.get_or_create(match=match, 134 | bookmaker=name)[0] 135 | # tu czyścimy wszystkie betsy 136 | # METODĄ ŻYDOWSKĄ SKURWYSYNY 137 | for field in football_bet._meta.get_fields(include_hidden=True): 138 | if any(x in field.name for x in ['_M', '_1hf', '_2hf']) \ 139 | and '+' not in field.name: 140 | ret = getattr(football_bet, field.name) 141 | if ret: 142 | try: 143 | ret.all().delete() 144 | except AttributeError: 145 | # ret.delete() 146 | # ret.save() 147 | pass 148 | football_bet.save() 149 | for period in ['full_time', 'first_half', 'second_half']: 150 | for betting_type in betting_types: 151 | # try: 152 | if betting_type in simple_bets: 153 | if betting_type in result['odds'][period] \ 154 | and bookmaker in result['odds'][period][betting_type]['0.00']: 155 | val = result['odds'][period][betting_type]['0.00'][bookmaker] 156 | else: 157 | continue 158 | 159 | def get_1x2(): 160 | try: 161 | if isinstance(val, dict): 162 | first = val['0'] 163 | second = val['1'] 164 | third = val['2'] 165 | else: 166 | first = val[0] 167 | second = val[1] 168 | third = val[2] 169 | except (KeyError, IndexError): 170 | return 171 | outcome_bet = OutcomeBet.objects.create( 172 | first=first, 173 | second=second, 174 | third=third 175 | ) 176 | if period == 'full_time': 177 | outcome_bet.part = OneWayBet.MATCH 178 | football_bet.outcome_M = outcome_bet 179 | elif period == 'first_half': 180 | outcome_bet.part = OneWayBet.FIRST_HALF 181 | football_bet.outcome_1hf = outcome_bet 182 | else: 183 | outcome_bet.part = OneWayBet.SECOND_HALF 184 | football_bet.outcome_2hf = outcome_bet 185 | outcome_bet.save() 186 | 187 | def get_odd_even(): 188 | try: 189 | if isinstance(val, dict): 190 | first = val['0'] 191 | second = val['1'] 192 | else: 193 | first = val[0] 194 | second = val[1] 195 | except (KeyError, IndexError): 196 | return 197 | odd_even_bet = OddEvenBet.objects.create( 198 | first=first, 199 | second=second 200 | ) 201 | if period == 'full_time': 202 | odd_even_bet.part = OneWayBet.MATCH 203 | football_bet.odd_even_M = odd_even_bet 204 | elif period == 'first_half': 205 | odd_even_bet.part = OneWayBet.FIRST_HALF 206 | football_bet.odd_even_1hf = odd_even_bet 207 | else: 208 | odd_even_bet.part = OneWayBet.SECOND_HALF 209 | football_bet.odd_even_2hf = odd_even_bet 210 | odd_even_bet.save() 211 | 212 | def get_dc(): 213 | try: 214 | if isinstance(val, dict): 215 | first = val['0'] 216 | second = val['1'] 217 | third = val['2'] 218 | else: 219 | first = val[0] 220 | second = val[1] 221 | third = val[2] 222 | except (KeyError, IndexError): 223 | return 224 | dc_bet = DoubleChanceBet.objects.create( 225 | first=first, 226 | second=second, 227 | third=third 228 | ) 229 | if period == 'full_time': 230 | dc_bet.part = OneWayBet.MATCH 231 | football_bet.double_chance_M = dc_bet 232 | elif period == 'first_half': 233 | dc_bet.part = OneWayBet.FIRST_HALF 234 | football_bet.double_chance_1hf = dc_bet 235 | else: 236 | dc_bet.part = OneWayBet.SECOND_HALF 237 | football_bet.double_chance_2hf = dc_bet 238 | dc_bet.save() 239 | 240 | def get_dnb(): 241 | try: 242 | if isinstance(val, dict): 243 | first = val['0'] 244 | second = val['1'] 245 | else: 246 | first = val[0] 247 | second = val[1] 248 | except (KeyError, IndexError): 249 | return 250 | dnb_bet = DrawNoBetBet.objects.create( 251 | first=first, 252 | second=second, 253 | ) 254 | if period == 'full_time': 255 | dnb_bet.part = OneWayBet.MATCH 256 | football_bet.draw_no_bet_M = dnb_bet 257 | elif period == 'first_half': 258 | dnb_bet.part = OneWayBet.FIRST_HALF 259 | football_bet.draw_no_bet_1hf = dnb_bet 260 | else: 261 | dnb_bet.part = OneWayBet.SECOND_HALF 262 | football_bet.draw_no_bet_2hf = dnb_bet 263 | dnb_bet.save() 264 | 265 | def get_bts(): 266 | try: 267 | if isinstance(val, dict): 268 | first = val['0'] 269 | second = val['1'] 270 | else: 271 | first = val[0] 272 | second = val[1] 273 | except (KeyError, IndexError): 274 | return 275 | bts_bet = BothTeamToScoreBet.objects.create( 276 | first=first, 277 | second=second 278 | ) 279 | if period == 'full_time': 280 | bts_bet.part = OneWayBet.MATCH 281 | football_bet.both_teams_to_score_M = bts_bet 282 | elif period == 'first_half': 283 | bts_bet.part = OneWayBet.FIRST_HALF 284 | football_bet.both_teams_to_score_1hf = bts_bet 285 | else: 286 | bts_bet.part = OneWayBet.SECOND_HALF 287 | football_bet.both_teams_to_score_2hf = bts_bet 288 | bts_bet.save() 289 | 290 | options = { 291 | '1x2': get_1x2, 292 | 'odd/even': get_odd_even, 293 | 'dc': get_dc, 294 | 'draw no bet': get_dnb, 295 | 'both teams to score': get_bts, 296 | } 297 | options[betting_type]() 298 | else: 299 | # correct_score, htft, euro, asian, over_under 300 | if betting_type in result['odds'][period]: 301 | odds_types = result['odds'][period][betting_type] 302 | else: 303 | continue 304 | for odd_type in odds_types: 305 | if bookmaker in odds_types[odd_type]: 306 | val = odds_types[odd_type][bookmaker] 307 | else: 308 | continue 309 | 310 | def get_correct_score(): 311 | goals = odd_type.split(':') 312 | correct_score_bet = CorrectScoreBet.objects.create( 313 | home=int(goals[0]), 314 | away=int(goals[1]), 315 | first=val[0] 316 | ) 317 | if period == 'full_time': 318 | correct_score_bet.part = OneWayBet.MATCH 319 | football_bet.correct_score_M.add(correct_score_bet) 320 | elif period == 'first_half': 321 | correct_score_bet.part = OneWayBet.FIRST_HALF 322 | football_bet.correct_score_1hf.add(correct_score_bet) 323 | else: 324 | correct_score_bet.part = OneWayBet.SECOND_HALF 325 | football_bet.correct_score_2hf.add(correct_score_bet) 326 | correct_score_bet.save() 327 | 328 | def get_htft(): 329 | htft_bet = HalfTimeFullTimeBet.objects.create( 330 | first=val[0], 331 | value=odd_type 332 | ) 333 | htft_bet.part = OneWayBet.MATCH 334 | football_bet.half_time_full_time_M.add(htft_bet) 335 | 336 | def get_euro(): 337 | try: 338 | if isinstance(val, dict): 339 | first = val['0'] 340 | second = val['1'] 341 | third = val['2'] 342 | else: 343 | first = val[0] 344 | second = val[1] 345 | third = val[2] 346 | except (KeyError, IndexError): 347 | return 348 | euro_bet = EuroHandicapBet.objects.create( 349 | first=first, 350 | second=second, 351 | third=third, 352 | value=odd_type 353 | ) 354 | if period == 'full_time': 355 | euro_bet.part = OneWayBet.MATCH 356 | football_bet.euro_handicap_M.add(euro_bet) 357 | elif period == 'first_half': 358 | euro_bet.part = OneWayBet.FIRST_HALF 359 | football_bet.euro_handicap_1hf.add(euro_bet) 360 | else: 361 | euro_bet.part = OneWayBet.SECOND_HALF 362 | football_bet.euro_handicap_2hf.add(euro_bet) 363 | euro_bet.save() 364 | 365 | def get_asian(): 366 | try: 367 | if isinstance(val, dict): 368 | first = val['0'] 369 | second = val['1'] 370 | else: 371 | first = val[0] 372 | second = val[1] 373 | except (KeyError, IndexError): 374 | return 375 | asian_bet = AsianHandicapBet.objects.create( 376 | first=first, 377 | second=second, 378 | value=odd_type 379 | ) 380 | if period == 'full_time': 381 | asian_bet.part = OneWayBet.MATCH 382 | football_bet.asian_handicap_M.add(asian_bet) 383 | elif period == 'first_half': 384 | asian_bet.part = OneWayBet.FIRST_HALF 385 | football_bet.asian_handicap_1hf.add(asian_bet) 386 | else: 387 | asian_bet.part = OneWayBet.SECOND_HALF 388 | football_bet.asian_handicap_2hf.add(asian_bet) 389 | asian_bet.save() 390 | 391 | def get_over_under(): 392 | try: 393 | if isinstance(val, dict): 394 | first = val['0'] 395 | second = val['1'] 396 | else: 397 | first = val[0] 398 | second = val[1] 399 | except (KeyError, IndexError): 400 | return 401 | over_under_bet = OverUnderBet.objects.create( 402 | first=first, 403 | second=second, 404 | value=odd_type 405 | ) 406 | if period == 'full_time': 407 | over_under_bet.part = OneWayBet.MATCH 408 | football_bet.over_under_M.add(over_under_bet) 409 | elif period == 'first_half': 410 | over_under_bet.part = OneWayBet.FIRST_HALF 411 | football_bet.over_under_1hf.add(over_under_bet) 412 | else: 413 | over_under_bet.part = OneWayBet.SECOND_HALF 414 | football_bet.over_under_2hf.add(over_under_bet) 415 | over_under_bet.save() 416 | 417 | options = { 418 | 'over under': get_over_under, 419 | 'asian': get_asian, 420 | 'correct score': get_correct_score, 421 | 'ht/ft': get_htft, 422 | 'euro handicap': get_euro 423 | } 424 | options[betting_type]() 425 | football_bet.save() 426 | 427 | 428 | if __name__ == "__main__": 429 | filename = str(sys.argv[1]) 430 | with open(filename) as f: 431 | lines = f.readlines() 432 | for line in lines: 433 | result = json.loads(line) 434 | parse_json(result) 435 | --------------------------------------------------------------------------------