├── api_server ├── README.md ├── api_server.py ├── ebay-search ├── etsy-search └── requirements.txt └── qtictactoe ├── README.md ├── agent.py ├── main.py ├── q.py └── tictactoe.py /api_server/README.md: -------------------------------------------------------------------------------- 1 | # [AutoScraper and Flask: Create an API From Any Website in Less Than 5 Minutes](https://medium.com/better-programming/autoscraper-and-flask-create-an-api-from-any-website-in-less-than-5-minutes-3f0f176fc4a3) 2 | -------------------------------------------------------------------------------- /api_server/api_server.py: -------------------------------------------------------------------------------- 1 | from autoscraper import AutoScraper 2 | from flask import Flask, request 3 | 4 | 5 | ebay_scraper = AutoScraper() 6 | etsy_scraper = AutoScraper() 7 | ebay_scraper.load('ebay-search') 8 | etsy_scraper.load('etsy-search') 9 | app = Flask(__name__) 10 | 11 | 12 | def get_ebay_result(search_query): 13 | url = 'https://www.ebay.com/sch/i.html?_nkw=%s' % search_query 14 | result = ebay_scraper.get_result_similar(url, group_by_alias=True) 15 | return _aggregate_result(result) 16 | 17 | 18 | def get_etsy_result(search_query): 19 | url = 'https://www.etsy.com/search?q=%s' % search_query 20 | result = etsy_scraper.get_result_similar(url, group_by_alias=True) 21 | result['url'] = [f'https://www.etsy.com/listing/{i}' for i in result['url']] 22 | return _aggregate_result(result) 23 | 24 | 25 | def _aggregate_result(result): 26 | final_result = [] 27 | for i in range(len(list(result.values())[0])): 28 | final_result.append({alias: result[alias][i] for alias in result}) 29 | return final_result 30 | 31 | 32 | @app.route('/', methods=['GET']) 33 | def search_api(): 34 | query = request.args.get('q') 35 | return dict(result=get_ebay_result(query) + get_etsy_result(query)) 36 | 37 | 38 | if __name__ == '__main__': 39 | app.run(port=8080, host='0.0.0.0') 40 | 41 | -------------------------------------------------------------------------------- /api_server/ebay-search: -------------------------------------------------------------------------------- 1 | {"stack_list": [{"content": [["html", {"class": ["srp-ds6"], "style": ""}, 0], ["body", {"style": "", "class": ""}, 0], ["div", {"class": ["srp-main", "srp-main--isLarge"], "style": ""}, 0], ["div", {"class": ["srp-main-content", "clearfix", "srp-main-content--flex"], "style": ""}, 0], ["div", {"class": ["srp-river", "srp-layout-inner"], "style": ""}, 0], ["div", {"class": ["srp-river-main", "clearfix"], "style": ""}, 0], ["div", {"class": ["srp-river-results", "clearfix"], "style": ""}, 0], ["ul", {"class": ["srp-results", "srp-list", "clearfix"], "style": ""}, 4], ["li", {"class": ["s-item"], "style": ""}, 0], ["div", {"class": ["s-item__wrapper", "clearfix"], "style": ""}, 0], ["div", {"class": ["s-item__info", "clearfix"], "style": ""}, 0], ["a", {"class": ["s-item__link"], "style": ""}]], "wanted_attr": null, "is_full_url": null, "url": "", "hash": "cc795f1cb4b1ba86de42110f89a781ef8e284e8d6f3fe399fcb4d56c862f9a00", "stack_id": "rule_0aok", "alias": "title"}, {"content": [["html", {"class": ["srp-ds6"], "style": ""}, 0], ["body", {"style": "", "class": ""}, 0], ["div", {"class": ["srp-main", "srp-main--isLarge"], "style": ""}, 0], ["div", {"class": ["srp-main-content", "clearfix", "srp-main-content--flex"], "style": ""}, 0], ["div", {"class": ["srp-river", "srp-layout-inner"], "style": ""}, 0], ["div", {"class": ["srp-river-main", "clearfix"], "style": ""}, 0], ["div", {"class": ["srp-river-results", "clearfix"], "style": ""}, 0], ["ul", {"class": ["srp-results", "srp-list", "clearfix"], "style": ""}, 30], ["li", {"class": ["s-item"], "style": ""}, 0], ["div", {"class": ["s-item__wrapper", "clearfix"], "style": ""}, 0], ["div", {"class": ["s-item__info", "clearfix"], "style": ""}, 0], ["div", {"class": ["s-item__details", "clearfix"], "style": ""}, 0], ["div", {"class": ["s-item__detail", "s-item__detail--primary"], "style": ""}]], "wanted_attr": null, "is_full_url": null, "url": "", "hash": "89612bf1108198b629efffd7d28d90b1e5e8ac52a5945c2d9b211093c5e0dadf", "stack_id": "rule_vn5z", "alias": "price"}, {"content": [["html", {"class": ["srp-ds6"], "style": ""}, 0], ["body", {"style": "", "class": ""}, 0], ["div", {"class": ["srp-main", "srp-main--isLarge"], "style": ""}, 0], ["div", {"class": ["srp-main-content", "clearfix", "srp-main-content--flex"], "style": ""}, 0], ["div", {"class": ["srp-river", "srp-layout-inner"], "style": ""}, 0], ["div", {"class": ["srp-river-main", "clearfix"], "style": ""}, 0], ["div", {"class": ["srp-river-results", "clearfix"], "style": ""}, 0], ["ul", {"class": ["srp-results", "srp-list", "clearfix"], "style": ""}, 4], ["li", {"class": ["s-item"], "style": ""}, 0], ["div", {"class": ["s-item__wrapper", "clearfix"], "style": ""}, 0], ["div", {"class": ["s-item__info", "clearfix"], "style": ""}, 0], ["a", {"class": ["s-item__link"], "style": ""}]], "wanted_attr": "href", "is_full_url": null, "url": "", "hash": "0ba599387e8d5c06dbb096209ccf8c8cb01e887ff7ecd0614f7f02de391245cd", "stack_id": "rule_buz1", "alias": "url"}]} -------------------------------------------------------------------------------- /api_server/etsy-search: -------------------------------------------------------------------------------- 1 | {"stack_list": [{"content": [["html", {"class": ["ui-toolkit"], "style": ""}, 0], ["body", {"class": ["transitional-wide", "guest", "no-touch", "is-responsive", "en-US", "USD", "US", "is-global-nav"], "style": ""}, 0], ["div", {"class": ["clear"], "style": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["div", {"class": ["content", "bg-white", "col-md-12", "pl-xs-1", "pr-xs-0", "pr-md-1", "pl-lg-0", "pr-lg-0", "bb-xs-1"], "style": ""}, 0], ["div", {"class": ["body-max-width"], "style": ""}, 0], ["div", {"class": ["mt-xs-3", "text-gray", "text-control"], "style": ""}, 1], ["div", {"class": ["col-group", "pl-xs-0", "search-listings-group", "pr-xs-1"], "style": ""}, 1], ["div", {"class": ["col-xs-12", "pl-xs-1", "pl-md-3"], "style": ""}, 0], ["div", {"class": ["bg-white", "display-block", "pb-xs-2", "mt-xs-0"], "style": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["div", {"class": ["wt-grid", "wt-grid--block", "wt-pl-xs-0"], "style": ""}, 0], ["li", {"class": ["wt-list-unstyled", "wt-grid__item-xs-6", "wt-grid__item-md-4", "wt-grid__item-lg-3", "wt-order-xs-0", "wt-order-sm-0", "wt-order-md-0", "wt-order-lg-0", "wt-order-xl-0", "wt-order-tv-0", "grid__item-xl-fifth", "tab-reorder"], "style": ""}, 0], ["div", {"class": ["js-merch-stash-check-listing", "v2-listing-card", "position-relative", "flex-xs-none"], "style": ""}, 0], ["a", {"class": ["b98e134f693f5746", "display-inline-block", "listing-link"], "style": ""}, 0], ["div", {"class": ["v2-listing-card__info"], "style": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["h3", {"class": ["text-gray", "text-truncate", "mb-xs-0", "text-body"], "style": ""}]], "wanted_attr": null, "is_full_url": null, "is_non_rec_text": null, "url": "", "hash": "288dc45d10d069f02c53c76370353f56125a0cdbfcfe494fae68070cfb4756e4", "stack_id": "rule_cc81", "alias": "title"}, {"content": [["html", {"class": ["ui-toolkit"], "style": ""}, 0], ["body", {"class": ["transitional-wide", "guest", "no-touch", "is-responsive", "en-US", "USD", "US", "is-global-nav"], "style": ""}, 0], ["div", {"class": ["clear"], "style": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["div", {"class": ["content", "bg-white", "col-md-12", "pl-xs-1", "pr-xs-0", "pr-md-1", "pl-lg-0", "pr-lg-0", "bb-xs-1"], "style": ""}, 0], ["div", {"class": ["body-max-width"], "style": ""}, 0], ["div", {"class": ["mt-xs-3", "text-gray", "text-control"], "style": ""}, 1], ["div", {"class": ["col-group", "pl-xs-0", "search-listings-group", "pr-xs-1"], "style": ""}, 1], ["div", {"class": ["col-xs-12", "pl-xs-1", "pl-md-3"], "style": ""}, 0], ["div", {"class": ["bg-white", "display-block", "pb-xs-2", "mt-xs-0"], "style": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["div", {"class": ["wt-grid", "wt-grid--block", "wt-pl-xs-0"], "style": ""}, 22], ["li", {"class": ["wt-list-unstyled", "wt-grid__item-xs-6", "wt-grid__item-md-4", "wt-grid__item-lg-3", "wt-order-xs-6", "wt-order-sm-6", "wt-order-md-4", "wt-order-lg-2", "wt-order-xl-2", "wt-order-tv-2", "grid__item-xl-fifth", "tab-reorder"], "style": ""}, 0], ["div", {"class": ["js-merch-stash-check-listing", "v2-listing-card", "position-relative", "flex-xs-none"], "style": ""}, 0], ["a", {"class": ["b98e134f693f5746", "display-inline-block", "listing-link"], "style": ""}, 0], ["div", {"class": ["v2-listing-card__info"], "style": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["span", {"class": ["n-listing-card__price", "text-gray", "mt-xs-0", "strong", "display-block", "text-body-larger"], "style": ""}, 0], ["span", {"style": "", "class": ""}, 0], ["span", {"class": ["currency-value"], "style": ""}]], "wanted_attr": null, "is_full_url": null, "is_non_rec_text": null, "url": "", "hash": "2e9cc75573453bfd423abd74db66924f115e7a971f469deb1eb1f7a0fdd6a3b2", "stack_id": "rule_p1m3", "alias": "price"}, {"content": [["html", {"class": ["ui-toolkit"], "style": ""}, 0], ["body", {"class": ["transitional-wide", "guest", "no-touch", "is-responsive", "en-US", "USD", "US", "is-global-nav"], "style": ""}, 0], ["div", {"class": ["clear"], "style": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["div", {"class": ["content", "bg-white", "col-md-12", "pl-xs-1", "pr-xs-0", "pr-md-1", "pl-lg-0", "pr-lg-0", "bb-xs-1"], "style": ""}, 0], ["div", {"class": ["body-max-width"], "style": ""}, 0], ["div", {"class": ["mt-xs-3", "text-gray", "text-control"], "style": ""}, 1], ["div", {"class": ["col-group", "pl-xs-0", "search-listings-group", "pr-xs-1"], "style": ""}, 1], ["div", {"class": ["col-xs-12", "pl-xs-1", "pl-md-3"], "style": ""}, 0], ["div", {"class": ["bg-white", "display-block", "pb-xs-2", "mt-xs-0"], "style": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["div", {"class": ["wt-grid", "wt-grid--block", "wt-pl-xs-0"], "style": ""}, 2], ["li", {"class": ["wt-list-unstyled", "wt-grid__item-xs-6", "wt-grid__item-md-4", "wt-grid__item-lg-3", "wt-order-xs-2", "wt-order-sm-2", "wt-order-md-0", "wt-order-lg-0", "wt-order-xl-0", "wt-order-tv-0", "grid__item-xl-fifth", "tab-reorder"], "style": ""}, 0], ["div", {"class": ["js-merch-stash-check-listing", "v2-listing-card", "position-relative", "flex-xs-none"], "style": ""}, 0], ["a", {"class": ["b98e134f693f5746", "display-inline-block", "listing-link"], "style": ""}, 0], ["div", {"class": ["v2-listing-card__info"], "style": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["span", {"class": ["n-listing-card__price", "text-gray", "mt-xs-0", "strong", "display-block", "text-body-larger"], "style": ""}, 0], ["span", {"class": ["currency-value"], "style": ""}]], "wanted_attr": null, "is_full_url": null, "is_non_rec_text": null, "url": "", "hash": "a7add00ff42ab038501d3253afedee6cc838c355cffda738eaaf7ee3969f112e", "stack_id": "rule_tj7h", "alias": "price"}, {"content": [["html", {"class": ["ui-toolkit"], "style": ""}, 0], ["body", {"class": ["transitional-wide", "guest", "no-touch", "is-responsive", "en-US", "USD", "US", "is-global-nav"], "style": ""}, 0], ["div", {"class": ["clear"], "style": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["div", {"class": ["content", "bg-white", "col-md-12", "pl-xs-1", "pr-xs-0", "pr-md-1", "pl-lg-0", "pr-lg-0", "bb-xs-1"], "style": ""}, 0], ["div", {"class": ["body-max-width"], "style": ""}, 0], ["div", {"class": ["mt-xs-3", "text-gray", "text-control"], "style": ""}, 1], ["div", {"class": ["col-group", "pl-xs-0", "search-listings-group", "pr-xs-1"], "style": ""}, 1], ["div", {"class": ["col-xs-12", "pl-xs-1", "pl-md-3"], "style": ""}, 0], ["div", {"class": ["bg-white", "display-block", "pb-xs-2", "mt-xs-0"], "style": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["div", {"style": "", "class": ""}, 0], ["div", {"class": ["wt-grid", "wt-grid--block", "wt-pl-xs-0"], "style": ""}, 0], ["li", {"class": ["wt-list-unstyled", "wt-grid__item-xs-6", "wt-grid__item-md-4", "wt-grid__item-lg-3", "wt-order-xs-0", "wt-order-sm-0", "wt-order-md-0", "wt-order-lg-0", "wt-order-xl-0", "wt-order-tv-0", "grid__item-xl-fifth", "tab-reorder"], "style": ""}, 0], ["div", {"class": ["js-merch-stash-check-listing", "v2-listing-card", "position-relative", "flex-xs-none"], "style": ""}]], "wanted_attr": "data-palette-listing-id", "is_full_url": null, "is_non_rec_text": null, "url": "", "hash": "29a0fb22de3a6015e1770d549dc8003b0b58ce6701a190d4d11226e18760f020", "stack_id": "rule_vfo2", "alias": "url"}]} -------------------------------------------------------------------------------- /api_server/requirements.txt: -------------------------------------------------------------------------------- 1 | autoscraper 2 | flask 3 | -------------------------------------------------------------------------------- /qtictactoe/README.md: -------------------------------------------------------------------------------- 1 | # [Creating a Tic-Tac-Toe game with a Q-learning AI which masters the game](https://towardsdatascience.com/creating-a-tic-tac-toe-game-with-a-q-learning-ai-which-masters-the-game-9b0567d24de) 2 | -------------------------------------------------------------------------------- /qtictactoe/agent.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from q import Q 4 | from tictactoe import TicTacToe 5 | 6 | 7 | class Agent: 8 | def __init__(self): 9 | self.eps = 1.0 10 | self.qlearner = Q() 11 | 12 | def _get_action(self, state, valid_actions): 13 | if random.random() < self.eps: 14 | return random.choice(valid_actions) 15 | best = self.qlearner.get_best_action(state) 16 | if best is None: 17 | return random.choice(valid_actions) 18 | return best 19 | 20 | def _learn_one_game(self): 21 | game = TicTacToe(render=False) 22 | while True: 23 | state = game.get_state() 24 | action = self._get_action(state, game.get_valid_actions()) 25 | winner = game.play(*action) 26 | 27 | if winner or game.is_ended(): 28 | self.qlearner.update(state, action, game.get_state(), 100) 29 | break 30 | 31 | winner = game.play(*random.choice(game.get_valid_actions())) 32 | if winner or game.is_ended(): 33 | self.qlearner.update(state, action, game.get_state(), -100) 34 | break 35 | self.qlearner.update(state, action, game.get_state(), 0) 36 | 37 | def learn(self, n=20000): 38 | for _ in range(n): 39 | self._learn_one_game() 40 | self.eps -= 0.0001 41 | -------------------------------------------------------------------------------- /qtictactoe/main.py: -------------------------------------------------------------------------------- 1 | from agent import Agent 2 | from tictactoe import TicTacToe 3 | 4 | 5 | def play(agent): 6 | game = TicTacToe() 7 | while True: 8 | action = agent.qlearner.get_best_action(game.get_state()) 9 | winner = game.play(*action) 10 | if winner: 11 | print("**** you lost ****") 12 | return 13 | if game.is_ended(): 14 | print("**** draw ****") 15 | return 16 | x, y = input("input x and y: ").split() 17 | winner = game.play(int(x), int(y)) 18 | if winner: 19 | print("**** you won ****") 20 | return 21 | if game.is_ended(): 22 | print("**** draw ****") 23 | return 24 | 25 | 26 | q_agent = Agent() 27 | print("learning...") 28 | q_agent.learn() 29 | print("done") 30 | 31 | while True: 32 | print("\nlet's play\n") 33 | play(q_agent) 34 | -------------------------------------------------------------------------------- /qtictactoe/q.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | 3 | 4 | class Q: 5 | def __init__(self, alpha=0.5, discount=0.5): 6 | self.alpha = alpha 7 | self.discount = discount 8 | self.values = defaultdict(lambda: defaultdict(lambda: 0.0)) 9 | 10 | def update(self, state, action, next_state, reward): 11 | value = self.values[state][action] 12 | v = list(self.values[next_state].values()) 13 | next_q = max(v) if v else 0 14 | value = value + self.alpha * (reward + self.discount * next_q - value) 15 | self.values[state][action] = value 16 | 17 | def get_best_action(self, state): 18 | keys = list(self.values[state].keys()) 19 | if not keys: 20 | return None 21 | return max(keys, key=lambda x: self.values[state][x]) 22 | -------------------------------------------------------------------------------- /qtictactoe/tictactoe.py: -------------------------------------------------------------------------------- 1 | class TicTacToe: 2 | def __init__(self, render=True): 3 | self.board = [[0, 0, 0] for _ in range(3)] 4 | self.player = 1 5 | self.repr = {0: ".", 1: "x", -1: "o"} 6 | self.render = render 7 | 8 | def _get_winner(self): 9 | # check horizontal 10 | for i in range(3): 11 | if abs(sum(self.board[i])) == 3: 12 | return self.board[i][0] 13 | 14 | # check vertical 15 | for i in range(3): 16 | if abs(sum(self.board[j][i] for j in range(3))) == 3: 17 | return self.board[0][i] 18 | 19 | # check diagonal 20 | if abs(sum(self.board[i][i] for i in range(3))) == 3: 21 | return self.board[0][0] 22 | if abs(sum(self.board[i][2 - i] for i in range(3))) == 3: 23 | return self.board[0][2] 24 | 25 | return None 26 | 27 | def get_state(self): 28 | return str(self.board) 29 | 30 | def get_valid_actions(self): 31 | actions = [] 32 | for i in range(3): 33 | for j in range(3): 34 | if self.board[i][j] == 0: 35 | actions.append((i, j)) 36 | return actions 37 | 38 | def is_ended(self): 39 | for i in range(3): 40 | for j in range(3): 41 | if self.board[i][j] == 0: 42 | return False 43 | return True 44 | 45 | def _print(self): 46 | for row in self.board: 47 | for item in row: 48 | print(self.repr[item], end="\t") 49 | print("\n") 50 | print("-----------------\n") 51 | 52 | def play(self, x, y): 53 | if self.board[x][y] != 0: 54 | return None 55 | 56 | self.board[x][y] = self.player 57 | if self.render: 58 | self._print() 59 | winner = self._get_winner() 60 | if winner: 61 | return winner 62 | self.player *= -1 63 | return None 64 | --------------------------------------------------------------------------------