├── tests ├── __init__.py ├── screens │ ├── pinata.png │ ├── unlocked.png │ ├── round-started.png │ ├── shop-upgrades.png │ ├── round-in-progress.png │ ├── round-finished-missions.png │ ├── round-finished-results.png │ ├── round-start-beaster-bunny.png │ └── round-in-progress-pineapple-spank.png ├── __pycache__ │ └── __init__.cpython-35.pyc └── test_vision.py ├── .gitignore ├── assets ├── unlocked.png ├── bison-head.png ├── cancel-button.png ├── full-rocket.png ├── left-goalpost.png ├── next-button.png ├── pineapple-head.png ├── tap-to-continue.png ├── bison-health-bar.png ├── filled-with-goodies.png └── pineapple-health-bar.png ├── requirements.txt ├── website └── index.html ├── run.py ├── controller.py ├── vision.py ├── README.md └── game.py /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | venv/ 3 | -------------------------------------------------------------------------------- /assets/unlocked.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/assets/unlocked.png -------------------------------------------------------------------------------- /assets/bison-head.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/assets/bison-head.png -------------------------------------------------------------------------------- /assets/cancel-button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/assets/cancel-button.png -------------------------------------------------------------------------------- /assets/full-rocket.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/assets/full-rocket.png -------------------------------------------------------------------------------- /assets/left-goalpost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/assets/left-goalpost.png -------------------------------------------------------------------------------- /assets/next-button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/assets/next-button.png -------------------------------------------------------------------------------- /tests/screens/pinata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/tests/screens/pinata.png -------------------------------------------------------------------------------- /assets/pineapple-head.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/assets/pineapple-head.png -------------------------------------------------------------------------------- /assets/tap-to-continue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/assets/tap-to-continue.png -------------------------------------------------------------------------------- /tests/screens/unlocked.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/tests/screens/unlocked.png -------------------------------------------------------------------------------- /assets/bison-health-bar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/assets/bison-health-bar.png -------------------------------------------------------------------------------- /assets/filled-with-goodies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/assets/filled-with-goodies.png -------------------------------------------------------------------------------- /assets/pineapple-health-bar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/assets/pineapple-health-bar.png -------------------------------------------------------------------------------- /tests/screens/round-started.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/tests/screens/round-started.png -------------------------------------------------------------------------------- /tests/screens/shop-upgrades.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/tests/screens/shop-upgrades.png -------------------------------------------------------------------------------- /tests/screens/round-in-progress.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/tests/screens/round-in-progress.png -------------------------------------------------------------------------------- /tests/__pycache__/__init__.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/tests/__pycache__/__init__.cpython-35.pyc -------------------------------------------------------------------------------- /tests/screens/round-finished-missions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/tests/screens/round-finished-missions.png -------------------------------------------------------------------------------- /tests/screens/round-finished-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/tests/screens/round-finished-results.png -------------------------------------------------------------------------------- /tests/screens/round-start-beaster-bunny.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/tests/screens/round-start-beaster-bunny.png -------------------------------------------------------------------------------- /tests/screens/round-in-progress-pineapple-spank.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flakas/burrito-bison-bot/HEAD/tests/screens/round-in-progress-pineapple-spank.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cycler==0.10.0 2 | matplotlib==2.1.2 3 | mss==3.1.2 4 | nose==1.3.7 5 | pkg-resources==0.0.0 6 | pynput==1.3.9 7 | python-dateutil==2.6.1 8 | python-xlib==0.21 9 | pytz==2017.3 10 | -------------------------------------------------------------------------------- /website/index.html: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | 4 | from vision import Vision 5 | from controller import Controller 6 | from game import Game 7 | 8 | vision = Vision() 9 | controller = Controller() 10 | game = Game(vision, controller) 11 | 12 | # screenshot = vision.get_image('tests/screens/round-finished-results.png') 13 | # print(screenshot) 14 | # match = vision.find_template('bison-head', image=screenshot) 15 | # print(np.shape(match)[1]) 16 | 17 | game.run() 18 | -------------------------------------------------------------------------------- /controller.py: -------------------------------------------------------------------------------- 1 | import time 2 | from pynput.mouse import Button, Controller as MouseController 3 | 4 | class Controller: 5 | def __init__(self): 6 | self.mouse = MouseController() 7 | 8 | def move_mouse(self, x, y): 9 | def set_mouse_position(x, y): 10 | self.mouse.position = (int(x), int(y)) 11 | def smooth_move_mouse(from_x, from_y, to_x, to_y, speed=0.2): 12 | steps = 40 13 | sleep_per_step = speed // steps 14 | x_delta = (to_x - from_x) / steps 15 | y_delta = (to_y - from_y) / steps 16 | for step in range(steps): 17 | new_x = x_delta * (step + 1) + from_x 18 | new_y = y_delta * (step + 1) + from_y 19 | set_mouse_position(new_x, new_y) 20 | time.sleep(sleep_per_step) 21 | return smooth_move_mouse( 22 | self.mouse.position[0], 23 | self.mouse.position[1], 24 | x, 25 | y 26 | ) 27 | 28 | def left_mouse_click(self): 29 | self.mouse.click(Button.left) 30 | 31 | def left_mouse_drag(self, start, end): 32 | self.move_mouse(*start) 33 | time.sleep(0.2) 34 | self.mouse.press(Button.left) 35 | time.sleep(0.2) 36 | self.move_mouse(*end) 37 | time.sleep(0.2) 38 | self.mouse.release(Button.left) 39 | time.sleep(0.2) 40 | -------------------------------------------------------------------------------- /tests/test_vision.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from unittest.mock import Mock 3 | from vision import Vision 4 | import numpy as np 5 | 6 | class VisionTest(TestCase): 7 | 8 | def setUp(self): 9 | self.vision = Vision() 10 | 11 | def test_finds_finished_mission(self): 12 | screenshot = self.vision.get_image('tests/screens/round-finished-missions.png') 13 | match = self.vision.find_template('tap-to-continue', screenshot) 14 | self.assertEqual(np.shape(match)[1], 1) 15 | 16 | def test_finds_next_button(self): 17 | screenshot = self.vision.get_image('tests/screens/round-finished-results.png') 18 | match = self.vision.find_template('next-button', screenshot) 19 | self.assertEqual(np.shape(match)[1], 1) 20 | 21 | def test_finds_round_starting_indicator(self): 22 | screenshot = self.vision.get_image('tests/screens/round-started.png') 23 | match = self.vision.find_template('bison-health-bar', screenshot) 24 | self.assertEqual(np.shape(match)[1], 1) 25 | 26 | def test_finds_bison_head(self): 27 | screenshot = self.vision.get_image('tests/screens/round-started.png') 28 | match = self.vision.find_template('bison-head', screenshot) 29 | self.assertEqual(np.shape(match)[1], 1) 30 | 31 | def test_finds_pineapple_head(self): 32 | screenshot = self.vision.get_image('tests/screens/round-in-progress-pineapple-spank.png') 33 | match = self.vision.scaled_find_template('left-goalpost', screenshot, threshold=0.75, scales=[1.1, 1.0, 0.99, 0.98, 0.97, 0.96, 0.95]) 34 | self.assertGreaterEqual(np.shape(match)[1], 1) 35 | 36 | def test_finds_full_rocket(self): 37 | screenshot = self.vision.get_image('tests/screens/round-in-progress.png') 38 | match = self.vision.find_template('full-rocket', screenshot, threshold=0.9) 39 | self.assertGreaterEqual(np.shape(match)[1], 1) 40 | 41 | def test_finds_filled_with_goodies(self): 42 | screenshot = self.vision.get_image('tests/screens/pinata.png') 43 | match = self.vision.find_template('filled-with-goodies', screenshot, threshold=0.9) 44 | self.assertGreaterEqual(np.shape(match)[1], 1) 45 | 46 | def test_finds_cancel_button(self): 47 | screenshot = self.vision.get_image('tests/screens/pinata.png') 48 | match = self.vision.find_template('cancel-button', screenshot, threshold=0.9) 49 | self.assertGreaterEqual(np.shape(match)[1], 1) 50 | 51 | def test_finds_left_goalpost_with_beaster_bunny(self): 52 | screenshot = self.vision.get_image('tests/screens/round-start-beaster-bunny.png') 53 | match = self.vision.find_template('left-goalpost', screenshot, threshold=0.99) 54 | self.assertEqual(np.shape(match)[1], 1) 55 | 56 | -------------------------------------------------------------------------------- /vision.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | from mss import mss 3 | from PIL import Image 4 | import numpy as np 5 | import time 6 | 7 | class Vision: 8 | def __init__(self): 9 | self.static_templates = { 10 | 'left-goalpost': 'assets/left-goalpost.png', 11 | 'bison-head': 'assets/bison-head.png', 12 | 'pineapple-head': 'assets/pineapple-head.png', 13 | 'bison-health-bar': 'assets/bison-health-bar.png', 14 | 'pineapple-health-bar': 'assets/pineapple-health-bar.png', 15 | 'cancel-button': 'assets/cancel-button.png', 16 | 'filled-with-goodies': 'assets/filled-with-goodies.png', 17 | 'next-button': 'assets/next-button.png', 18 | 'tap-to-continue': 'assets/tap-to-continue.png', 19 | 'unlocked': 'assets/unlocked.png', 20 | 'full-rocket': 'assets/full-rocket.png' 21 | } 22 | 23 | self.templates = { k: cv2.imread(v, 0) for (k, v) in self.static_templates.items() } 24 | 25 | self.monitor = {'top': 0, 'left': 0, 'width': 1920, 'height': 1080} 26 | self.screen = mss() 27 | 28 | self.frame = None 29 | 30 | def take_screenshot(self): 31 | sct_img = self.screen.grab(self.monitor) 32 | img = Image.frombytes('RGB', sct_img.size, sct_img.rgb) 33 | img = np.array(img) 34 | img = self.convert_rgb_to_bgr(img) 35 | img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 36 | 37 | return img_gray 38 | 39 | def get_image(self, path): 40 | return cv2.imread(path, 0) 41 | 42 | def bgr_to_rgb(self, img): 43 | b,g,r = cv2.split(img) 44 | return cv2.merge([r,g,b]) 45 | 46 | def convert_rgb_to_bgr(self, img): 47 | return img[:, :, ::-1] 48 | 49 | def match_template(self, img_grayscale, template, threshold=0.9): 50 | """ 51 | Matches template image in a target grayscaled image 52 | """ 53 | 54 | res = cv2.matchTemplate(img_grayscale, template, cv2.TM_CCOEFF_NORMED) 55 | matches = np.where(res >= threshold) 56 | return matches 57 | 58 | def find_template(self, name, image=None, threshold=0.9): 59 | if image is None: 60 | if self.frame is None: 61 | self.refresh_frame() 62 | 63 | image = self.frame 64 | 65 | return self.match_template( 66 | image, 67 | self.templates[name], 68 | threshold 69 | ) 70 | 71 | def scaled_find_template(self, name, image=None, threshold=0.9, scales=[1.0, 0.9, 1.1]): 72 | if image is None: 73 | if self.frame is None: 74 | self.refresh_frame() 75 | 76 | image = self.frame 77 | 78 | initial_template = self.templates[name] 79 | for scale in scales: 80 | scaled_template = cv2.resize(initial_template, (0,0), fx=scale, fy=scale) 81 | matches = self.match_template( 82 | image, 83 | scaled_template, 84 | threshold 85 | ) 86 | if np.shape(matches)[1] >= 1: 87 | return matches 88 | return matches 89 | 90 | def refresh_frame(self): 91 | self.frame = self.take_screenshot() 92 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Burrito Bison Bot 2 | 3 | This is an example showing how a basic Computer Vision technique of Template Matching can be used to automate gameplay. 4 | Goal of this exercise is simply to explore how far one can get by using only template matching in OpenCV to automate primary tasks within the game. 5 | 6 | This exercise is based on [Burrito Bison: Launcha Libre](http://www.kongregate.com/games/JuicyBeast/burrito-bison-launcha-libre) free online game, a goal of which is to throw the main character as far as possible, crush gummy bears to gain gold, use gold to buy upgrades, and use upgrades to throw yourself as far as possible. 7 | If you've never seen the game before, here is [a human playing it](https://youtu.be/VQve8LoiFyQ). 8 | 9 | Why this game? The main reason is a fairly easily automatable gameplay. Controls require the player only use the mouse. 10 | There is also the grinding factor, requiring the player grind coins for upgrades to progress. 11 | 12 | ## Notes 13 | 14 | What works: 15 | 16 | - Launching the character; 17 | - Skipping Pinata ads; 18 | - Skipping mission screen; 19 | - Restarting the round; 20 | - Using the rocket; 21 | 22 | What doesn't work or could be improved: 23 | 24 | - Rocket is used whenever it fills up. It could be better timed for when there are targets below; 25 | - Only a single blast of a rocket is used. At later game stages multiple available blasts could be used when necessary; 26 | - Special capabilities gained from gummy bears aren't used (targeting with barrels, rockets, jumping jack); 27 | - Player launch isn't timed or targeted. This may cause slower speeds at start, or completely miss the opponent; 28 | - Some screens are not skipped, e.g. cards gained, unlocked items; 29 | - Shopping for upgrades. 30 | 31 | As it is right now, the bot can work semi-autonomously, requiring help to start the game, buy new upgrades and help it get past non-automated screens (such as new upgrade notices). 32 | 33 | ## Requirements 34 | 35 | - Python 3 36 | - OpenCV 37 | - Numpy 38 | - Pynput 39 | 40 | ## Installation 41 | 42 | - Clone this repository 43 | - Install opencv globally (see [these instructions](https://docs.opencv.org/trunk/d7/d9f/tutorial_linux_install.html)) 44 | - Create a virtual environment using system packages: `virtualenv -p python3 --system-site-packages venv` 45 | - Use virtualenv with `source venv/bin/activate` 46 | - Install required packages: `pip install -r requirements.txt` 47 | 48 | ## Running the bot 49 | 50 | 1) Run a resized version of Burrito Bison in your browser with `cd website && python -m SimpleHTTPServer 8080` 51 | 2) Open up `http://localhost:8080` in your browser 52 | 3) Run the bot with `python run.py` 53 | 4) Switch to the browser and ensure that the game screen is fully visible on your screen in the top left corner. 54 | 5) Go through the interfaces manually and put the game into a state where you would launch the bison from the Ring. The bot should pick it up and launch the hero from there. 55 | 56 | Tips: 57 | 58 | - This bot can no longer be run on Kongregate's website, you have to run a resized version of the game locally. This bot was tailored for Burrito Bison game to be run in 1173x660 px game screen, however Kongregate has since changed their design slightly, running the game in 1280x768 resolution, which throws off scaling in image templates. 59 | - The bot is configured to work only with 1920x1080 screen resolution. If you have an HD (1366x768) or a 4K monitor, you may need to play around with game screen placement, or change observed area in `vision.py` 60 | -------------------------------------------------------------------------------- /game.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import time 3 | 4 | class Game: 5 | 6 | def __init__(self, vision, controller): 7 | self.vision = vision 8 | self.controller = controller 9 | self.state = 'not started' 10 | 11 | def run(self): 12 | while True: 13 | self.vision.refresh_frame() 14 | if self.state == 'not started' and self.round_starting('bison'): 15 | self.log('Round needs to be started, launching bison') 16 | self.launch_player() 17 | self.state = 'started' 18 | if self.state == 'not started' and self.round_starting('pineapple'): 19 | self.log('Round needs to be started, launching pineapple') 20 | try: 21 | self.launch_player() 22 | self.state = 'started' 23 | except Exception as ex: 24 | self.log('Failed to find pineapple character') 25 | elif self.state == 'started' and self.found_pinata(): 26 | self.log('Found a pinata, attempting to skip') 27 | self.click_cancel() 28 | elif self.state == 'started' and self.round_finished(): 29 | self.log('Round finished, clicking to continue') 30 | self.click_to_continue() 31 | self.state = 'mission_finished' 32 | elif self.state == 'started' and self.has_full_rocket(): 33 | self.log('Round in progress, has full rocket, attempting to use it') 34 | self.use_full_rocket() 35 | elif self.state == 'mission_finished' and self.can_start_round(): 36 | self.log('Mission finished, trying to restart round') 37 | self.start_round() 38 | self.state = 'not started' 39 | else: 40 | self.log('Not doing anything') 41 | time.sleep(1) 42 | 43 | def round_starting(self, player): 44 | matches = self.vision.find_template('%s-health-bar' % player) 45 | return np.shape(matches)[1] >= 1 46 | 47 | def launch_player(self): 48 | # Try multiple sizes of goalpost due to perspective changes for 49 | # different opponents 50 | scales = [1.2, 1.1, 1.05, 1.04, 1.03, 1.02, 1.01, 1.0, 0.99, 0.98, 0.97, 0.96, 0.95] 51 | matches = self.vision.scaled_find_template('left-goalpost', threshold=0.75, scales=scales) 52 | x = matches[1][0] 53 | y = matches[0][0] 54 | 55 | self.controller.left_mouse_drag( 56 | (x, y), 57 | (x-200, y+10) 58 | ) 59 | 60 | time.sleep(0.5) 61 | 62 | def round_finished(self): 63 | matches = self.vision.find_template('tap-to-continue') 64 | return np.shape(matches)[1] >= 1 65 | 66 | def click_to_continue(self): 67 | matches = self.vision.find_template('tap-to-continue') 68 | 69 | x = matches[1][0] 70 | y = matches[0][0] 71 | 72 | self.controller.move_mouse(x+50, y+30) 73 | self.controller.left_mouse_click() 74 | 75 | time.sleep(0.5) 76 | 77 | def can_start_round(self): 78 | matches = self.vision.find_template('next-button') 79 | return np.shape(matches)[1] >= 1 80 | 81 | def start_round(self): 82 | matches = self.vision.find_template('next-button') 83 | 84 | x = matches[1][0] 85 | y = matches[0][0] 86 | 87 | self.controller.move_mouse(x+100, y+30) 88 | self.controller.left_mouse_click() 89 | 90 | time.sleep(0.5) 91 | 92 | def has_full_rocket(self): 93 | matches = self.vision.find_template('full-rocket', threshold=0.9) 94 | return np.shape(matches)[1] >= 1 95 | 96 | def use_full_rocket(self): 97 | matches = self.vision.find_template('full-rocket') 98 | 99 | x = matches[1][0] 100 | y = matches[0][0] 101 | 102 | self.controller.move_mouse(x, y) 103 | self.controller.left_mouse_click() 104 | 105 | time.sleep(0.5) 106 | 107 | def found_pinata(self): 108 | matches = self.vision.find_template('filled-with-goodies', threshold=0.9) 109 | return np.shape(matches)[1] >= 1 110 | 111 | def click_cancel(self): 112 | matches = self.vision.find_template('cancel-button') 113 | 114 | x = matches[1][0] 115 | y = matches[0][0] 116 | 117 | self.controller.move_mouse(x, y) 118 | self.controller.left_mouse_click() 119 | 120 | time.sleep(0.5) 121 | 122 | def log(self, text): 123 | print('[%s] %s' % (time.strftime('%H:%M:%S'), text)) 124 | --------------------------------------------------------------------------------