├── labels.txt ├── images ├── bk.jpg ├── Inky.png ├── Blinky.png ├── Clyde.png ├── Pinky.png ├── pacman.jpg └── Trollman.png ├── output_13_1.png ├── src └── freesansbold.ttf ├── data ├── up │ ├── saved_v1_0000.jpg │ ├── saved_v1_0001.jpg │ ├── saved_v1_0002.jpg │ ├── saved_v1_0003.jpg │ └── saved_v1_0004.jpg ├── down │ ├── saved_v1_0000.jpg │ ├── saved_v1_0001.jpg │ ├── saved_v1_0002.jpg │ ├── saved_v1_0003.jpg │ └── saved_v1_0004.jpg ├── left │ ├── saved_v1_0000.jpg │ ├── saved_v1_0001.jpg │ ├── saved_v1_0002.jpg │ ├── saved_v1_0003.jpg │ └── saved_v1_0004.jpg ├── pause │ ├── saved_v1_0000.jpg │ ├── saved_v1_0001.jpg │ ├── saved_v1_0002.jpg │ ├── saved_v1_0003.jpg │ └── saved_v1_0004.jpg └── right │ ├── saved_v1_0000.jpg │ ├── saved_v1_0001.jpg │ ├── saved_v1_0002.jpg │ ├── saved_v1_0003.jpg │ └── saved_v1_0004.jpg ├── tools ├── __pycache__ │ ├── utils.cpython-36.pyc │ └── config.cpython-36.pyc ├── test.py ├── train.py ├── utils.py └── config.py ├── collect └── PalmTracker.py ├── utils.py ├── config.py ├── demo.py ├── README.md ├── train_list.txt └── game.py /labels.txt: -------------------------------------------------------------------------------- 1 | down 2 | left 3 | pause 4 | right 5 | up 6 | -------------------------------------------------------------------------------- /images/bk.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/images/bk.jpg -------------------------------------------------------------------------------- /images/Inky.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/images/Inky.png -------------------------------------------------------------------------------- /output_13_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/output_13_1.png -------------------------------------------------------------------------------- /images/Blinky.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/images/Blinky.png -------------------------------------------------------------------------------- /images/Clyde.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/images/Clyde.png -------------------------------------------------------------------------------- /images/Pinky.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/images/Pinky.png -------------------------------------------------------------------------------- /images/pacman.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/images/pacman.jpg -------------------------------------------------------------------------------- /images/Trollman.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/images/Trollman.png -------------------------------------------------------------------------------- /src/freesansbold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/src/freesansbold.ttf -------------------------------------------------------------------------------- /data/up/saved_v1_0000.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/up/saved_v1_0000.jpg -------------------------------------------------------------------------------- /data/up/saved_v1_0001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/up/saved_v1_0001.jpg -------------------------------------------------------------------------------- /data/up/saved_v1_0002.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/up/saved_v1_0002.jpg -------------------------------------------------------------------------------- /data/up/saved_v1_0003.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/up/saved_v1_0003.jpg -------------------------------------------------------------------------------- /data/up/saved_v1_0004.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/up/saved_v1_0004.jpg -------------------------------------------------------------------------------- /data/down/saved_v1_0000.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/down/saved_v1_0000.jpg -------------------------------------------------------------------------------- /data/down/saved_v1_0001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/down/saved_v1_0001.jpg -------------------------------------------------------------------------------- /data/down/saved_v1_0002.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/down/saved_v1_0002.jpg -------------------------------------------------------------------------------- /data/down/saved_v1_0003.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/down/saved_v1_0003.jpg -------------------------------------------------------------------------------- /data/down/saved_v1_0004.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/down/saved_v1_0004.jpg -------------------------------------------------------------------------------- /data/left/saved_v1_0000.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/left/saved_v1_0000.jpg -------------------------------------------------------------------------------- /data/left/saved_v1_0001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/left/saved_v1_0001.jpg -------------------------------------------------------------------------------- /data/left/saved_v1_0002.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/left/saved_v1_0002.jpg -------------------------------------------------------------------------------- /data/left/saved_v1_0003.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/left/saved_v1_0003.jpg -------------------------------------------------------------------------------- /data/left/saved_v1_0004.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/left/saved_v1_0004.jpg -------------------------------------------------------------------------------- /data/pause/saved_v1_0000.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/pause/saved_v1_0000.jpg -------------------------------------------------------------------------------- /data/pause/saved_v1_0001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/pause/saved_v1_0001.jpg -------------------------------------------------------------------------------- /data/pause/saved_v1_0002.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/pause/saved_v1_0002.jpg -------------------------------------------------------------------------------- /data/pause/saved_v1_0003.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/pause/saved_v1_0003.jpg -------------------------------------------------------------------------------- /data/pause/saved_v1_0004.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/pause/saved_v1_0004.jpg -------------------------------------------------------------------------------- /data/right/saved_v1_0000.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/right/saved_v1_0000.jpg -------------------------------------------------------------------------------- /data/right/saved_v1_0001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/right/saved_v1_0001.jpg -------------------------------------------------------------------------------- /data/right/saved_v1_0002.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/right/saved_v1_0002.jpg -------------------------------------------------------------------------------- /data/right/saved_v1_0003.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/right/saved_v1_0003.jpg -------------------------------------------------------------------------------- /data/right/saved_v1_0004.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/data/right/saved_v1_0004.jpg -------------------------------------------------------------------------------- /tools/__pycache__/utils.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/tools/__pycache__/utils.cpython-36.pyc -------------------------------------------------------------------------------- /tools/__pycache__/config.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/pacman-with-paddlepaddle-gesture-control/HEAD/tools/__pycache__/config.cpython-36.pyc -------------------------------------------------------------------------------- /tools/test.py: -------------------------------------------------------------------------------- 1 | from paddlex.cls import transforms 2 | import paddlex 3 | import cv2 4 | 5 | base = './data' 6 | 7 | train_transforms = transforms.Compose([ 8 | transforms.ResizeByShort(256), 9 | transforms.RandomCrop(crop_size=224), 10 | transforms.RandomRotate(rotate_range=30, prob=0.5), 11 | transforms.RandomDistort(), 12 | transforms.Normalize() 13 | ]) 14 | 15 | train_dataset = paddlex.datasets.ImageNet( 16 | data_dir=base, 17 | file_list='train_list.txt', 18 | label_list='labels.txt', 19 | transforms=train_transforms, 20 | shuffle=True) 21 | 22 | model = paddlex.load_model('weights/final') 23 | im = cv2.imread('test.jpg') 24 | 25 | print( 26 | model.evaluate(eval_dataset=train_dataset) 27 | ) 28 | -------------------------------------------------------------------------------- /tools/train.py: -------------------------------------------------------------------------------- 1 | from paddlex.cls import transforms 2 | import os 3 | import cv2 4 | import numpy as np 5 | import paddlex as pdx 6 | 7 | base = './data' 8 | 9 | with open(os.path.join('train_list.txt'), 'w') as f: 10 | for i, cls_fold in enumerate(os.listdir(base)): 11 | cls_base = os.path.join(base, cls_fold) 12 | files = os.listdir(cls_base) 13 | print('{} train num:'.format(cls_fold), len(files)) 14 | for pt in files: 15 | img = os.path.join(cls_fold, pt) 16 | info = img + ' ' + str(i) + '\n' 17 | f.write(info) 18 | 19 | with open(os.path.join('labels.txt'), 'w') as f: 20 | for i, cls_fold in enumerate(os.listdir(base)): 21 | f.write(cls_fold+'\n') 22 | 23 | train_transforms = transforms.Compose([ 24 | transforms.RandomCrop(crop_size=224), 25 | transforms.Normalize() 26 | ]) 27 | 28 | train_dataset = pdx.datasets.ImageNet( 29 | data_dir=base, 30 | file_list='train_list.txt', 31 | label_list='labels.txt', 32 | transforms=train_transforms, 33 | shuffle=True) 34 | 35 | num_classes = len(train_dataset.labels) 36 | model = pdx.cls.ResNet18(num_classes=num_classes) 37 | model.train(num_epochs=20, 38 | train_dataset=train_dataset, 39 | train_batch_size=32, 40 | lr_decay_epochs=[5, 10, 15], 41 | learning_rate=2e-3, 42 | save_dir='new', 43 | log_interval_steps=5, 44 | save_interval_epochs=4, 45 | pretrain_weights='IMAGENET', 46 | use_vdl=True) 47 | 48 | im = cv2.imread('test.jpg') 49 | result = model.predict('test.jpg', topk=1, transforms=train_transforms) 50 | print("Predict Result:", result) -------------------------------------------------------------------------------- /collect/PalmTracker.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import imutils 3 | import numpy as np 4 | import argparse 5 | bg = None 6 | 7 | 8 | def run_avg(image, aWeight): 9 | global bg 10 | if bg is None: 11 | bg = image.copy().astype('float') 12 | return 13 | 14 | cv2.accumulateWeighted(image, bg, aWeight) 15 | 16 | 17 | def segment(image, threshold=25): 18 | global bg 19 | diff = cv2.absdiff(bg.astype('uint8'), image) 20 | 21 | thresholded = cv2.threshold(diff, 22 | threshold, 23 | 255, 24 | cv2.THRESH_BINARY)[1] 25 | 26 | (cnts, _) = cv2.findContours(thresholded.copy(), 27 | cv2.RETR_EXTERNAL, 28 | cv2.CHAIN_APPROX_SIMPLE) 29 | 30 | if len(cnts) == 0: 31 | return 32 | else: 33 | segmented = max(cnts, key=cv2.contourArea) 34 | return (thresholded, segmented) 35 | 36 | 37 | def main(dtype): 38 | aWeight = 0.5 39 | 40 | camera = cv2.VideoCapture(0) 41 | 42 | top, right, bottom, left = 90, 380, 285, 590 43 | 44 | num_frames = 0 45 | thresholded = None 46 | 47 | count = 0 48 | 49 | while(True): 50 | (grabbed, frame) = camera.read() 51 | if grabbed: 52 | 53 | frame = imutils.resize(frame, width=700) 54 | 55 | frame = cv2.flip(frame, 1) 56 | 57 | clone = frame.copy() 58 | 59 | (height, width) = frame.shape[:2] 60 | 61 | roi = frame[top:bottom, right:left] 62 | 63 | thresholded = roi 64 | 65 | cv2.rectangle(clone, (left, top), (right, bottom), (0, 255, 0), 2) 66 | 67 | num_frames += 1 68 | 69 | cv2.imshow('Video Feed', clone) 70 | if not thresholded is None: 71 | cv2.imshow('Thesholded', thresholded) 72 | 73 | keypress = cv2.waitKey(1) & 0xFF 74 | 75 | if keypress == ord('q'): 76 | break 77 | 78 | if keypress == ord('s'): 79 | cv2.imwrite('data/{}/saved_v1_{:04}.jpg'.format(dtype, count), thresholded) 80 | count += 1 81 | print(count, 'saved.') 82 | 83 | else: 84 | camera.release() 85 | break 86 | 87 | if __name__ == '__main__': 88 | parser = argparse.ArgumentParser() 89 | parser.add_argument('--dtype', type=str, default='down') 90 | args = parser.parse_args() 91 | main(args.dtype) 92 | cv2.destroyAllWindows() 93 | -------------------------------------------------------------------------------- /tools/utils.py: -------------------------------------------------------------------------------- 1 | import pygame 2 | from tools.config import * 3 | 4 | 5 | class Player(pygame.sprite.Sprite): 6 | 7 | change_x = 0 8 | change_y = 0 9 | 10 | def __init__(self, x, y, filename): 11 | pygame.sprite.Sprite.__init__(self) 12 | 13 | self.image = pygame.image.load(filename).convert() 14 | 15 | self.rect = self.image.get_rect() 16 | self.rect.top = y 17 | self.rect.left = x 18 | self.prev_x = x 19 | self.prev_y = y 20 | 21 | def prevdirection(self): 22 | self.prev_x = self.change_x 23 | self.prev_y = self.change_y 24 | 25 | def changespeed(self, x, y): 26 | self.change_x = x 27 | self.change_y = y 28 | 29 | def update(self, walls, gate): 30 | 31 | old_x = self.rect.left 32 | new_x = old_x+self.change_x 33 | prev_x = old_x+self.prev_x 34 | self.rect.left = new_x 35 | 36 | old_y = self.rect.top 37 | new_y = old_y+self.change_y 38 | prev_y = old_y+self.prev_y 39 | 40 | x_collide = pygame.sprite.spritecollide(self, walls, False) 41 | if x_collide: 42 | self.rect.left = old_x 43 | else: 44 | self.rect.top = new_y 45 | 46 | y_collide = pygame.sprite.spritecollide(self, walls, False) 47 | if y_collide: 48 | self.rect.top = old_y 49 | 50 | if gate != False: 51 | gate_hit = pygame.sprite.spritecollide(self, gate, False) 52 | if gate_hit: 53 | self.rect.left = old_x 54 | self.rect.top = old_y 55 | 56 | 57 | class Wall(pygame.sprite.Sprite): 58 | def __init__(self, x, y, width, height, color): 59 | pygame.sprite.Sprite.__init__(self) 60 | 61 | self.image = pygame.Surface([width, height]) 62 | self.image.fill(color) 63 | 64 | self.rect = self.image.get_rect() 65 | self.rect.top = y 66 | self.rect.left = x 67 | 68 | 69 | class Block(pygame.sprite.Sprite): 70 | def __init__(self, color, width, height): 71 | pygame.sprite.Sprite.__init__(self) 72 | 73 | self.image = pygame.Surface([width, height]) 74 | self.image.fill(white) 75 | self.image.set_colorkey(white) 76 | pygame.draw.ellipse(self.image, color, [0, 0, width, height]) 77 | 78 | self.rect = self.image.get_rect() 79 | 80 | 81 | class Ghost(Player): 82 | def changespeed(self, list, ghost, turn, steps, l): 83 | try: 84 | z = list[turn][2] 85 | if steps < z: 86 | self.change_x = list[turn][0] 87 | self.change_y = list[turn][1] 88 | steps += 1 89 | else: 90 | if turn < l: 91 | turn += 1 92 | elif ghost == 'clyde': 93 | turn = 2 94 | else: 95 | turn = 0 96 | self.change_x = list[turn][0] 97 | self.change_y = list[turn][1] 98 | steps = 0 99 | return [turn, steps] 100 | except IndexError: 101 | return [0, 0] 102 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import pygame 2 | from config import * 3 | 4 | 5 | class Player(pygame.sprite.Sprite): 6 | 7 | change_x = 0 8 | change_y = 0 9 | 10 | def __init__(self, x, y, filename): 11 | pygame.sprite.Sprite.__init__(self) 12 | self.count = 0 13 | self.image = pygame.image.load(filename).convert_alpha() 14 | 15 | self.rect = self.image.get_rect() 16 | self.rect.top = y 17 | self.rect.left = x 18 | self.prev_x = x 19 | self.prev_y = y 20 | 21 | def prevdirection(self): 22 | self.prev_x = self.change_x 23 | self.prev_y = self.change_y 24 | 25 | def changespeed(self, x, y): 26 | self.change_x = x 27 | self.change_y = y 28 | 29 | def update(self, walls, gate): 30 | 31 | old_x = self.rect.left 32 | new_x = old_x+self.change_x 33 | prev_x = old_x+self.prev_x 34 | self.rect.left = new_x 35 | 36 | old_y = self.rect.top 37 | new_y = old_y+self.change_y 38 | prev_y = old_y+self.prev_y 39 | 40 | x_collide = pygame.sprite.spritecollide(self, walls, False) 41 | if x_collide: 42 | self.rect.left = old_x 43 | else: 44 | self.rect.top = new_y 45 | 46 | y_collide = pygame.sprite.spritecollide(self, walls, False) 47 | if y_collide: 48 | self.rect.top = old_y 49 | 50 | if gate != False: 51 | gate_hit = pygame.sprite.spritecollide(self, gate, False) 52 | if gate_hit: 53 | self.rect.left = old_x 54 | self.rect.top = old_y 55 | 56 | 57 | class Wall(pygame.sprite.Sprite): 58 | def __init__(self, x, y, width, height, color): 59 | pygame.sprite.Sprite.__init__(self) 60 | 61 | self.image = pygame.Surface([width, height]) 62 | self.image.fill(color) 63 | 64 | self.rect = self.image.get_rect() 65 | self.rect.top = y 66 | self.rect.left = x 67 | 68 | 69 | class Block(pygame.sprite.Sprite): 70 | def __init__(self, color, width, height): 71 | pygame.sprite.Sprite.__init__(self) 72 | 73 | self.image = pygame.Surface([width, height]) 74 | self.image.fill(white) 75 | self.image.set_colorkey(white) 76 | pygame.draw.ellipse(self.image, color, [0, 0, width, height]) 77 | 78 | self.rect = self.image.get_rect() 79 | 80 | 81 | class Ghost(Player): 82 | def changespeed(self, list, ghost, turn, steps, l): 83 | try: 84 | z = list[turn][2] 85 | if steps < z: 86 | self.change_x = list[turn][0] 87 | self.change_y = list[turn][1] 88 | steps += 1 89 | else: 90 | if turn < l: 91 | turn += 1 92 | elif ghost == 'clyde': 93 | turn = 2 94 | else: 95 | turn = 0 96 | self.change_x = list[turn][0] 97 | self.change_y = list[turn][1] 98 | steps = 0 99 | return [turn, steps] 100 | except IndexError: 101 | return [0, 0] 102 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | import pygame 2 | 3 | Trollicon = pygame.image.load('images/Trollman.png') 4 | 5 | walls = [[0, 0, 6, 600], 6 | [0, 0, 600, 6], 7 | [0, 600, 606, 6], 8 | [600, 0, 6, 606], 9 | [300, 0, 6, 66], 10 | [60, 60, 186, 6], 11 | [360, 60, 186, 6], 12 | [60, 120, 66, 6], 13 | [60, 120, 6, 126], 14 | [180, 120, 246, 6], 15 | [300, 120, 6, 66], 16 | [480, 120, 66, 6], 17 | [540, 120, 6, 126], 18 | [120, 180, 126, 6], 19 | [120, 180, 6, 126], 20 | [360, 180, 126, 6], 21 | [480, 180, 6, 126], 22 | [180, 240, 6, 126], 23 | [180, 360, 246, 6], 24 | [420, 240, 6, 126], 25 | [240, 240, 42, 6], 26 | [324, 240, 42, 6], 27 | [240, 240, 6, 66], 28 | [240, 300, 126, 6], 29 | [360, 240, 6, 66], 30 | [0, 300, 66, 6], 31 | [540, 300, 66, 6], 32 | [60, 360, 66, 6], 33 | [60, 360, 6, 186], 34 | [480, 360, 66, 6], 35 | [540, 360, 6, 186], 36 | [120, 420, 366, 6], 37 | [120, 420, 6, 66], 38 | [480, 420, 6, 66], 39 | [180, 480, 246, 6], 40 | [300, 480, 6, 66], 41 | [120, 540, 126, 6], 42 | [360, 540, 126, 6] 43 | ] 44 | Pinky_directions = [ 45 | [0, -30, 4], 46 | [15, 0, 9], 47 | [0, 15, 11], 48 | [-15, 0, 23], 49 | [0, 15, 7], 50 | [15, 0, 3], 51 | [0, -15, 3], 52 | [15, 0, 19], 53 | [0, 15, 3], 54 | [15, 0, 3], 55 | [0, 15, 3], 56 | [15, 0, 3], 57 | [0, -15, 15], 58 | [-15, 0, 7], 59 | [0, 15, 3], 60 | [-15, 0, 19], 61 | [0, -15, 11], 62 | [15, 0, 9] 63 | ] 64 | 65 | Blinky_directions = [ 66 | [0, -15, 4], 67 | [15, 0, 9], 68 | [0, 15, 11], 69 | [15, 0, 3], 70 | [0, 15, 7], 71 | [-15, 0, 11], 72 | [0, 15, 3], 73 | [15, 0, 15], 74 | [0, -15, 15], 75 | [15, 0, 3], 76 | [0, -15, 11], 77 | [-15, 0, 3], 78 | [0, -15, 11], 79 | [-15, 0, 3], 80 | [0, -15, 3], 81 | [-15, 0, 7], 82 | [0, -15, 3], 83 | [15, 0, 15], 84 | [0, 15, 15], 85 | [-15, 0, 3], 86 | [0, 15, 3], 87 | [-15, 0, 3], 88 | [0, -15, 7], 89 | [-15, 0, 3], 90 | [0, 15, 7], 91 | [-15, 0, 11], 92 | [0, -15, 7], 93 | [15, 0, 5] 94 | ] 95 | 96 | Inky_directions = [ 97 | [30, 0, 2], 98 | [0, -15, 4], 99 | [15, 0, 10], 100 | [0, 15, 7], 101 | [15, 0, 3], 102 | [0, -15, 3], 103 | [15, 0, 3], 104 | [0, -15, 15], 105 | [-15, 0, 15], 106 | [0, 15, 3], 107 | [15, 0, 15], 108 | [0, 15, 11], 109 | [-15, 0, 3], 110 | [0, -15, 7], 111 | [-15, 0, 11], 112 | [0, 15, 3], 113 | [-15, 0, 11], 114 | [0, 15, 7], 115 | [-15, 0, 3], 116 | [0, -15, 3], 117 | [-15, 0, 3], 118 | [0, -15, 15], 119 | [15, 0, 15], 120 | [0, 15, 3], 121 | [-15, 0, 15], 122 | [0, 15, 11], 123 | [15, 0, 3], 124 | [0, -15, 11], 125 | [15, 0, 11], 126 | [0, 15, 3], 127 | [15, 0, 1], 128 | ] 129 | 130 | Clyde_directions = [ 131 | [-30, 0, 2], 132 | [0, -15, 4], 133 | [15, 0, 5], 134 | [0, 15, 7], 135 | [-15, 0, 11], 136 | [0, -15, 7], 137 | [-15, 0, 3], 138 | [0, 15, 7], 139 | [-15, 0, 7], 140 | [0, 15, 15], 141 | [15, 0, 15], 142 | [0, -15, 3], 143 | [-15, 0, 11], 144 | [0, -15, 7], 145 | [15, 0, 3], 146 | [0, -15, 11], 147 | [15, 0, 9], 148 | ] 149 | 150 | black = (100, 100, 100) 151 | white = (255, 255, 255) 152 | blue = (0, 0, 200) 153 | green = (0, 255, 0) 154 | red = (255, 0, 0) 155 | purple = (255, 0, 255) 156 | yellow = (0, 200, 0) 157 | CLASSES = ['pause', 'up', 'down', 'left', 'right'] 158 | 159 | w = 303-16 # Width 160 | p_h = (7*60)+19 # Pacman height 161 | m_h = (4*60)+19 # Monster height 162 | b_h = (3*60)+19 # Binky height 163 | i_w = 303-16-32 # Inky width 164 | c_w = 303+(32-16) # Clyde width -------------------------------------------------------------------------------- /tools/config.py: -------------------------------------------------------------------------------- 1 | import pygame 2 | 3 | Trollicon = pygame.image.load('images/Trollman.png') 4 | 5 | walls = [[0, 0, 6, 600], 6 | [0, 0, 600, 6], 7 | [0, 600, 606, 6], 8 | [600, 0, 6, 606], 9 | [300, 0, 6, 66], 10 | [60, 60, 186, 6], 11 | [360, 60, 186, 6], 12 | [60, 120, 66, 6], 13 | [60, 120, 6, 126], 14 | [180, 120, 246, 6], 15 | [300, 120, 6, 66], 16 | [480, 120, 66, 6], 17 | [540, 120, 6, 126], 18 | [120, 180, 126, 6], 19 | [120, 180, 6, 126], 20 | [360, 180, 126, 6], 21 | [480, 180, 6, 126], 22 | [180, 240, 6, 126], 23 | [180, 360, 246, 6], 24 | [420, 240, 6, 126], 25 | [240, 240, 42, 6], 26 | [324, 240, 42, 6], 27 | [240, 240, 6, 66], 28 | [240, 300, 126, 6], 29 | [360, 240, 6, 66], 30 | [0, 300, 66, 6], 31 | [540, 300, 66, 6], 32 | [60, 360, 66, 6], 33 | [60, 360, 6, 186], 34 | [480, 360, 66, 6], 35 | [540, 360, 6, 186], 36 | [120, 420, 366, 6], 37 | [120, 420, 6, 66], 38 | [480, 420, 6, 66], 39 | [180, 480, 246, 6], 40 | [300, 480, 6, 66], 41 | [120, 540, 126, 6], 42 | [360, 540, 126, 6] 43 | ] 44 | Pinky_directions = [ 45 | [0, -30, 4], 46 | [15, 0, 9], 47 | [0, 15, 11], 48 | [-15, 0, 23], 49 | [0, 15, 7], 50 | [15, 0, 3], 51 | [0, -15, 3], 52 | [15, 0, 19], 53 | [0, 15, 3], 54 | [15, 0, 3], 55 | [0, 15, 3], 56 | [15, 0, 3], 57 | [0, -15, 15], 58 | [-15, 0, 7], 59 | [0, 15, 3], 60 | [-15, 0, 19], 61 | [0, -15, 11], 62 | [15, 0, 9] 63 | ] 64 | 65 | Blinky_directions = [ 66 | [0, -15, 4], 67 | [15, 0, 9], 68 | [0, 15, 11], 69 | [15, 0, 3], 70 | [0, 15, 7], 71 | [-15, 0, 11], 72 | [0, 15, 3], 73 | [15, 0, 15], 74 | [0, -15, 15], 75 | [15, 0, 3], 76 | [0, -15, 11], 77 | [-15, 0, 3], 78 | [0, -15, 11], 79 | [-15, 0, 3], 80 | [0, -15, 3], 81 | [-15, 0, 7], 82 | [0, -15, 3], 83 | [15, 0, 15], 84 | [0, 15, 15], 85 | [-15, 0, 3], 86 | [0, 15, 3], 87 | [-15, 0, 3], 88 | [0, -15, 7], 89 | [-15, 0, 3], 90 | [0, 15, 7], 91 | [-15, 0, 11], 92 | [0, -15, 7], 93 | [15, 0, 5] 94 | ] 95 | 96 | Inky_directions = [ 97 | [30, 0, 2], 98 | [0, -15, 4], 99 | [15, 0, 10], 100 | [0, 15, 7], 101 | [15, 0, 3], 102 | [0, -15, 3], 103 | [15, 0, 3], 104 | [0, -15, 15], 105 | [-15, 0, 15], 106 | [0, 15, 3], 107 | [15, 0, 15], 108 | [0, 15, 11], 109 | [-15, 0, 3], 110 | [0, -15, 7], 111 | [-15, 0, 11], 112 | [0, 15, 3], 113 | [-15, 0, 11], 114 | [0, 15, 7], 115 | [-15, 0, 3], 116 | [0, -15, 3], 117 | [-15, 0, 3], 118 | [0, -15, 15], 119 | [15, 0, 15], 120 | [0, 15, 3], 121 | [-15, 0, 15], 122 | [0, 15, 11], 123 | [15, 0, 3], 124 | [0, -15, 11], 125 | [15, 0, 11], 126 | [0, 15, 3], 127 | [15, 0, 1], 128 | ] 129 | 130 | Clyde_directions = [ 131 | [-30, 0, 2], 132 | [0, -15, 4], 133 | [15, 0, 5], 134 | [0, 15, 7], 135 | [-15, 0, 11], 136 | [0, -15, 7], 137 | [-15, 0, 3], 138 | [0, 15, 7], 139 | [-15, 0, 7], 140 | [0, 15, 15], 141 | [15, 0, 15], 142 | [0, -15, 3], 143 | [-15, 0, 11], 144 | [0, -15, 7], 145 | [15, 0, 3], 146 | [0, -15, 11], 147 | [15, 0, 9], 148 | ] 149 | 150 | black = (0, 0, 0) 151 | white = (255, 255, 255) 152 | blue = (0, 0, 255) 153 | green = (0, 255, 0) 154 | red = (255, 0, 0) 155 | purple = (255, 0, 255) 156 | yellow = (255, 255, 0) 157 | CLASSES = ['pause', 'up', 'down', 'left', 'right'] 158 | 159 | w = 303-16 # Width 160 | p_h = (7*60)+19 # Pacman height 161 | m_h = (4*60)+19 # Monster height 162 | b_h = (3*60)+19 # Binky height 163 | i_w = 303-16-32 # Inky width 164 | c_w = 303+(32-16) # Clyde width -------------------------------------------------------------------------------- /demo.py: -------------------------------------------------------------------------------- 1 | import paddlex 2 | from paddlex.cls import transforms 3 | import cv2 4 | import imutils 5 | import numpy as np 6 | 7 | bg = None 8 | 9 | train_transforms = transforms.Compose([ 10 | transforms.RandomCrop(crop_size=224), 11 | transforms.Normalize() 12 | ]) 13 | 14 | model = paddlex.load_model('weights/final') 15 | CLASSES = ['pause', 'up', 'down', 'left', 'right'] 16 | 17 | 18 | def run_avg(image, aWeight): 19 | global bg 20 | if bg is None: 21 | bg = image.copy().astype('float') 22 | return 23 | 24 | cv2.accumulateWeighted(image, bg, aWeight) 25 | 26 | 27 | def segment(image, threshold=25): 28 | global bg 29 | diff = cv2.absdiff(bg.astype('uint8'), image) 30 | 31 | thresholded = cv2.threshold(diff, 32 | threshold, 33 | 255, 34 | cv2.THRESH_BINARY)[1] 35 | 36 | (cnts, _) = cv2.findContours(thresholded.copy(), 37 | cv2.RETR_EXTERNAL, 38 | cv2.CHAIN_APPROX_SIMPLE) 39 | 40 | if len(cnts) == 0: 41 | return 42 | else: 43 | segmented = max(cnts, key=cv2.contourArea) 44 | return (thresholded, segmented) 45 | 46 | 47 | def main(): 48 | action = 'pause' 49 | aWeight = 0.5 50 | 51 | camera = cv2.VideoCapture(0) 52 | 53 | top, right, bottom, left = 90, 380, 285, 590 54 | 55 | num_frames = 0 56 | thresholded = None 57 | 58 | count = 0 59 | 60 | while(True): 61 | (grabbed, frame) = camera.read() 62 | if grabbed: 63 | 64 | frame = imutils.resize(frame, width=700) 65 | 66 | frame = cv2.flip(frame, 1) 67 | 68 | clone = frame.copy() 69 | 70 | (height, width) = frame.shape[:2] 71 | 72 | roi = frame[top:bottom, right:left] 73 | 74 | gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY) 75 | gray = cv2.GaussianBlur(gray, (7, 7), 0) 76 | 77 | if num_frames < 30: 78 | run_avg(gray, aWeight) 79 | else: 80 | hand = segment(gray) 81 | 82 | if hand is not None: 83 | (thresholded, segmented) = hand 84 | 85 | cv2.drawContours( 86 | clone, [segmented + (right, top)], -1, (0, 0, 255)) 87 | 88 | cv2.rectangle(clone, (left, top), (right, bottom), (0, 255, 0), 2) 89 | 90 | num_frames += 1 91 | 92 | cv2.imshow('Video Feed', clone) 93 | 94 | if not thresholded is None: 95 | 96 | input_im = cv2.merge( 97 | [thresholded, thresholded, thresholded]) 98 | result = model.predict( 99 | input_im, topk=5, transforms=train_transforms) 100 | action = result[0]['category'] 101 | cv2.putText(input_im, action, (0, 20), 102 | cv2.FONT_HERSHEY_PLAIN, 2.0, (0, 255, 0), 2) 103 | 104 | layout = np.zeros(input_im.shape) 105 | final = [] 106 | for clas in CLASSES: 107 | for v in result: 108 | if v['category'] == clas: 109 | final.append(v['score']) 110 | break 111 | 112 | for (i, score) in enumerate(final): 113 | # construct the label text 114 | text = "{}: {:.2f}%".format(CLASSES[i], score * 100) 115 | 116 | w = int(score * 300) 117 | cv2.rectangle(layout, (7, (i * 35) + 5), 118 | (w, (i * 35) + 35), (0, 0, 255), -1) 119 | cv2.putText(layout, text, (10, (i * 35) + 23), 120 | cv2.FONT_HERSHEY_SIMPLEX, 0.45, 121 | (255, 255, 255), 2) 122 | 123 | cv2.imshow('Thesholded', np.vstack([input_im, layout])) 124 | 125 | keypress = cv2.waitKey(1) & 0xFF 126 | 127 | if keypress == ord('q'): 128 | break 129 | else: 130 | camera.release() 131 | break 132 | 133 | 134 | main() 135 | cv2.destroyAllWindows() 136 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PaddlePaddle实现手势识别玩转吃豆豆！ 2 | 3 | ![](https://ai-studio-static-online.cdn.bcebos.com/2207b0c19f2944cd997f003f3ddb8b8fc735d852ba584d1dadb12bb5c94d0697) 4 | 5 | 6 | 7 | ## 文章目录： 8 | 9 | ### 1. 手势数据采集 10 | ### 2. PaddleX训练模型 11 | ### 3. 测试手势识别模型 12 | ### 4. 测试游戏种手势控制 13 | ### 5. 大功告成~ 14 | 15 | 16 | ```python 17 | # 解压代码 18 | !unzip /home/aistudio/data/data41298/code.zip -d /home/aistudio/work/ 19 | ``` 20 | 21 | 22 | ```python 23 | # !pip install paddlex 24 | # !pip install imgaug 25 | ``` 26 | 27 | 拳头表示向下走： 28 | 29 | ![](https://ai-studio-static-online.cdn.bcebos.com/968be5f9b32f4840a0c3d7d4fd477359a6a41afec8cb400f81557827ae65b5de) 30 | 31 | 手掌表示向上走： 32 | 33 | ![](https://ai-studio-static-online.cdn.bcebos.com/a380aea8ff944a9fbc6f3677fb90b418aa8e05fdf6c1419495e1ad8cd49aa613) 34 | 35 | 下面两个分别是向左和向右： 36 | 37 | ![](https://ai-studio-static-online.cdn.bcebos.com/3b1a1c71eb9941e7ba41cc11cc3444f3994c17b8373f4d81b7ec7d8b18013e8f) 38 | ![](https://ai-studio-static-online.cdn.bcebos.com/0155857100374323bf167f0fdb90973e1ef9b0f4880d408080e5e83a1d740397) 39 | 40 | 空白表示按位不动： 41 | 42 | ![](https://ai-studio-static-online.cdn.bcebos.com/f0d252786c2a428abc721c786b53d0244ca803b928f541dc8b2cbb5085f204b2) 43 | 44 | 45 | 46 | ```python 47 | # 设置工作路径 48 | import os 49 | os.chdir('/home/aistudio/work/Pacman-master/') 50 | ``` 51 | 52 | ## 1. 手势数据采集： 53 | 54 | 这一步需要在本地运行collect文件夹下PalmTracker.py文件进行手势数据采集； 55 | 56 | 运行该程序时会打开摄像头，在指定区域做出手势，按s保存； 57 | 58 | ![](https://ai-studio-static-online.cdn.bcebos.com/06c664b20f794a4da1d3cffc5c57aad616dc037a5c8542fa802c879bb67fdac9) 59 | 60 | 61 | 62 | ```python 63 | # !python collect/PalmTracker.py 64 | ``` 65 | 66 | collect data game.py pacman.py test.jpg utils.py 67 | config.py demo.py images src tools weights 68 | 69 | 70 | ## 2. PaddleX训练模型 71 | 72 | 这一步使用PaddleX提供的ResNet18进行训练； 73 | 74 | 预训练模型使用在'IMAGENET'上训练的权重，PaddleX选择参数 pretrain_weights='IMAGENET' 即可； 75 | 76 | 我这里每种手势共收集了40张左右，训练结果准确率在93%以上； 77 | 78 | ### 2.1 定义数据集 79 | 80 | 81 | ```python 82 | from paddlex.cls import transforms 83 | import os 84 | import cv2 85 | import numpy as np 86 | import paddlex as pdx 87 | import imgaug.augmenters as iaa 88 | 89 | base = './data' 90 | 91 | with open(os.path.join('train_list.txt'), 'w') as f: 92 | for i, cls_fold in enumerate(os.listdir(base)): 93 | cls_base = os.path.join(base, cls_fold) 94 | files = os.listdir(cls_base) 95 | print('{} train num:'.format(cls_fold), len(files)) 96 | for pt in files: 97 | img = os.path.join(cls_fold, pt) 98 | info = img + ' ' + str(i) + '\n' 99 | f.write(info) 100 | 101 | with open(os.path.join('labels.txt'), 'w') as f: 102 | for i, cls_fold in enumerate(os.listdir(base)): 103 | f.write(cls_fold+'\n') 104 | 105 | train_transforms = transforms.Compose([ 106 | transforms.ResizeByShort(short_size=224), 107 | iaa.AddToBrightness((-30, 30)), 108 | iaa.pillike.EnhanceSharpness(), 109 | iaa.LinearContrast((0.8, 1.2)), 110 | transforms.RandomRotate(rotate_range=30, prob=0.5), 111 | iaa.Dropout(p=(0, 0.1)), 112 | transforms.Normalize() 113 | ]) 114 | 115 | train_dataset = pdx.datasets.ImageNet( 116 | data_dir=base, 117 | file_list='train_list.txt', 118 | label_list='labels.txt', 119 | transforms=train_transforms, 120 | shuffle=True) 121 | ``` 122 | 123 | data41298 train num: 1 124 | 2020-07-03 10:19:56 [INFO] Starting to read file list from dataset... 125 | 2020-07-03 10:19:56 [INFO] 0 samples in file train_list.txt 126 | 127 | 128 | ### 2.2 使用ResNet18训练模型 129 | 130 | 此处训练20个epoch，初始学习率为2e-2 131 | 132 | 133 | ```python 134 | num_classes = len(train_dataset.labels) 135 | model = pdx.cls.ResNet18(num_classes=num_classes) 136 | model.train(num_epochs=20, 137 | train_dataset=train_dataset, 138 | train_batch_size=32, 139 | lr_decay_epochs=[5, 10, 15], 140 | learning_rate=2e-2, 141 | save_dir='w', 142 | log_interval_steps=5, 143 | save_interval_epochs=4) 144 | 145 | ``` 146 | 147 | ## 3 测试手势识别模型： 148 | 149 | 150 | ```python 151 | from paddlex.cls import transforms 152 | import matplotlib.pyplot as plt 153 | import paddlex 154 | import cv2 155 | import warnings 156 | 157 | warnings.filterwarnings('ignore') 158 | 159 | train_transforms = transforms.Compose([ 160 | transforms.RandomCrop(crop_size=224), 161 | transforms.Normalize() 162 | ]) 163 | 164 | model = paddlex.load_model('weights/final') 165 | im = cv2.imread('test.jpg') 166 | result = model.predict(im, topk=1) 167 | print("Predict Result:", result) 168 | 169 | %matplotlib inline 170 | plt.imshow(im) 171 | plt.show() 172 | ``` 173 | 174 | 2020-06-23 09:27:29 [INFO] Model[ResNet18] loaded. 175 | Predict Result: [{'category_id': 1, 'category': 'left', 'score': 0.9999609}] 176 | 177 | 178 | 179 | ![png](output_13_1.png) 180 | 181 | 182 | ## 4. 测试游戏中手势控制： 183 | 184 | 本地运行demo.py即可； 185 | 186 | ![](https://ai-studio-static-online.cdn.bcebos.com/39bb2143fff544e0b553e5499ec7f3f49346affa73b94f42902d2314b9d9c47d) 187 | 188 | 189 | 190 | ```python 191 | !python demo.py 192 | ``` 193 | 194 | ## 5. 大功告成 195 | 196 | 然后将该控制嵌入到游戏中即可~ 197 | 198 | 游戏代码来自：https://github.com/hbokmann/Pacman 199 | 200 | 201 | ```python 202 | !python game.py 203 | ``` 204 | 205 | ![](https://ai-studio-static-online.cdn.bcebos.com/0f06ca2879024729a5f3411a4c7fac5d370e1b7ce4754833b084b624de9187c2) 206 | 207 | 208 | ### 演示视频我放到Youtube了（因为B站审核太慢了，，，） 209 | 210 | 链接地址：[https://youtu.be/tlZT2WeaK1U](https://youtu.be/tlZT2WeaK1U) 211 | 212 | ## 更新，B站审核通过啦！ 213 | 214 | 链接地址：[https://www.bilibili.com/video/BV1xa4y1Y7Mb/](https://www.bilibili.com/video/BV1xa4y1Y7Mb/) 215 | 216 | ## 关于作者： 217 | > 北京理工大学 大二在读 218 | 219 | > 感兴趣的方向为：目标检测、人脸识别、EEG识别等 220 | 221 | > 将会定期分享一些小项目，感兴趣的朋友可以互相关注一下：[主页链接](http://aistudio.baidu.com/aistudio/personalcenter/thirdview/67156) 222 | 223 | > 也欢迎大家fork、评论交流 224 | 225 | > 作者博客主页：[https://blog.csdn.net/weixin_44936889](https://blog.csdn.net/weixin_44936889) 226 | 227 | 228 | ## 联系作者~ 229 | ![](https://ai-studio-static-online.cdn.bcebos.com/8d46082a0464465da53ba213bd875537b94dc8c7de4543e9b0fc5c9e39404bc0) 230 | 231 | -------------------------------------------------------------------------------- /train_list.txt: -------------------------------------------------------------------------------- 1 | down\saved_0000.jpg 0 2 | down\saved_0001.jpg 0 3 | down\saved_v2_0000.jpg 0 4 | down\saved_v2_0001.jpg 0 5 | down\saved_v2_0002.jpg 0 6 | down\saved_v2_0003.jpg 0 7 | down\saved_v2_0004.jpg 0 8 | down\saved_v2_0005.jpg 0 9 | down\saved_v2_0006.jpg 0 10 | down\saved_v2_0007.jpg 0 11 | down\saved_v2_0008.jpg 0 12 | down\saved_v2_0009.jpg 0 13 | down\saved_v2_0010.jpg 0 14 | down\saved_v2_0011.jpg 0 15 | down\saved_v2_0012.jpg 0 16 | down\saved_v2_0013.jpg 0 17 | down\saved_v2_0014.jpg 0 18 | down\saved_v2_0015.jpg 0 19 | down\saved_v2_0016.jpg 0 20 | down\saved_v2_0017.jpg 0 21 | down\saved_v2_0018.jpg 0 22 | down\saved_v2_0019.jpg 0 23 | down\saved_v2_0020.jpg 0 24 | down\saved_v2_0021.jpg 0 25 | down\saved_v2_0022.jpg 0 26 | down\saved_v2_0023.jpg 0 27 | down\saved_v2_0024.jpg 0 28 | down\saved_v2_0025.jpg 0 29 | down\saved_v2_0026.jpg 0 30 | down\saved_v2_0027.jpg 0 31 | down\saved_v2_0028.jpg 0 32 | down\saved_v2_0029.jpg 0 33 | down\saved_v2_0030.jpg 0 34 | down\saved_v2_0031.jpg 0 35 | down\saved_v2_0032.jpg 0 36 | down\saved_v2_0033.jpg 0 37 | down\saved_v2_0034.jpg 0 38 | down\saved_v2_0035.jpg 0 39 | down\saved_v2_0036.jpg 0 40 | down\saved_v2_0037.jpg 0 41 | down\saved_v2_0038.jpg 0 42 | down\saved_v2_0039.jpg 0 43 | down\saved_v2_0040.jpg 0 44 | down\saved_v2_0041.jpg 0 45 | down\saved_v2_0042.jpg 0 46 | down\saved_v2_0043.jpg 0 47 | down\saved_v2_0044.jpg 0 48 | down\saved_v2_0045.jpg 0 49 | down\saved_v2_0046.jpg 0 50 | down\saved_v2_0047.jpg 0 51 | down\saved_v2_0048.jpg 0 52 | down\saved_v2_0049.jpg 0 53 | down\saved_v2_0050.jpg 0 54 | down\saved_v2_0051.jpg 0 55 | down\saved_v2_0052.jpg 0 56 | down\saved_v2_0053.jpg 0 57 | down\saved_v2_0054.jpg 0 58 | down\saved_v2_0055.jpg 0 59 | down\saved_v2_0056.jpg 0 60 | down\saved_v2_0057.jpg 0 61 | down\saved_v2_0058.jpg 0 62 | down\saved_v2_0059.jpg 0 63 | down\saved_v2_0060.jpg 0 64 | down\saved_v2_0061.jpg 0 65 | down\saved_v2_0062.jpg 0 66 | down\saved_v2_0063.jpg 0 67 | down\saved_v2_0064.jpg 0 68 | left\saved_0001.jpg 1 69 | left\saved_v1_0000.jpg 1 70 | left\saved_v1_0001.jpg 1 71 | left\saved_v1_0002.jpg 1 72 | left\saved_v1_0003.jpg 1 73 | left\saved_v1_0004.jpg 1 74 | left\saved_v1_0005.jpg 1 75 | left\saved_v1_0006.jpg 1 76 | left\saved_v1_0007.jpg 1 77 | left\saved_v1_0008.jpg 1 78 | left\saved_v1_0009.jpg 1 79 | left\saved_v1_0010.jpg 1 80 | left\saved_v1_0011.jpg 1 81 | left\saved_v1_0012.jpg 1 82 | left\saved_v1_0013.jpg 1 83 | left\saved_v1_0014.jpg 1 84 | left\saved_v1_0015.jpg 1 85 | left\saved_v1_0016.jpg 1 86 | left\saved_v1_0017.jpg 1 87 | left\saved_v1_0018.jpg 1 88 | left\saved_v1_0019.jpg 1 89 | left\saved_v1_0020.jpg 1 90 | left\saved_v1_0021.jpg 1 91 | left\saved_v1_0022.jpg 1 92 | left\saved_v2_0000.jpg 1 93 | left\saved_v2_0001.jpg 1 94 | left\saved_v2_0002.jpg 1 95 | left\saved_v2_0003.jpg 1 96 | left\saved_v2_0004.jpg 1 97 | left\saved_v2_0005.jpg 1 98 | left\saved_v2_0006.jpg 1 99 | left\saved_v2_0007.jpg 1 100 | left\saved_v2_0008.jpg 1 101 | left\saved_v2_0009.jpg 1 102 | left\saved_v2_0010.jpg 1 103 | left\saved_v2_0011.jpg 1 104 | left\saved_v2_0012.jpg 1 105 | left\saved_v2_0013.jpg 1 106 | left\saved_v2_0014.jpg 1 107 | left\saved_v2_0015.jpg 1 108 | left\saved_v2_0016.jpg 1 109 | left\saved_v2_0017.jpg 1 110 | left\saved_v2_0018.jpg 1 111 | left\saved_v2_0019.jpg 1 112 | left\saved_v2_0020.jpg 1 113 | left\saved_v2_0021.jpg 1 114 | left\saved_v2_0022.jpg 1 115 | left\saved_v2_0023.jpg 1 116 | left\saved_v2_0024.jpg 1 117 | left\saved_v2_0025.jpg 1 118 | left\saved_v2_0026.jpg 1 119 | left\saved_v2_0027.jpg 1 120 | left\saved_v2_0028.jpg 1 121 | left\saved_v2_0029.jpg 1 122 | left\saved_v2_0030.jpg 1 123 | left\saved_v2_0031.jpg 1 124 | pause\saved_0000.jpg 2 125 | pause\saved_0001.jpg 2 126 | pause\saved_v2_0000.jpg 2 127 | pause\saved_v2_0001.jpg 2 128 | pause\saved_v2_0002.jpg 2 129 | pause\saved_v2_0003.jpg 2 130 | pause\saved_v2_0004.jpg 2 131 | pause\saved_v2_0005.jpg 2 132 | pause\saved_v2_0006.jpg 2 133 | pause\saved_v2_0007.jpg 2 134 | pause\saved_v2_0008.jpg 2 135 | pause\saved_v2_0009.jpg 2 136 | pause\saved_v2_0010.jpg 2 137 | pause\saved_v2_0011.jpg 2 138 | pause\saved_v2_0012.jpg 2 139 | pause\saved_v2_0013.jpg 2 140 | pause\saved_v2_0014.jpg 2 141 | pause\saved_v2_0015.jpg 2 142 | pause\saved_v2_0016.jpg 2 143 | pause\saved_v2_0017.jpg 2 144 | pause\saved_v2_0018.jpg 2 145 | pause\saved_v2_0019.jpg 2 146 | right\saved_v2_0000.jpg 3 147 | right\saved_v2_0001.jpg 3 148 | right\saved_v2_0002.jpg 3 149 | right\saved_v2_0003.jpg 3 150 | right\saved_v2_0004.jpg 3 151 | right\saved_v2_0005.jpg 3 152 | right\saved_v2_0006.jpg 3 153 | right\saved_v2_0007.jpg 3 154 | right\saved_v2_0008.jpg 3 155 | right\saved_v2_0009.jpg 3 156 | right\saved_v2_0010.jpg 3 157 | right\saved_v2_0011.jpg 3 158 | right\saved_v2_0012.jpg 3 159 | right\saved_v2_0013.jpg 3 160 | right\saved_v2_0014.jpg 3 161 | right\saved_v2_0015.jpg 3 162 | right\saved_v2_0016.jpg 3 163 | right\saved_v2_0017.jpg 3 164 | right\saved_v2_0018.jpg 3 165 | right\saved_v2_0019.jpg 3 166 | right\saved_v2_0020.jpg 3 167 | right\saved_v2_0021.jpg 3 168 | right\saved_v2_0022.jpg 3 169 | right\saved_v2_0023.jpg 3 170 | right\saved_v2_0024.jpg 3 171 | right\saved_v2_0025.jpg 3 172 | right\saved_v2_0026.jpg 3 173 | right\saved_v2_0027.jpg 3 174 | right\saved_v2_0028.jpg 3 175 | right\saved_v2_0029.jpg 3 176 | right\saved_v2_0030.jpg 3 177 | right\saved_v2_0031.jpg 3 178 | right\saved_v2_0032.jpg 3 179 | right\saved_v2_0033.jpg 3 180 | right\saved_v2_0034.jpg 3 181 | right\saved_v2_0035.jpg 3 182 | right\saved_v2_0036.jpg 3 183 | right\saved_v2_0037.jpg 3 184 | right\saved_v2_0038.jpg 3 185 | right\saved_v2_0039.jpg 3 186 | right\saved_v2_0040.jpg 3 187 | up\saved_0000.jpg 4 188 | up\saved_0001.jpg 4 189 | up\saved_v1_0000.jpg 4 190 | up\saved_v1_0001.jpg 4 191 | up\saved_v1_0002.jpg 4 192 | up\saved_v1_0003.jpg 4 193 | up\saved_v1_0004.jpg 4 194 | up\saved_v1_0005.jpg 4 195 | up\saved_v1_0006.jpg 4 196 | up\saved_v1_0007.jpg 4 197 | up\saved_v1_0008.jpg 4 198 | up\saved_v1_0009.jpg 4 199 | up\saved_v1_0010.jpg 4 200 | up\saved_v1_0011.jpg 4 201 | up\saved_v1_0012.jpg 4 202 | up\saved_v1_0013.jpg 4 203 | up\saved_v1_0014.jpg 4 204 | up\saved_v1_0015.jpg 4 205 | up\saved_v1_0016.jpg 4 206 | up\saved_v1_0017.jpg 4 207 | up\saved_v1_0018.jpg 4 208 | up\saved_v1_0019.jpg 4 209 | up\saved_v1_0020.jpg 4 210 | up\saved_v1_0021.jpg 4 211 | up\saved_v1_0022.jpg 4 212 | up\saved_v1_0023.jpg 4 213 | up\saved_v1_0024.jpg 4 214 | up\saved_v1_0025.jpg 4 215 | up\saved_v1_0026.jpg 4 216 | up\saved_v1_0027.jpg 4 217 | up\saved_v1_0028.jpg 4 218 | up\saved_v1_0029.jpg 4 219 | up\saved_v1_0030.jpg 4 220 | up\saved_v1_0031.jpg 4 221 | -------------------------------------------------------------------------------- /game.py: -------------------------------------------------------------------------------- 1 | 2 | import cv2 3 | import paddlex 4 | from paddlex.cls import transforms 5 | from utils import * 6 | from config import * 7 | import imutils 8 | import numpy as np 9 | 10 | train_transforms = transforms.Compose([ 11 | transforms.RandomCrop(crop_size=224), 12 | transforms.Normalize() 13 | ]) 14 | 15 | bg = None 16 | model = paddlex.load_model('weights/final') 17 | pygame.display.set_icon(Trollicon) 18 | 19 | 20 | def process_gesture(thresholded): 21 | input_im = cv2.merge( 22 | [thresholded, thresholded, thresholded]) 23 | result = model.predict( 24 | input_im, topk=5, transforms=train_transforms) 25 | gesture = result[0]['category'] 26 | cv2.putText(input_im, gesture, (0, 20), 27 | cv2.FONT_HERSHEY_PLAIN, 2.0, (0, 255, 0), 2) 28 | 29 | layout = np.zeros(input_im.shape) 30 | final = [] 31 | for clas in CLASSES: 32 | for v in result: 33 | if v['category'] == clas: 34 | final.append(v['score']) 35 | break 36 | 37 | for (i, score) in enumerate(final): 38 | # construct the label text 39 | text = "{}: {:.2f}%".format(CLASSES[i], score * 100) 40 | 41 | w = int(score * 300) 42 | cv2.rectangle(layout, (7, (i * 35) + 5), 43 | (w, (i * 35) + 35), (0, 0, 255), -1) 44 | cv2.putText(layout, text, (10, (i * 35) + 23), 45 | cv2.FONT_HERSHEY_SIMPLEX, 0.45, 46 | (255, 255, 255), 2) 47 | 48 | return gesture, input_im, layout 49 | 50 | 51 | def run_avg(image, aWeight): 52 | global bg 53 | if bg is None: 54 | bg = image.copy().astype('float') 55 | return 56 | 57 | cv2.accumulateWeighted(image, bg, aWeight) 58 | 59 | 60 | def segment(image, threshold=25): 61 | global bg 62 | diff = cv2.absdiff(bg.astype('uint8'), image) 63 | 64 | thresholded = cv2.threshold(diff, 65 | threshold, 66 | 255, 67 | cv2.THRESH_BINARY)[1] 68 | 69 | (cnts, _) = cv2.findContours(thresholded.copy(), 70 | cv2.RETR_EXTERNAL, 71 | cv2.CHAIN_APPROX_SIMPLE) 72 | 73 | if len(cnts) == 0: 74 | return 75 | else: 76 | segmented = max(cnts, key=cv2.contourArea) 77 | return (thresholded, segmented) 78 | 79 | 80 | def setupRoomOne(all_sprites_list): 81 | wall_list = pygame.sprite.RenderPlain() 82 | 83 | for item in walls: 84 | wall = Wall(item[0], item[1], item[2], item[3], blue) 85 | wall_list.add(wall) 86 | all_sprites_list.add(wall) 87 | 88 | return wall_list 89 | 90 | 91 | def setupGate(all_sprites_list): 92 | gate = pygame.sprite.RenderPlain() 93 | gate.add(Wall(282, 242, 42, 2, white)) 94 | all_sprites_list.add(gate) 95 | return gate 96 | 97 | 98 | def startGame(): 99 | 100 | all_sprites_list = pygame.sprite.RenderPlain() 101 | block_list = pygame.sprite.RenderPlain() 102 | monsta_list = pygame.sprite.RenderPlain() 103 | pacman_collide = pygame.sprite.RenderPlain() 104 | wall_list = setupRoomOne(all_sprites_list) 105 | gate = setupGate(all_sprites_list) 106 | 107 | p_turn = 0 108 | p_steps = 0 109 | b_turn = 0 110 | b_steps = 0 111 | i_turn = 0 112 | i_steps = 0 113 | c_turn = 0 114 | c_steps = 0 115 | 116 | top, right, bottom, left = 90, 360, 285, 580 117 | 118 | Pacman = Player(w, p_h, 'images/Trollman.png') 119 | all_sprites_list.add(Pacman) 120 | pacman_collide.add(Pacman) 121 | 122 | Blinky = Ghost(w, b_h, 'images/Blinky.png') 123 | monsta_list.add(Blinky) 124 | all_sprites_list.add(Blinky) 125 | 126 | Pinky = Ghost(w, m_h, 'images/Pinky.png') 127 | monsta_list.add(Pinky) 128 | all_sprites_list.add(Pinky) 129 | 130 | Inky = Ghost(i_w, m_h, 'images/Inky.png') 131 | monsta_list.add(Inky) 132 | all_sprites_list.add(Inky) 133 | 134 | Clyde = Ghost(c_w, m_h, 'images/Clyde.png') 135 | monsta_list.add(Clyde) 136 | all_sprites_list.add(Clyde) 137 | 138 | for row in range(19): 139 | for column in range(19): 140 | if (row == 7 or row == 8) and (column == 8 or column == 9 or column == 10): 141 | continue 142 | else: 143 | block = Block(yellow, 4, 4) 144 | 145 | # Set a random location for the block 146 | block.rect.x = (30*column+6)+26 147 | block.rect.y = (30*row+6)+26 148 | 149 | b_collide = pygame.sprite.spritecollide( 150 | block, wall_list, False) 151 | p_collide = pygame.sprite.spritecollide( 152 | block, pacman_collide, False) 153 | if b_collide: 154 | continue 155 | elif p_collide: 156 | continue 157 | else: 158 | # Add the block to the list of objects 159 | block_list.add(block) 160 | all_sprites_list.add(block) 161 | 162 | bll = len(block_list) 163 | score = 0 164 | num_frames = 0 165 | i = 0 166 | gesture = 'pause' 167 | thresholded = None 168 | 169 | bk_image = pygame.image.load("images/bk.jpg") 170 | 171 | camera = cv2.VideoCapture(0) 172 | 173 | while True: 174 | i += 1 175 | screen.blit(bk_image,(0,0)) #对齐的坐标 176 | 177 | grabbed, frame = camera.read() 178 | if not grabbed: 179 | break 180 | 181 | frame = imutils.resize(frame, width=600) 182 | 183 | frame = cv2.flip(frame, 1) 184 | 185 | clone = frame.copy() 186 | 187 | (height, width) = frame.shape[:2] 188 | 189 | roi = frame[top:bottom, right:left] 190 | 191 | gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY) 192 | gray = cv2.GaussianBlur(gray, (7, 7), 0) 193 | 194 | if num_frames < 10: 195 | run_avg(gray, 0.5) 196 | 197 | else: 198 | hand = segment(gray) 199 | 200 | if hand is not None: 201 | (thresholded, segmented) = hand 202 | cv2.drawContours( 203 | clone, [segmented + (right, top)], -1, (0, 0, 255)) 204 | cv2.rectangle(clone, (left, top), (right, bottom), (0, 255, 0), 2) 205 | num_frames += 1 206 | 207 | if not thresholded is None: 208 | gesture, input_im, layout = process_gesture(thresholded) 209 | cv2.imshow('Thesholded', np.vstack([input_im, layout])) 210 | cv2.imshow('Video Feed', clone) 211 | 212 | for event in pygame.event.get(): 213 | if event.type == pygame.QUIT: 214 | pygame.quit() 215 | 216 | if gesture == 'left': 217 | Pacman.changespeed(-30, 0) 218 | if gesture == 'right': 219 | Pacman.changespeed(30, 0) 220 | if gesture == 'up': 221 | Pacman.changespeed(0, -30) 222 | if gesture == 'down': 223 | Pacman.changespeed(0, 30) 224 | if gesture == 'pause': 225 | Pacman.changespeed(0, 0) 226 | 227 | Pacman.update(wall_list, gate) 228 | if i % 2 == 0: 229 | returned = Pinky.changespeed( 230 | Pinky_directions, False, p_turn, p_steps, pl) 231 | p_turn = returned[0] 232 | p_steps = returned[1] 233 | Pinky.changespeed(Pinky_directions, False, p_turn, p_steps, pl) 234 | Pinky.update(wall_list, False) 235 | 236 | returned = Blinky.changespeed( 237 | Blinky_directions, False, b_turn, b_steps, bl) 238 | b_turn = returned[0] 239 | b_steps = returned[1] 240 | Blinky.changespeed(Blinky_directions, False, b_turn, b_steps, bl) 241 | Blinky.update(wall_list, False) 242 | print(returned) 243 | returned = Inky.changespeed( 244 | Inky_directions, False, i_turn, i_steps, il) 245 | i_turn = returned[0] 246 | i_steps = returned[1] 247 | Inky.changespeed(Inky_directions, False, i_turn, i_steps, il) 248 | Inky.update(wall_list, False) 249 | 250 | returned = Clyde.changespeed( 251 | Clyde_directions, 'clyde', c_turn, c_steps, cl) 252 | c_turn = returned[0] 253 | c_steps = returned[1] 254 | Clyde.changespeed(Clyde_directions, 'clyde', c_turn, c_steps, cl) 255 | Clyde.update(wall_list, False) 256 | 257 | # See if the Pacman block has collided with anything. 258 | blocks_hit_list = pygame.sprite.spritecollide(Pacman, block_list, True) 259 | 260 | # Check the list of collisions. 261 | if len(blocks_hit_list) > 0: 262 | score += len(blocks_hit_list) 263 | 264 | # screen.fill(black) 265 | 266 | wall_list.draw(screen) 267 | gate.draw(screen) 268 | all_sprites_list.draw(screen) 269 | monsta_list.draw(screen) 270 | 271 | text = font.render('Score: '+str(score)+'/'+str(bll), True, red) 272 | screen.blit(text, [10, 10]) 273 | 274 | if score == bll: 275 | doNext('Congratulations, you won!', 145, all_sprites_list, 276 | block_list, monsta_list, pacman_collide, wall_list, gate) 277 | 278 | monsta_hit_list = pygame.sprite.spritecollide( 279 | Pacman, monsta_list, False) 280 | 281 | if monsta_hit_list: 282 | doNext('Game Over', 235, all_sprites_list, block_list, 283 | monsta_list, pacman_collide, wall_list, gate) 284 | 285 | # ALL CODE TO DRAW SHOULD GO ABOVE THIS COMMENT 286 | try: 287 | pygame.display.flip() 288 | except Exception as e: 289 | return 290 | clock.tick(10) 291 | 292 | 293 | def doNext(message, left, all_sprites_list, block_list, monsta_list, pacman_collide, wall_list, gate): 294 | while True: 295 | try: 296 | # ALL EVENT PROCESSING SHOULD GO BELOW THIS COMMENT 297 | for event in pygame.event.get(): 298 | if event.type == pygame.QUIT: 299 | pygame.quit() 300 | if event.type == pygame.KEYDOWN: 301 | if event.key == pygame.K_ESCAPE: 302 | pygame.quit() 303 | if event.key == pygame.K_RETURN: 304 | del all_sprites_list 305 | del block_list 306 | del monsta_list 307 | del pacman_collide 308 | del wall_list 309 | del gate 310 | startGame() 311 | 312 | # Grey background 313 | w = pygame.Surface((400, 200)) # the size of your rect 314 | w.set_alpha(10) # alpha level 315 | w.fill((128, 128, 128)) # this fills the entire surface 316 | screen.blit(w, (100, 200)) # (0,0) are the top-left coordinates 317 | 318 | #Won or lost 319 | text1 = font.render(message, True, white) 320 | screen.blit(text1, [left, 233]) 321 | 322 | text2 = font.render('To play again, press ENTER.', True, white) 323 | screen.blit(text2, [135, 303]) 324 | text3 = font.render('To quit, press ESCAPE.', True, white) 325 | screen.blit(text3, [165, 333]) 326 | 327 | pygame.display.flip() 328 | 329 | clock.tick(10) 330 | 331 | except Exception as e: 332 | break 333 | 334 | 335 | if __name__ == '__main__': 336 | 337 | pl = len(Pinky_directions)-1 338 | bl = len(Blinky_directions)-1 339 | il = len(Inky_directions)-1 340 | cl = len(Clyde_directions)-1 341 | 342 | pygame.init() 343 | 344 | screen = pygame.display.set_mode([606, 606]) 345 | 346 | pygame.display.set_caption('Pacman') 347 | 348 | # background = pygame.Surface(screen.get_size()) 349 | 350 | # background = background.convert() 351 | 352 | # background.fill(black) 353 | 354 | clock = pygame.time.Clock() 355 | 356 | pygame.font.init() 357 | font = pygame.font.Font('src/freesansbold.ttf', 24) 358 | 359 | startGame() 360 | 361 | pygame.quit() 362 | --------------------------------------------------------------------------------