├── rec_inference_model ├── .success ├── __model__ ├── __params__ └── model.yml ├── README.md └── game.py /rec_inference_model/.success: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rec_inference_model/__model__: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/Play-Tetris-Using-hand-gesture-recognition/HEAD/rec_inference_model/__model__ -------------------------------------------------------------------------------- /rec_inference_model/__params__: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Sharpiless/Play-Tetris-Using-hand-gesture-recognition/HEAD/rec_inference_model/__params__ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Play-Tetris-Using-hand-gesture-recognition 2 | # 使用手势识别算法玩俄罗斯方块 3 | 4 | ## 源码地址： 5 | 6 | [https://github.com/Sharpiless/Play-Tetris-Using-hand-gesture-recognition](https://github.com/Sharpiless/Play-Tetris-Using-hand-gesture-recognition) 7 | 8 | ## 演示视频： 9 | 10 | [https://www.bilibili.com/video/BV1Ct4y1r7RL](https://www.bilibili.com/video/BV1Ct4y1r7RL) 11 | 12 | ![在这里插入图片描述](https://img-blog.csdnimg.cn/20210330112632699.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkzNjg4OQ==,size_16,color_FFFFFF,t_70) 13 | 14 | ## 我的公众号： 15 | 16 | ![在这里插入图片描述](https://img-blog.csdnimg.cn/2021033011081636.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NDkzNjg4OQ==,size_16,color_FFFFFF,t_70) 17 | -------------------------------------------------------------------------------- /rec_inference_model/model.yml: -------------------------------------------------------------------------------- 1 | Model: MobileNetV3_small_ssld 2 | Transforms: 3 | - RandomCrop: 4 | crop_size: 224 5 | lower_ratio: 0.75 6 | lower_scale: 0.08 7 | upper_ratio: 1.3333333333333333 8 | - Normalize: 9 | mean: 10 | - 0.485 11 | - 0.456 12 | - 0.406 13 | std: 14 | - 0.229 15 | - 0.224 16 | - 0.225 17 | TransformsMode: RGB 18 | _Attributes: 19 | eval_metrics: 20 | acc1: 0.9161490683229814 21 | fixed_input_shape: null 22 | labels: 23 | - pause 24 | - down 25 | - left 26 | - right 27 | - up 28 | model_type: classifier 29 | num_classes: 5 30 | _ModelInputsOutputs: 31 | test_inputs: 32 | - - image 33 | - image 34 | test_outputs: 35 | - - predict 36 | - softmax_0.tmp_0 37 | _init_params: 38 | num_classes: 5 39 | completed_epochs: 0 40 | status: Infer 41 | version: 1.3.1 42 | -------------------------------------------------------------------------------- /game.py: -------------------------------------------------------------------------------- 1 | import pygame 2 | import random 3 | import cv2 4 | import imutils 5 | from paddlex.deploy import Predictor 6 | 7 | """ 8 | 10 x 20 square grid 9 | shapes: S, Z, I, O, J, L, T 10 | represented in order by 0 - 6 11 | """ 12 | 13 | pygame.font.init() 14 | 15 | # GLOBALS VARS 16 | s_width = 800 17 | s_height = 700 18 | play_width = 300 # meaning 300 // 10 = 30 width per block 19 | play_height = 600 # meaning 600 // 20 = 20 height per blo ck 20 | block_size = 30 21 | 22 | top_left_x = (s_width - play_width) // 2 23 | top_left_y = s_height - play_height 24 | 25 | 26 | # SHAPE FORMATS 27 | 28 | S = [['.....', 29 | '.....', 30 | '..00.', 31 | '.00..', 32 | '.....'], 33 | ['.....', 34 | '..0..', 35 | '..00.', 36 | '...0.', 37 | '.....']] 38 | 39 | Z = [['.....', 40 | '.....', 41 | '.00..', 42 | '..00.', 43 | '.....'], 44 | ['.....', 45 | '..0..', 46 | '.00..', 47 | '.0...', 48 | '.....']] 49 | 50 | I = [['..0..', 51 | '..0..', 52 | '..0..', 53 | '..0..', 54 | '.....'], 55 | ['.....', 56 | '0000.', 57 | '.....', 58 | '.....', 59 | '.....']] 60 | 61 | O = [['.....', 62 | '.....', 63 | '.00..', 64 | '.00..', 65 | '.....']] 66 | 67 | J = [['.....', 68 | '.0...', 69 | '.000.', 70 | '.....', 71 | '.....'], 72 | ['.....', 73 | '..00.', 74 | '..0..', 75 | '..0..', 76 | '.....'], 77 | ['.....', 78 | '.....', 79 | '.000.', 80 | '...0.', 81 | '.....'], 82 | ['.....', 83 | '..0..', 84 | '..0..', 85 | '.00..', 86 | '.....']] 87 | 88 | L = [['.....', 89 | '...0.', 90 | '.000.', 91 | '.....', 92 | '.....'], 93 | ['.....', 94 | '..0..', 95 | '..0..', 96 | '..00.', 97 | '.....'], 98 | ['.....', 99 | '.....', 100 | '.000.', 101 | '.0...', 102 | '.....'], 103 | ['.....', 104 | '.00..', 105 | '..0..', 106 | '..0..', 107 | '.....']] 108 | 109 | T = [['.....', 110 | '..0..', 111 | '.000.', 112 | '.....', 113 | '.....'], 114 | ['.....', 115 | '..0..', 116 | '..00.', 117 | '..0..', 118 | '.....'], 119 | ['.....', 120 | '.....', 121 | '.000.', 122 | '..0..', 123 | '.....'], 124 | ['.....', 125 | '..0..', 126 | '.00..', 127 | '..0..', 128 | '.....']] 129 | 130 | shapes = [S, Z, I, O, J, L, T] 131 | shape_colors = [(0, 255, 0), (255, 0, 0), (0, 255, 255), 132 | (255, 255, 0), (255, 165, 0), (0, 0, 255), (128, 0, 128)] 133 | # index 0 - 6 represent shape 134 | 135 | 136 | class Piece(object): 137 | rows = 20 # y 138 | columns = 10 # x 139 | # 方块类 140 | def __init__(self, column, row, shape): 141 | self.x = column 142 | self.y = row 143 | self.shape = shape 144 | self.color = shape_colors[shapes.index(shape)] 145 | self.rotation = 0 # number from 0-3 146 | 147 | 148 | def create_grid(locked_positions={}): 149 | # 绘制网格 150 | grid = [[(0, 0, 0) for x in range(10)] for x in range(20)] 151 | 152 | for i in range(len(grid)): 153 | for j in range(len(grid[i])): 154 | if (j, i) in locked_positions: 155 | c = locked_positions[(j, i)] 156 | grid[i][j] = c 157 | return grid 158 | 159 | 160 | def convert_shape_format(shape): 161 | # 生成方块 162 | positions = [] 163 | format = shape.shape[shape.rotation % len(shape.shape)] 164 | 165 | for i, line in enumerate(format): 166 | row = list(line) 167 | for j, column in enumerate(row): 168 | if column == '0': 169 | positions.append((shape.x + j, shape.y + i)) 170 | 171 | for i, pos in enumerate(positions): 172 | positions[i] = (pos[0] - 2, pos[1] - 4) 173 | 174 | return positions 175 | 176 | 177 | def valid_space(shape, grid): 178 | accepted_positions = [[(j, i) for j in range( 179 | 10) if grid[i][j] == (0, 0, 0)] for i in range(20)] 180 | accepted_positions = [j for sub in accepted_positions for j in sub] 181 | formatted = convert_shape_format(shape) 182 | 183 | for pos in formatted: 184 | if pos not in accepted_positions: 185 | if pos[1] > -1: 186 | return False 187 | 188 | return True 189 | 190 | 191 | def check_lost(positions): 192 | for pos in positions: 193 | x, y = pos 194 | if y < 1: 195 | return True 196 | return False 197 | 198 | 199 | def get_shape(): 200 | global shapes, shape_colors 201 | 202 | return Piece(5, 0, random.choice(shapes)) 203 | 204 | 205 | def draw_text_middle(text, size, color, surface): 206 | font = pygame.font.SysFont('comicsans', size, bold=True) 207 | label = font.render(text, 1, color) 208 | 209 | surface.blit(label, (top_left_x + play_width/2 - (label.get_width() / 2), 210 | top_left_y + play_height/2 - label.get_height()/2)) 211 | 212 | 213 | def draw_grid(surface, row, col): 214 | sx = top_left_x 215 | sy = top_left_y 216 | for i in range(row): 217 | pygame.draw.line(surface, (128, 128, 128), (sx, sy + i*30), 218 | (sx + play_width, sy + i * 30)) # horizontal lines 219 | for j in range(col): 220 | pygame.draw.line(surface, (128, 128, 128), (sx + j * 30, sy), 221 | (sx + j * 30, sy + play_height)) # vertical lines 222 | 223 | 224 | def clear_rows(grid, locked): 225 | # need to see if row is clear the shift every other row above down one 226 | 227 | inc = 0 228 | for i in range(len(grid)-1, -1, -1): 229 | row = grid[i] 230 | if (0, 0, 0) not in row: 231 | inc += 1 232 | # add positions to remove from locked 233 | ind = i 234 | for j in range(len(row)): 235 | try: 236 | del locked[(j, i)] 237 | except: 238 | continue 239 | if inc > 0: 240 | for key in sorted(list(locked), key=lambda x: x[1])[::-1]: 241 | x, y = key 242 | if y < ind: 243 | newKey = (x, y + inc) 244 | locked[newKey] = locked.pop(key) 245 | return 1 246 | else: 247 | return 0 248 | 249 | def draw_next_shape(shape, surface): 250 | font = pygame.font.SysFont('comicsans', 30) 251 | label = font.render('Next Shape', 1, (255, 255, 255)) 252 | 253 | sx = top_left_x + play_width + 50 254 | sy = top_left_y + play_height/2 - 100 255 | format = shape.shape[shape.rotation % len(shape.shape)] 256 | 257 | for i, line in enumerate(format): 258 | row = list(line) 259 | for j, column in enumerate(row): 260 | if column == '0': 261 | pygame.draw.rect(surface, shape.color, 262 | (sx + j*30, sy + i*30, 30, 30), 0) 263 | 264 | surface.blit(label, (sx + 10, sy - 30)) 265 | 266 | 267 | class Button(object): 268 | # 按钮类 269 | def __init__(self, text, color, x=None, y=None, **kwargs): 270 | font = pygame.font.SysFont('comicsans', 30) 271 | self.surface = font.render(text, True, color) 272 | self.WIDTH = self.surface.get_width() 273 | self.HEIGHT = self.surface.get_height() 274 | self.x = x 275 | self.y = y 276 | 277 | def display(self): 278 | # 在主界面显示按钮 279 | win.blit(self.surface, (self.x, self.y)) 280 | 281 | def check_click(self, position): 282 | # 检查按键上是否有鼠标 283 | x_match = position[0] > self.x and position[0] < self.x + self.WIDTH 284 | y_match = position[1] > self.y and position[1] < self.y + self.HEIGHT 285 | 286 | if x_match and y_match: 287 | return True 288 | else: 289 | return False 290 | 291 | sx = top_left_x + play_width + 80 292 | play_button = Button('Play', (255, 255, 255), sx, 450) 293 | exit_button = Button('Exit', (255, 255, 255), sx, 500) 294 | pause_button = Button('Pause', (255, 255, 255), sx, 550) 295 | 296 | 297 | def draw_window(surface, score): 298 | surface.fill((0, 0, 0)) 299 | # 绘制得分 300 | font = pygame.font.SysFont('comicsans', 60) 301 | label = font.render('Score:{}'.format(score), 1, (255, 255, 255)) 302 | 303 | surface.blit(label, (top_left_x + play_width / 304 | 2 - (label.get_width() / 2), 30)) 305 | 306 | for i in range(len(grid)): 307 | for j in range(len(grid[i])): 308 | pygame.draw.rect( 309 | surface, grid[i][j], (top_left_x + j * 30, top_left_y + i * 30, 30, 30), 0) 310 | 311 | draw_grid(surface, 20, 10) 312 | pygame.draw.rect(surface, (255, 0, 0), (top_left_x, 313 | top_left_y, play_width, play_height), 5) 314 | 315 | # 显示三个按钮 316 | play_button.display() 317 | exit_button.display() 318 | pause_button.display() 319 | 320 | 321 | def main(): 322 | global grid 323 | 324 | locked_positions = {} # (x,y):(255,0,0) 325 | grid = create_grid(locked_positions) 326 | 327 | change_piece = False 328 | run = True 329 | current_piece = get_shape() 330 | next_piece = get_shape() 331 | clock = pygame.time.Clock() 332 | fall_time = 0 # 计时 333 | level_time = 0 334 | fall_speed = 0.7 # 下落速度 335 | fps_clock = pygame.time.Clock() 336 | fps = 10 337 | score = 0 # 得分 338 | width = 600 # 视频图像宽度 339 | top, right, bottom, left = 90, 360, 285, 580 # ROI（即手势区域） 340 | camera = cv2.VideoCapture(0) # 调用摄像头 341 | model = Predictor('rec_inference_model', use_gpu=False) # 初始化模型 342 | count = 0 # 计数器 343 | 344 | while run: 345 | count += 1 346 | grid = create_grid(locked_positions) 347 | fall_time += clock.get_rawtime() 348 | level_time += clock.get_rawtime() 349 | clock.tick() 350 | 351 | readable, frame = camera.read() # 读取视频帧 352 | if not readable: 353 | break 354 | 355 | frame = imutils.resize(frame, width=width) # 对摄像头图像放缩 356 | frame = cv2.flip(frame, 1) # 图像镜像处理（左右手问题） 357 | roi = frame[top:bottom, right:left].copy() # 手势位置 358 | cv2.rectangle(frame, (left, top), 359 | (right, bottom), (0, 255, 0), 2) # 绘制绿框 360 | 361 | direction = 'pause' 362 | if count % 3 == 0: 363 | pred = model.predict(roi) # 使用CNN模型进行预测 364 | direction = pred[0]['category'] 365 | cv2.putText(roi, direction, (10, 50), 366 | cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 2) 367 | cv2.imshow('roi', roi) # 显示ROI 368 | print('-[INFO] Update movement to ', direction) 369 | 370 | if direction == 'left': 371 | # 左移 372 | current_piece.x -= 1 373 | if not valid_space(current_piece, grid): 374 | current_piece.x += 1 375 | elif direction == 'right': 376 | # 右移 377 | current_piece.x += 1 378 | if not valid_space(current_piece, grid): 379 | current_piece.x -= 1 380 | elif direction == 'up' and count % 6 == 0: 381 | # 变换形状 382 | current_piece.rotation = current_piece.rotation + \ 383 | 1 % len(current_piece.shape) 384 | if not valid_space(current_piece, grid): 385 | current_piece.rotation = current_piece.rotation - \ 386 | 1 % len(current_piece.shape) 387 | 388 | if fall_time/1000 >= fall_speed: 389 | # 变速 390 | fall_time = 0 391 | current_piece.y += 1 392 | if not (valid_space(current_piece, grid)) and current_piece.y > 0: 393 | current_piece.y -= 1 394 | change_piece = True 395 | 396 | for event in pygame.event.get(): 397 | # 获取键盘相应 398 | if event.type == pygame.QUIT: 399 | # 点×退出游戏 400 | run = False 401 | pygame.display.quit() 402 | quit() 403 | 404 | shape_pos = convert_shape_format(current_piece) 405 | 406 | # add piece to the grid for drawing 407 | for i in range(len(shape_pos)): 408 | x, y = shape_pos[i] 409 | if y > -1: 410 | grid[y][x] = current_piece.color 411 | 412 | # IF PIECE HIT GROUND 413 | if change_piece: 414 | for pos in shape_pos: 415 | p = (pos[0], pos[1]) 416 | locked_positions[p] = current_piece.color 417 | current_piece = next_piece 418 | next_piece = get_shape() 419 | change_piece = False 420 | 421 | # call four times to check for multiple clear rows 422 | if clear_rows(grid, locked_positions): 423 | score += 10 424 | 425 | draw_window(win, score) 426 | draw_next_shape(next_piece, win) 427 | pygame.display.update() 428 | 429 | if check_lost(locked_positions): 430 | run = False 431 | if pygame.mouse.get_pressed()[0]: 432 | # 获取鼠标左键点击相应 433 | if play_button.check_click(pygame.mouse.get_pos()): 434 | pass 435 | if exit_button.check_click(pygame.mouse.get_pos()): 436 | run = False 437 | pygame.display.quit() 438 | quit() 439 | if pause_button.check_click(pygame.mouse.get_pos()): 440 | run = False 441 | 442 | cv2.imshow('frame', frame) 443 | # cv2.waitKey(100) 444 | fps_clock.tick(fps) # 计时器 445 | 446 | draw_text_middle("You Lost", 40, (255, 255, 255), win) 447 | pygame.display.update() 448 | pygame.time.delay(2000) 449 | 450 | 451 | def main_menu(): 452 | run = True 453 | while run: 454 | win.fill((0, 0, 0)) 455 | draw_text_middle('Press any key to begin.', 60, (255, 255, 255), win) 456 | pygame.display.update() 457 | for event in pygame.event.get(): 458 | if event.type == pygame.QUIT: 459 | run = False 460 | if event.type == pygame.KEYDOWN: 461 | main() 462 | pygame.quit() 463 | 464 | 465 | win = pygame.display.set_mode((s_width, s_height)) 466 | pygame.display.set_caption('Tetris') 467 | 468 | main_menu() # start game 469 | --------------------------------------------------------------------------------