├── .gitignore ├── Maze.py ├── QLearner.py ├── README.md ├── grading.py ├── mazeqlearning.py ├── model-based.png └── testworlds ├── world01.csv ├── world02.csv ├── world03.csv ├── world04.csv ├── world05.csv ├── world06.csv ├── world07.csv ├── world08.csv ├── world09.csv └── world10.csv /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | -------------------------------------------------------------------------------- /Maze.py: -------------------------------------------------------------------------------- 1 | """ 2 | Template for implementing Maze 3 | """ 4 | 5 | import numpy as np 6 | import random as rand 7 | 8 | class Maze(object): 9 | def __init__(self, 10 | data, 11 | reward_walk = -1, 12 | reward_obstacle = -1, 13 | reward_quicksand = -100, 14 | reward_goal = 1, 15 | random_walk_rate = 0.2, 16 | verbose = False): 17 | 18 | #TODO 19 | self.data = data 20 | self.reward_walk = reward_walk 21 | self.reward_obstacle = reward_obstacle 22 | self.reward_quicksand = reward_quicksand 23 | self.reward_goal = reward_goal 24 | self.random_walk_rate = random_walk_rate 25 | self.verbose = verbose 26 | 27 | #return the start position of the robot 28 | def get_start_pos(self): 29 | #TODO 30 | return (0,0) 31 | 32 | #return the goal position of the robot 33 | def get_goal_pos(self): 34 | #TODO 35 | return (0,0) 36 | 37 | # move the robot and report new position and reward 38 | # Note that robot cannot step into obstacles nor step out of the map 39 | # Note that robot may ignore the given action and choose a random action 40 | def move(self, oldpos, a): 41 | #TODO 42 | #newpos = oldpos 43 | newpos = (oldpos[0], oldpos[1]) 44 | reward = 0 45 | 46 | # return the new, legal location and reward 47 | return newpos, reward 48 | 49 | # print out the map 50 | def print_map(self): 51 | data = self.data 52 | print("--------------------") 53 | for row in range(0, data.shape[0]): 54 | for col in range(0, data.shape[1]): 55 | if data[row,col] == 0: # Empty space 56 | print(" ",end="") 57 | if data[row,col] == 1: # Obstacle 58 | print("X",end="") 59 | if data[row,col] == 2: # Start 60 | print("S",end="") 61 | if data[row,col] == 3: # Goal 62 | print("G",end="") 63 | if data[row,col] == 5: # Quick sand 64 | print("~",end="") 65 | print() 66 | print("--------------------") 67 | 68 | # print the map and the trail of robot 69 | def print_trail(self, trail): 70 | data = self.data 71 | trail = data.copy() 72 | for pos in trail: 73 | 74 | #check if position is valid 75 | if not ( 0 <= pos[0] < data.shape[0] 76 | and 0 <= pos[1] < data.shape[1]): 77 | print("Warning: Invalid position in trail, out of the world") 78 | return 79 | 80 | if data[pos] == 1: # Obstacle 81 | print("Warning: Invalid position in trail, step on obstacle") 82 | return 83 | 84 | #mark the trail 85 | if data[pos] == 0: # mark enter empty space 86 | trail[pos] = "." 87 | if data[pos] == 5: # make enter quicksand 88 | trail[pos] = "@" 89 | 90 | print("--------------------") 91 | for row in range(0, trail.shape[0]): 92 | for col in range(0, trail.shape[1]): 93 | if trail[row, col] == 0: # Empty space 94 | trail[row, col] = " " 95 | if trail[row, col] == 1: # Obstacle 96 | trail[row, col] = "X" 97 | if trail[row, col] == 2: # Start 98 | trail[row, col] = "S" 99 | if trail[row, col] == 3: # Goal 100 | trail[row, col] = "G" 101 | if trail[row, col] == 5: # Quick sand 102 | trail[row, col] = "~" 103 | 104 | print(trail[row, col], end="") 105 | print() 106 | print("--------------------") 107 | -------------------------------------------------------------------------------- /QLearner.py: -------------------------------------------------------------------------------- 1 | """ 2 | Template for implementing QLearner 3 | """ 4 | 5 | import numpy as np 6 | import random as rand 7 | 8 | class QLearner(object): 9 | 10 | def __init__(self, 11 | num_states=100, 12 | num_actions = 4, 13 | alpha = 0.2, 14 | gamma = 0.9, 15 | rar = 0.5, 16 | radr = 0.99, 17 | verbose = False): 18 | 19 | #TODO 20 | self.verbose = verbose 21 | self.num_actions = num_actions 22 | self.s = 0 23 | self.a = 0 24 | 25 | def querysetstate(self, s): 26 | """ 27 | @summary: Update the state without updating the Q-table 28 | @param s: The new state 29 | @returns: The selected action 30 | """ 31 | #TODO 32 | self.s = s 33 | action = rand.randint(0, self.num_actions-1) 34 | if self.verbose: print("s =", s,"a =",action) 35 | return action 36 | 37 | def query(self,s_prime,r): 38 | """ 39 | @summary: Update the Q table and return an action 40 | @param s_prime: The new state 41 | @param r: The ne state 42 | @returns: The selected action 43 | """ 44 | #TODO 45 | action = rand.randint(0, self.num_actions-1) 46 | if self.verbose: print("s =", s_prime,"a =",action,"r =",r) 47 | return action 48 | 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### 概述 2 | 在这个项目中，你会需要实现一个 Q-learning算法来解决一个增强学习问题 -- 走迷宫。 3 | 4 | 5 | ### Github Repo 6 | - 更新你的 `qlearning_robot` 目录 7 | ``` 8 | git clone https://github.com/nd009/qlearning_robot.git 9 | ``` 10 | - `Qlearner.py` 提供了实现 `QLearner` 类的模版。 11 | - `maze.py` 提供了实现`Maze` 类的模版。 12 | - `mazeqlearning.py` 利用 `QLearner` 类和 `Maze`类解决走迷宫问题 13 | - `testworlds` 目录下提供了一些迷宫可以用来测试。 14 | 15 | ## 定义迷宫问题 16 | 17 | #### 地图 18 | 我们用一个二位数组定义了整个迷宫。迷宫的纬度是 10 * 10， 每一个迷宫都存储在csv文件中，用 integer 表示每个位置的属性，具体含义如下 19 | 20 | - 0: 空地. 21 | - 1: 障碍物. 22 | - 2: 机器人的起始点. 23 | - 3: 目标终点. 24 | - 5: 陷阱. 25 | 26 | 一个迷宫 (world01.csv) 如下图所示 27 | 28 | ``` 29 | 3,0,0,0,0,0,0,0,0,0 30 | 0,0,0,0,0,0,0,0,0,0 31 | 0,0,0,0,0,0,0,0,0,0 32 | 0,0,1,1,1,1,1,0,0,0 33 | 0,5,1,0,0,0,1,0,0,0 34 | 0,5,1,0,0,0,1,0,0,0 35 | 0,0,1,0,0,0,1,0,0,0 36 | 0,0,0,0,0,0,0,0,0,0 37 | 0,0,0,0,0,0,0,0,0,0 38 | 0,0,0,0,2,0,0,0,0,0 39 | ``` 40 | 41 | 在这个例子中，机器人从最后一行的中间位置开始，目标为第0行第0列，中间连续的障碍物组成一面墙阻挡路线，同时左边有很多陷阱。 42 | 43 | #### 机器人行走 44 | 有四个可能的行为： 向上走, 向右走, 向下走, 向左走。如果机器人尝试走入陷阱，则会真的走入陷阱。如果机器人尝试走入障碍物或走出地图，则会停留在原地，但依旧算作一步。 45 | 46 | #### 随机行为 47 | 机器人有 0.2 的概率不执行指令，而是在四种行为中随机选择。 48 | 例如，如果机器人收到指令 “向上走”，会有一定的概率不往上走，而走其他方向。因此，一个 “聪明的” 机器人应该尽可能得远离陷阱。 49 | 50 | #### 目标 51 | 52 | 我们的目标是让机器人在不走入陷阱的情况下，用最少的步数从起点到达终点。 53 | 54 | ## 定义迷宫问题为 Markov 决策过程 (MDP) 55 | 在使用 QLearning 解决走迷宫问题之前，我们首先要重新定义走迷宫问题为一个 Markov 决策过程， 因为 QLearning 是用来解决 Markov 决策过程的。 56 | 57 | Markov 决策过程包含四个元素，状态，行为，模型和奖励。 58 | #### 状态 59 | `S`: 10*10 地图上的每个位置，都对应一个状态，共 100 个状态。我们可以用 0 ~ 99 来代表所有状态。 60 | #### 行为 61 | `A`: 向上走，向下走，向左走，向右走，共 4 个行为。我们可以用 0 ~ 3 来代表所有行为。 62 | #### 模型 63 | `T(s, a, s') = P(s|s, a)`: 在状态 `s`, 执行行为 `a`, 进入状态 `s'` 的概率。模型可以被地图完全定义。例如从一个格子向上走，如果四周都没有障碍物，那么进入上下左右四个格子的概率分别为 0.85, 0.05, 0.05, 0.05, 进入其他各自的概率为0 。 64 | 65 | #### 奖励 66 | `R(s)`: 进入状态 `s` 的奖励。根据我们的目标: 67 | >让机器人在不走入陷阱的情况下，用最少的步数从起点到达终点。 68 | 69 | 我们可以选择了最直接的奖励／惩罚。 70 | 71 | - **reward = -1** 如果机器人走进了一个空地。 72 | - **reward = -1** 如果机器人尝试走进障碍物，或走出地图。 73 | - **reward = -100** 如果机器人走进了陷阱。 74 | - **reward = 1** 如果机器人走到了终点。 75 | 76 | 如果你觉得选择其他的奖励函数更好得达到目标（更快收敛，更好收敛），也可以使用其他奖励函数。 77 | 78 | ## 定义迷宫问题为增强学习问题 79 | 80 | 在强化学习的问题中，我们并不知道完整的模型 `T(s, a, s')` 和奖励 `R(s)`。我们只知道四元组 ``, 既在状态`s`下， 执行行为 `a`, 会进入`s'`, 获得奖励 `r`。 81 | 82 | ![](model-based.png) 83 | 84 | 我们的 Qlearner 会不断和世界互动，在状态 `s` 下， 执行行为 `a`，观察新的状态 `s'` 和获得的奖励 `r`。不断收集四元组，来学习这个世界的规则，找到最优策略。这也就是增强学习的学习过程。 85 | 86 | ## 实现 QLearner 87 | 你不可以导入任何额外的库，你需要按照下面定义的 API，在 `QLearner.py` 中实现 QLearner 类。 注意你的 QLearner 不应该知道任何有关走迷宫的信息。 88 | 89 | #### QLearner() 90 | 91 | QLearner 的构造函数，应该预留空间存放 所有状态和行为的 Q-table Q[s, a], 并将整个矩阵初始化为 0. 构造函数的每一个参数如下定义： 92 | 93 | - `num_states` integer, 所有状态个数。 94 | - `num_actions` integer, 所有行为个数。 95 | - `alpha` float, 更新Q-table时的学习率，范围 0.0 ~ 1.0, 常用值 0.2。 96 | - `gamma` float, 更新Q-table时的衰减率，范围 0.0 ~ 1.0, 常用值 0.9。 97 | - `rar` float, 随机行为比例, 每一步随机选择行为的概率。范围 0.0（从不随机） ~ 1.0（永远随机）, 常用值 0.5。 98 | - `radr` float, 随机行为比例衰减率, 每一步都更新 rar = rar * radr. 0.0（直接衰减到0） ~ 1.0（从不衰减）, 常用值 0.99。 99 | - `verbose` boolean, 如果为真，你的类可以打印调试语句，否则，禁止所有打印语句。 100 | 101 | #### query(s_prime, r) 102 | 103 | QLearner 的核心方法。他应该记录最后的状态 s 和最后的行为 a，然后使用新的信息 s_prime 和 r 来更新 Q-Table。 学习实例是四元组 ``. query() 应该返回一个 integer, 代表下一个行为。注意这里应该以 rar 的概率随机选择一个行为，并根据 radr 来更新 rar的值。 104 | 105 | 参数定义： 106 | 107 | - `s_prime` integer, 新的状态 108 | - `r` float, 即时奖励／惩罚，可以为正，可以为负。 109 | 110 | #### querysetstate(s) 111 | query() 方法的特殊版本。设置状态为 s，并且返回下一个行为 a （和 query() 方法规则一致，例如包括以一定概率随机选择行为）。但是这个方法不更新 Q-table，不更新 rar。我们主要会在两个地方用到它： 1）设置初始状态 2) 使用学习后的策略，但不更新它 112 | 113 | 这里是一个使用 API 的例子 114 | 115 | ``` 116 | import QLearner as ql 117 | 118 | learner = ql.QLearner(num_states = 100, \ 119 | num_actions = 4, \ 120 | alpha = 0.2, \ 121 | gamma = 0.9, \ 122 | rar = 0.98, \ 123 | radr = 0.999, \ 124 | verbose = False) 125 | 126 | s = 99 # 初始状态 127 | 128 | a = learner.querysetstate(s) # 状态s下的执行行为 a 129 | 130 | s_prime = 5 # 在状态 s，执行行为 a 之后，进入新状态 s_prime 131 | 132 | r = 0 # 在状态 s，执行行为 a 之后，获得即使奖励／惩罚 r 133 | 134 | next_action = learner.query(s_prime, r) 135 | ``` 136 | 137 | 重声一次，QLearner 不应该知道任何有关迷宫的信息。 138 | 139 | ## 实现 Maze 140 | Maze 类定义了迷宫的世界，起点，终点，障碍物和陷阱。 141 | #### Maze() 142 | Maze 的构造函数，定义了地图，随机行走概率，以及每一步的奖励／惩罚。你也可以在构造函数中定义自己的成员变量。例如起始地点，目标地点等。 143 | 144 | #### get\_start\_pos() 145 | 返回机器人的起始地点。即地图中，数值为2的位置。 146 | 147 | #### get\_goal\_pos() 148 | 返回机器人的目标地点。即地图中，数值为3的位置。 149 | 150 | #### move() 151 | 根据地图信息，现在位置和行为指令来移动机器人。机器人有 0.2 的概率不执行指令，而是在4个行为中随机选择。如果机器人尝试走入障碍物或走出地图，则会停留在原地。 152 | 153 | 返回新的位置和得到的奖励。 154 | 155 | #### print\_map() 156 | 工具函数，打印地图，无需修改。 157 | 158 | #### print\_trail() 159 | 工具函数，打印地图和路径，无需修改。参数 `trail` 是一个坐标的数组。例如 `[(0,0), (0,1), (0,2), (1,2)]` 160 | 161 | 162 | ## 实现 mazeqlearning 163 | 164 | #### to_state() 165 | 将位置用 0～99 的数字来表达，每个数字代表一个状态。 166 | 167 | 返回位置所对应的状态 168 | 169 | #### train() 170 | 在给定的地图中进行多次行走，每次行走都会让机器人从起点走到终点，或者超时（超过100,000步）。 171 | 172 | 返回所有行走的奖励。 173 | 174 | 每一次尝试的伪代码: 175 | 176 | ``` 177 | total_reward = 0 178 | robopos = startpos 179 | action = learner.querysetstate(to_state(robopos)) 180 | while not at goal and not timeout: 181 | newpos, reward = maze.move(robopos, action) 182 | robopos = newpos 183 | action = learner.query(to_state(robopos), reward) 184 | totol_reward += reward 185 | ``` 186 | 187 | #### maze_qlearning() 188 | 定义 QLearner 和 Maze，你可以使用默认参数，或使用自己的参数。调用 train() 进行训练， 189 | 190 | 返回所有行走的奖励的中位数。 191 | 192 | 193 | -------------------------------------------------------------------------------- /grading.py: -------------------------------------------------------------------------------- 1 | import mazeqlearning as mq 2 | 3 | 4 | def test_world(filename, median_reward): 5 | student_reward = mq.maze_qlearning(filename) 6 | if student_reward < 1.5*median_reward: 7 | return "Reward too low, expected %s, found %s"%(median_reward,student_reward) 8 | else: 9 | return "pass" 10 | 11 | print(test_world('testworlds/world01.csv', -29)) 12 | print(test_world('testworlds/world02.csv', -19)) 13 | print(test_world('testworlds/world03.csv', -80)) 14 | print(test_world('testworlds/world04.csv', -33)) 15 | print(test_world('testworlds/world05.csv', -24)) 16 | print(test_world('testworlds/world06.csv', -23)) 17 | print(test_world('testworlds/world07.csv', -26)) 18 | print(test_world('testworlds/world08.csv', -19)) 19 | print(test_world('testworlds/world09.csv', -20)) 20 | print(test_world('testworlds/world10.csv', -42)) 21 | -------------------------------------------------------------------------------- /mazeqlearning.py: -------------------------------------------------------------------------------- 1 | """ 2 | Train a Q Learner in a navigation problem. 3 | """ 4 | 5 | import numpy as np 6 | import random as rand 7 | import QLearner as ql 8 | import Maze 9 | 10 | 11 | # convert the position to a single integer state 12 | def to_state(pos): 13 | #TODO 14 | return 0 15 | 16 | 17 | # train learner to go through maze multiple epochs 18 | # each epoch involves one trip from start to the goal or timeout before reaching the goal 19 | # return list of rewards of each trip 20 | def train(maze, learner, epochs=500, timeout = 100000, verbose = False): 21 | #TODO 22 | rewards = np.zeros(epochs) 23 | return rewards 24 | 25 | 26 | # run the code to train a learner on a maze 27 | def maze_qlearning(filename): 28 | #TODO 29 | #initialize maze object 30 | #initialize learner object 31 | #execute train(maze, learner) 32 | #return median of all rewards 33 | 34 | return 0 35 | 36 | if __name__=="__main__": 37 | rand.seed(5) 38 | maze_qlearning('testworlds/world01.csv') 39 | -------------------------------------------------------------------------------- /model-based.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nd009/qlearning_robot/c40f04c83e9e707b3436da8ab00b3cb57fe2bed6/model-based.png -------------------------------------------------------------------------------- /testworlds/world01.csv: -------------------------------------------------------------------------------- 1 | 3,0,0,0,0,0,0,0,0,0 2 | 0,0,0,0,0,0,0,0,0,0 3 | 0,0,0,0,0,0,0,0,0,0 4 | 0,0,1,1,1,1,1,0,0,0 5 | 0,5,1,0,0,0,1,0,0,0 6 | 0,5,1,0,0,0,1,0,0,0 7 | 0,0,1,0,0,0,1,0,0,0 8 | 0,0,0,0,0,0,0,0,0,0 9 | 0,0,0,0,0,0,0,0,0,0 10 | 0,0,0,0,2,0,0,0,0,0 11 | -------------------------------------------------------------------------------- /testworlds/world02.csv: -------------------------------------------------------------------------------- 1 | 0,1,0,1,0,0,0,0,0,0 2 | 0,1,0,1,0,0,0,0,0,0 3 | 0,1,0,0,0,0,0,0,0,0 4 | 0,1,0,1,1,1,1,1,1,1 5 | 2,1,0,1,0,0,0,0,0,0 6 | 0,1,0,1,0,0,1,0,0,3 7 | 0,0,0,1,0,0,1,0,0,0 8 | 0,1,0,0,0,0,1,1,1,1 9 | 0,1,0,1,0,0,0,0,0,0 10 | 0,0,0,1,0,0,0,0,0,0 11 | -------------------------------------------------------------------------------- /testworlds/world03.csv: -------------------------------------------------------------------------------- 1 | 0,0,0,1,0,0,0,1,0,3 2 | 0,1,0,1,0,1,0,1,0,0 3 | 0,1,0,1,0,1,0,1,0,1 4 | 0,1,0,1,0,1,0,1,0,0 5 | 0,1,0,1,0,1,0,1,1,0 6 | 0,1,0,1,0,1,0,1,0,0 7 | 0,1,0,1,0,1,0,1,0,1 8 | 0,1,0,1,0,1,0,1,0,0 9 | 0,1,0,1,0,1,0,1,1,0 10 | 2,1,0,0,0,1,0,0,0,0 11 | -------------------------------------------------------------------------------- /testworlds/world04.csv: -------------------------------------------------------------------------------- 1 | 0,0,0,0,0,1,0,1,0,3 2 | 0,0,0,0,0,1,0,1,0,0 3 | 0,0,0,1,0,1,0,1,0,1 4 | 0,0,0,1,0,1,0,1,0,0 5 | 0,0,0,1,0,0,0,1,1,0 6 | 2,0,0,1,1,1,0,1,0,0 7 | 0,0,0,1,0,1,0,0,0,1 8 | 0,0,5,0,0,1,0,1,0,0 9 | 0,0,1,1,1,1,0,1,1,0 10 | 0,0,0,0,0,1,0,0,0,0 11 | -------------------------------------------------------------------------------- /testworlds/world05.csv: -------------------------------------------------------------------------------- 1 | 0,1,0,0,0,1,0,1,0,3 2 | 1,0,0,0,0,0,1,0,0,0 3 | 0,0,1,1,0,1,0,1,0,1 4 | 1,1,0,0,0,0,1,0,0,0 5 | 0,0,0,0,0,0,0,0,1,0 6 | 0,1,0,1,1,0,0,1,0,0 7 | 1,0,0,0,0,1,0,0,0,1 8 | 0,0,0,0,0,0,0,1,0,0 9 | 1,0,1,0,0,1,0,0,1,0 10 | 2,0,0,0,0,0,1,0,0,0 11 | -------------------------------------------------------------------------------- /testworlds/world06.csv: -------------------------------------------------------------------------------- 1 | 0,1,0,0,0,1,0,1,0,2 2 | 1,0,0,0,0,0,1,0,0,0 3 | 0,0,1,1,0,1,0,1,0,1 4 | 1,1,0,0,0,0,1,0,0,0 5 | 0,0,0,0,0,0,0,0,1,0 6 | 0,1,0,1,1,0,0,1,0,0 7 | 1,0,0,0,0,1,0,0,0,1 8 | 0,0,0,0,0,0,0,1,0,0 9 | 1,0,1,0,0,1,0,0,1,0 10 | 3,0,0,0,0,0,1,0,0,0 11 | -------------------------------------------------------------------------------- /testworlds/world07.csv: -------------------------------------------------------------------------------- 1 | 0,0,0,0,2,0,0,0,0,0 2 | 0,0,0,0,0,0,0,0,0,0 3 | 0,0,0,0,0,0,0,0,0,0 4 | 0,0,1,1,1,1,1,0,0,0 5 | 0,0,1,0,3,0,1,0,0,0 6 | 0,0,1,0,0,0,1,0,0,0 7 | 0,0,1,0,0,0,1,0,0,0 8 | 0,0,0,0,0,0,0,0,0,0 9 | 0,0,0,0,0,0,0,0,0,0 10 | 0,0,0,0,0,0,0,0,0,0 11 | -------------------------------------------------------------------------------- /testworlds/world08.csv: -------------------------------------------------------------------------------- 1 | 0,1,0,1,0,0,0,0,0,0 2 | 0,1,0,1,0,0,0,0,0,0 3 | 0,1,0,0,0,0,0,0,0,0 4 | 0,1,0,1,1,1,1,1,1,1 5 | 3,1,0,1,0,0,0,0,0,0 6 | 0,1,0,1,0,0,1,0,0,2 7 | 0,0,0,1,0,0,1,0,0,0 8 | 0,1,0,0,0,0,1,1,1,1 9 | 0,1,0,1,0,0,0,0,0,0 10 | 0,0,0,1,0,0,0,0,0,0 11 | -------------------------------------------------------------------------------- /testworlds/world09.csv: -------------------------------------------------------------------------------- 1 | 0,0,0,0,0,2,0,0,0,0 2 | 0,0,0,1,0,0,1,0,0,0 3 | 0,1,0,1,0,0,1,0,1,0 4 | 0,1,0,1,1,1,1,0,1,0 5 | 0,1,0,0,1,0,0,0,1,0 6 | 0,1,1,1,1,1,1,1,1,0 7 | 0,0,0,0,1,0,0,0,0,0 8 | 0,0,0,0,1,0,0,1,0,0 9 | 0,0,0,0,1,0,0,1,0,0 10 | 0,0,0,0,1,3,0,1,0,0 11 | -------------------------------------------------------------------------------- /testworlds/world10.csv: -------------------------------------------------------------------------------- 1 | 0,0,0,0,0,0,0,0,0,0 2 | 0,0,0,1,0,0,1,0,0,0 3 | 0,1,0,1,0,0,1,0,1,0 4 | 0,1,0,1,1,1,1,0,1,0 5 | 0,1,0,0,1,0,0,0,1,0 6 | 0,1,1,1,1,0,1,1,1,0 7 | 0,0,0,0,1,0,0,0,0,0 8 | 0,0,0,0,1,0,0,1,0,0 9 | 0,0,0,0,1,0,0,1,0,0 10 | 0,0,0,2,1,3,0,1,0,0 11 | --------------------------------------------------------------------------------