├── .gitignore
├── .idea
└── vcs.xml
├── .travis.yml
├── README.md
├── assets
├── bird.png
└── pipe.png
├── bower.json
├── brain
├── agent.js
└── brain.js
├── game
├── bird.js
├── main.js
├── pipes.js
└── score_chart.js
├── index.html
├── package.json
├── styles
└── main.css
└── tests
└── test.js
/.gitignore:
--------------------------------------------------------------------------------
1 | # See https://help.github.com/articles/ignoring-files for more about ignoring files.
2 | #
3 | # If you find yourself ignoring temporary files generated by your text editor
4 | # or operating system, you probably want to add a global ignore instead:
5 | # git config --global core.excludesfile '~/.gitignore_global'
6 |
7 | # Ignore bundler config.
8 | /.bundle
9 |
10 | # Ignore the default SQLite database.
11 | /db/*.sqlite3
12 | /db/*.sqlite3-journal
13 |
14 | # Ignore all logfiles and tempfiles.
15 | /log/*
16 | !/log/.keep
17 | /tmp
18 | /public/bower_components/*
19 | /public/node_modules/*
20 | /.idea/*
21 | *.png
22 | /node_modules/*
23 | /doc/**
24 | /repositories/**/*
25 | /bower_components/**/*
26 | /http-server/**/*
27 | /coverage/**/*
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: node_js
2 | node_js:
3 | - "4.3.1"
4 |
5 | after_script:
6 | - istanbul cover ./node_modules/mocha/bin/_mocha --report lcovonly tests -- -R spec && cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js && rm -rf ./coverage
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # FlappyBirdLearning
2 |
3 | [](https://gitter.im/gcaaa31928/FlappyBirdLearning?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
4 |
5 | [](https://travis-ci.org/gcaaa31928/FlappyBirdLearning)
6 | [](https://coveralls.io/github/gcaaa31928/FlappyBirdLearning?branch=master)
7 |
8 |
9 | 此為利用機器學習的方式自動學習flappy bird的專案，而學習方法則是用Q Learning
10 |
11 | 部份參考至http://sarvagyavaish.github.io/FlappyBirdRL/
12 |
13 |
14 | ### Get started
15 | ```bash
16 | npm install
17 | bower install
18 | ```
19 |
20 | ### Game Framework
21 | 利用Phaser.js製作出flappy bird遊戲，如下圖
22 |
23 | (參考至 http://www.lessmilk.com/tutorial/flappy-bird-phaser-1)
24 | 
25 |
26 | ### Q Learning
27 | 
28 |
29 | 重點在於這一個公式
30 |
31 | 而一開始利用這個公式訓練時碰到了一些困難
32 |
33 | 
34 |
35 | 當只使用這兩個狀態空間時，也就是QState是一個二維的空間
36 |
37 | 造成在低點的障礙物無法得知離地面或是離天空的距離而常常超出邊界
38 |
39 | 所以我加上了一個狀態空間，為到天空的距離
40 |
41 | 
42 |
43 | 但這又引發了別的問題，當我一般的速度通過磚塊時，理論上會以這個方式行動
44 | 
45 |
46 | 紅點的位置會慢慢訓練成不按的情況下Q Value會比按的情況下高
47 |
48 | 但在這個情況時
49 | 
50 | 由於下降的速度太快，導致於Q Value訓練成必須要按下之後才能避免撞到磚塊
51 |
52 | 也因為這兩個狀態沒辦法收斂到正確的位置，而收斂到了其他的位置
53 |
54 | 所以我們必須再加一個狀態空間為速度這個空間
55 |
56 |
57 | 基本上這樣就可以完成練習了
58 |
59 |
60 |
61 |
62 |
63 | ### Authors and Contributors
64 | @gcaaa31928
65 |
66 | ### Support or Contact
67 | 有任何意見可以開issues或是pull request
68 |
69 |
70 |
--------------------------------------------------------------------------------
/assets/bird.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/gcaaa31928/FlappyBirdLearning/3918fe3b93bf5790a53859bde299f1c0d84bfd86/assets/bird.png
--------------------------------------------------------------------------------
/assets/pipe.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/gcaaa31928/FlappyBirdLearning/3918fe3b93bf5790a53859bde299f1c0d84bfd86/assets/pipe.png
--------------------------------------------------------------------------------
/bower.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "FlappyBirdLearning",
3 | "authors": [
4 | "GCA "
5 | ],
6 | "description": "",
7 | "main": "",
8 | "moduleType": [],
9 | "license": "MIT",
10 | "homepage": "",
11 | "ignore": [
12 | "**/.*",
13 | "node_modules",
14 | "bower_components",
15 | "test",
16 | "tests"
17 | ],
18 | "dependencies": {
19 | "phaser": "^2.4.6",
20 | "jquery": "^2.2.3"
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/brain/agent.js:
--------------------------------------------------------------------------------
1 | var Agent = function (width_low, width_high, height_low, height_high, sky_high, velocity_low, velocity_high) {
2 | this.width_range = [width_low, width_high];
3 | this.height_range = [height_low, height_high];
4 | this.velocity_range = [velocity_low, velocity_high];
5 | this.brain = new Brain(width_low, width_high, height_low, height_high, sky_high, velocity_low, velocity_high);
6 | this.brain.initQState();
7 | };
8 |
9 | Agent.prototype = {
10 | think: function (bird, pipes, reward, first) {
11 |
12 | var bird_back_x = bird.sprite.x;
13 | var bird_back_y = bird.sprite.y;
14 |
15 | var closest_pipe_x = 9999;
16 | var closest_pipe_y = 9999;
17 | for (var i = 0; i < pipes.groups.length; i++) {
18 | var pipe = pipes.groups.getChildAt(i);
19 | if (bird_back_x >= pipe.x + pipe.width)
20 | continue;
21 | if (pipe.marked && pipe.x + pipe.width < closest_pipe_x) {
22 | closest_pipe_x = pipe.x + pipe.width;
23 | closest_pipe_y = pipe.y;
24 | }
25 | }
26 | var vertical_dist = closest_pipe_y - bird_back_y - this.height_range[0];
27 | var horizontal_dist = closest_pipe_x - bird_back_x - this.width_range[0];
28 | var sky_dist = bird_back_y;
29 | var velocity_dist = bird.sprite.body.velocity.y - this.velocity_range[0];
30 | if (isNaN(velocity_dist))
31 | velocity_dist = 0;
32 | return this.brain.learning(horizontal_dist, vertical_dist, sky_dist, velocity_dist, reward);
33 | }
34 | };
--------------------------------------------------------------------------------
/brain/brain.js:
--------------------------------------------------------------------------------
1 | var Brain = function (width_low, width_high, height_low, height_high, sky_height, velocity_low, velocity_height) {
2 | // 0 for click, 1 for don't click
3 | this.width_range = [width_low, width_high];
4 |
5 | this.height_range = [height_low, height_high];
6 | this.width_dist = width_high - width_low;
7 | this.height_dist = height_high - height_low;
8 | this.sky_height = sky_height;
9 | this.velocity_range = [velocity_low, velocity_height];
10 | this.velocity_dist = velocity_height - velocity_low;
11 | this.action = 'noClick';
12 | this.QState = [];
13 | this.current_state = [0, 0, 0, 0];
14 | this.next_state = [0, 0, 0, 0];
15 | this.resolution = 4;
16 | this.velocity_grid = 60;
17 | this.sky_resolution = 150;
18 | this.learning_rate = 0.7;
19 | this.random_explore = 0.0001;
20 | this.vertical_bin_offset = 25;
21 |
22 | if (width_low > width_high)
23 | throw 'width low must be lower than high value';
24 | if (height_low > height_high)
25 | throw 'height low must be lower than high value';
26 | if (sky_height < 0)
27 | throw 'sky height must be higher than zero';
28 | if (velocity_low > velocity_height)
29 | throw 'velocity low must be lower than high value';
30 |
31 | this.initQState = function () {
32 | for (var i = 0; i <= this.width_dist / this.resolution; i++) {
33 | this.QState[i] = [];
34 | for (var j = 0; j <= this.height_dist / this.resolution; j++) {
35 | this.QState[i][j] = [];
36 | for (var k = 0; k <= this.sky_height / this.sky_resolution; k++) {
37 | this.QState[i][j][k] = [];
38 | for (var v = 0; v <= this.velocity_dist / this.velocity_grid; v++) {
39 | this.QState[i][j][k][v] = {
40 | 'click': 0,
41 | 'noClick': 0
42 | };
43 | }
44 | }
45 | }
46 | }
47 | };
48 |
49 |
50 | this.restart = function () {
51 | this.current_state = [0, 0, 0, 0];
52 | this.next_state = [1, 1, 1, 1];
53 | this.action = 'noClick';
54 | this.next_action = 'noClick';
55 | };
56 |
57 | this.setNextState = function (vertical_dist, horizontal_dist, sky_dist, velocity) {
58 | this.next_state = [vertical_dist, horizontal_dist, sky_dist, velocity];
59 | };
60 |
61 | this.updateCurrentState = function () {
62 | this.action = this.next_action;
63 | this.current_state = [this.next_state[0], this.next_state[1], this.next_state[2], this.next_state[3]];
64 | };
65 |
66 | this.binVerticalState = function (vertical_state) {
67 | var vertical_bin = Math.min(
68 | this.height_dist, vertical_state
69 | );
70 | vertical_bin /= this.resolution;
71 | return vertical_bin < 0 ? 0 : Math.floor(vertical_bin);
72 | };
73 |
74 | this.binHorizontalState = function (horizontal_state) {
75 | var horizontal_bin = Math.min(
76 | this.width_dist, horizontal_state
77 | );
78 | horizontal_bin /= this.resolution;
79 | return horizontal_bin < 0 ? 0 : Math.floor(horizontal_bin);
80 | };
81 |
82 | this.binSkyDist = function (sky_state) {
83 | var sky_bin = Math.min(this.sky_height, sky_state);
84 | sky_bin /= this.sky_resolution;
85 | return sky_bin < 0 ? 0 : Math.floor(sky_bin);
86 | };
87 |
88 | this.binVelocityState = function (velocity) {
89 | var velocity_bin = Math.min(this.velocity_dist, velocity);
90 | velocity_bin /= this.velocity_grid;
91 | return velocity_bin < 0 ? 0 : Math.floor(velocity_bin);
92 | };
93 |
94 | this.updateQState = function (horizontal_state,
95 | vertical_state,
96 | sky_state,
97 | velocity_state,
98 | horizontal_next_state,
99 | vertical_next_state,
100 | sky_next_state,
101 | velocity_next_state,
102 | previous_action,
103 | reward) {
104 | var action = null;
105 | if (vertical_state >= (this.height_dist / this.resolution) - this.vertical_bin_offset || vertical_state <= this.vertical_bin_offset) {
106 | action = vertical_state >= (this.height_dist / this.resolution) - this.vertical_bin_offset ? 'noClick' : 'click';
107 | } else {
108 | var click_q_next_value = this.QState[horizontal_next_state][vertical_next_state][sky_next_state][velocity_next_state]['click'];
109 | var no_click_q_next_value = this.QState[horizontal_next_state][vertical_next_state][sky_next_state][velocity_state]['noClick'];
110 | action = click_q_next_value > no_click_q_next_value ? 'click' : 'noClick';
111 | }
112 | var max_next_q = this.QState[horizontal_next_state][vertical_next_state][sky_next_state][velocity_next_state][action];
113 | var current_q_value = this.QState[horizontal_state][vertical_state][sky_state][velocity_state][previous_action];
114 | this.QState[horizontal_state][vertical_state][sky_state][velocity_state][previous_action] = current_q_value + this.learning_rate * (reward + max_next_q - current_q_value);
115 | return action;
116 | };
117 |
118 |
119 | this.learning = function (horizontal_dist, vertical_dist, sky_dist, velocity, reward) {
120 | // step 1: get state
121 | this.setNextState(vertical_dist, horizontal_dist, sky_dist, velocity);
122 | var vertical_state = this.binVerticalState(this.current_state[0]);
123 | var horizontal_state = this.binHorizontalState(this.current_state[1]);
124 | var sky_state = this.binSkyDist(this.current_state[2]);
125 | var velocity_state = this.binVelocityState(this.current_state[3]);
126 |
127 | var vertical_next_state = this.binVerticalState(this.next_state[0]);
128 | var horizontal_next_state = this.binHorizontalState(this.next_state[1]);
129 | var sky_next_state = this.binSkyDist(this.next_state[2]);
130 | var velocity_next_state = this.binVelocityState(this.next_state[3]);
131 |
132 | // step 2: update
133 |
134 | this.next_action = this.updateQState(
135 | horizontal_state,
136 | vertical_state,
137 | sky_state,
138 | velocity_state,
139 | horizontal_next_state,
140 | vertical_next_state,
141 | sky_next_state,
142 | velocity_next_state,
143 | this.action,
144 | reward
145 | );
146 |
147 | // step 4: update s with s'
148 | this.updateCurrentState();
149 |
150 | // step 3: take the action a
151 | return this.next_action;
152 | };
153 |
154 | this.toJson = function () {
155 | return JSON.stringify(this.QState);
156 | };
157 |
158 | this.fromJson = function (json) {
159 | this.QState = JSON.parse(json);
160 | };
161 | };
162 |
163 |
164 | module.exports = Brain;
--------------------------------------------------------------------------------
/game/bird.js:
--------------------------------------------------------------------------------
1 | function Bird(game) {
2 | this.game = game;
3 |
4 | this.preload = function () {
5 | this.game.load.image('bird', 'assets/bird.png')
6 | };
7 |
8 | this.create = function () {
9 | this.sprite = this.game.add.sprite(100, 245, 'bird');
10 | this.sprite.anchor.setTo(-0.2, 0.5);
11 | this.game.physics.arcade.enable(this.sprite);
12 | this.sprite.body.gravity.y = 1000;
13 | };
14 |
15 | this.setInputHandler = function () {
16 | var spaceKey = this.game.input.keyboard.addKey(
17 | Phaser.Keyboard.SPACEBAR);
18 | spaceKey.onDown.add(this.jump, this);
19 | };
20 |
21 | this.jump = function () {
22 | // Add a vertical velocity to the bird
23 | this.sprite.body.velocity.y = -400;
24 | // Create an animation on the bird
25 | var animation = this.game.add.tween(this.sprite);
26 |
27 | // Change the angle of the bird to -20° in 100 milliseconds
28 | animation.to({angle: -20}, 100);
29 |
30 | // And start the animation
31 | animation.start();
32 | this.sprite.body.velocity.y = -300;
33 | };
34 |
35 | this.update = function () {
36 | if (this.sprite.angle < 20) {
37 | this.sprite.angle += 1;
38 | }
39 | this.sprite.body.velocity.y = Math.max(
40 | Math.min(550,this.sprite.body.velocity.y), -300);
41 | };
42 |
43 | this.died = function() {
44 | return (this.sprite.y < 0 || this.sprite.y > 450);
45 | };
46 |
47 |
48 | }
--------------------------------------------------------------------------------
/game/main.js:
--------------------------------------------------------------------------------
1 | var game = new Phaser.Game(800, 490, Phaser.AUTO, 'game_area');
2 | var agent = new Agent(0, 380, -140, 250, 450, -400, 550);
3 | var score_chart = new score_chart();
4 | var reward_arr = [];
5 | var mainState = {
6 | state: 'init',
7 | times: 0,
8 | preload: function () {
9 |
10 | game.stage.disableVisibilityChange = true;
11 | game.config.forceSetTimeOut = true;
12 | this.created = false;
13 | // This function will be executed at the beginning
14 | // That's where we load the images and sounds
15 | this.context = {
16 | score: 0,
17 | game: game
18 | };
19 | this.bird = new Bird(game);
20 | this.bird.preload();
21 |
22 | this.pipes = new Pipes(this.context);
23 | this.pipes.preload();
24 | this.state = 'preload';
25 | this.timer = this.game.time.events.loop(140, this.scoreIncrement, this);
26 | },
27 |
28 | create: function () {
29 | // Change the background color of the game to blue
30 |
31 | game.stage.backgroundColor = '#71c5cf';
32 | this.score = 0;
33 | this.labelScore = game.add.text(20, 20, "0",
34 | {font: "30px Arial", fill: "#ffffff"});
35 | // Set the physics system
36 | game.physics.startSystem(Phaser.Physics.ARCADE);
37 |
38 | // Display the bird at the position x=100 and y=245
39 | // set bird
40 | this.bird.create();
41 | this.bird.setInputHandler();
42 | this.pipes.create();
43 |
44 |
45 | this.created = true;
46 | this.state = 'created';
47 | },
48 |
49 |
50 | update: function () {
51 | if (this.state == 'died') {
52 | return;
53 | }
54 | this.state = 'playing';
55 | if (!this.created)
56 | return;
57 | this.reward = 1;
58 | if (this.bird.died()) {
59 | this.reward = -1000;
60 | this.first_update = true;
61 | this.restartGame();
62 | return;
63 | }
64 | if (game.physics.arcade.overlap(
65 | this.bird.sprite, this.pipes.groups, null, null, this)) {
66 | this.reward = -1000;
67 | this.first_update = true;
68 | this.restartGame();
69 | return;
70 | }
71 |
72 | this.bird.update();
73 | this.labelScore.text = parseInt(this.context.score).toString();
74 | this.learning(this.first_update);
75 | this.first_update = false;
76 | },
77 |
78 | destroy: function () {
79 | this.bird.sprite.destroy();
80 | this.pipes.groups.destroy();
81 | },
82 |
83 | // Restart the game
84 | restartGame: function () {
85 | // Start the 'main' state, which restarts the game
86 |
87 | this.reward = -1000;
88 | drawChart();
89 | game.time.events.remove(this.timer);
90 | game.state.start('main');
91 | this.state = 'died';
92 | this.learning();
93 | agent.brain.restart();
94 | this.times++;
95 | },
96 |
97 | scoreIncrement: function () {
98 | this.context.score += 0.1;
99 | },
100 |
101 | learning: function (first) {
102 | var actionix = agent.think(this.bird, this.pipes, this.reward, first);
103 | if (actionix == 'click') {
104 | this.bird.jump();
105 | }
106 | }
107 | };
108 |
109 | function drawChart() {
110 | reward_arr.push([mainState.times, mainState.context.score]);
111 | score_chart.updateData(reward_arr);
112 | }
113 |
114 | function saveLearningData() {
115 | var blob = new Blob(["some text"], {
116 | type: "text/plain;charset=utf-8;"
117 | });
118 | var text = agent.brain.toJson();
119 | saveAs(blob, text);
120 | }
121 |
122 | var start = function () {
123 | game.paused = false;
124 | };
125 | var stop = function () {
126 | game.paused = true;
127 | };
128 | // Add and start the 'main' state to start the game
129 | game.state.add('main', mainState);
130 | game.state.start('main');
131 |
--------------------------------------------------------------------------------
/game/pipes.js:
--------------------------------------------------------------------------------
1 | function Pipes(context) {
2 | this.context = context;
3 | this.game = this.context.game;
4 | this.groups = this.game.add.group();
5 |
6 | this.preload = function () {
7 | this.game.load.image('pipes', 'assets/pipe.png');
8 | };
9 |
10 | this.create = function () {
11 | this.groups = this.game.add.group();
12 | this.addRowOfPipes();
13 | this.timer = this.game.time.events.loop(1400, this.addRowOfPipes, this);
14 | };
15 |
16 | this.addOnePipe = function (x, y, mark) {
17 | var pipe = this.game.add.sprite(x, y, 'pipes');
18 | pipe.marked = mark;
19 | this.groups.add(pipe);
20 | this.game.physics.arcade.enable(pipe);
21 | pipe.body.velocity.x = -200;
22 | pipe.checkWorldBounds = true;
23 | pipe.outOfBoundsKill = true;
24 | };
25 |
26 | this.addRowOfPipes = function () {
27 | var hole = Math.floor(Math.random() * 5) + 1;
28 |
29 | for (var i = 0; i < 8; i++) {
30 | if (i == hole + 2) {
31 | this.addOnePipe(800, i * 60 + 10, true);
32 | } else if (i != hole && i != hole + 1) {
33 | this.addOnePipe(800, i * 60 + 10, false);
34 | }
35 | }
36 | var that = this;
37 | this.groups.forEachDead(function (pipe) {
38 | that.groups.remove(pipe);
39 | });
40 | };
41 |
42 |
43 | }
--------------------------------------------------------------------------------
/game/score_chart.js:
--------------------------------------------------------------------------------
1 | var score_chart = function() {
2 |
3 |
4 | var margin = {top: 20, right: 20, bottom: 30, left: 50},
5 | width = 800 - margin.left - margin.right,
6 | height = 250 - margin.top - margin.bottom;
7 |
8 | var formatDate = d3.time.format("%d-%b-%y");
9 |
10 | var x = d3.time.scale()
11 | .range([0, width]);
12 |
13 | var y = d3.scale.linear()
14 | .range([height, 0]);
15 |
16 | var xAxis = d3.svg.axis()
17 | .scale(x)
18 | .orient("bottom");
19 |
20 | var yAxis = d3.svg.axis()
21 | .scale(y)
22 | .orient("left");
23 |
24 | var valueline = d3.svg.line()
25 | .x(function(d) { return x(d.date); })
26 | .y(function(d) { return y(d.close); });
27 |
28 |
29 | this.svg = d3.select("#score_chart").append("svg")
30 | .attr("width", width + margin.left + margin.right)
31 | .attr("height", height + margin.top + margin.bottom)
32 | .append("g")
33 | .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
34 |
35 | var parseDate = d3.time.format("%Y-%m-%d").parse;
36 |
37 |
38 | this.draw = function(arrData) {
39 | this.svg.append("g")
40 | .attr("class", "y axis")
41 | .call(yAxis)
42 | .append("text")
43 | .attr("transform", "rotate(-90)")
44 | .attr("y", 6)
45 | .attr("dy", ".71em")
46 | .style("text-anchor", "end")
47 | .text("Score");
48 |
49 | this.svg.append("path")
50 | .attr("class", "line")
51 |
52 | };
53 |
54 | this.updateData = function(arrData) {
55 | var data = arrData.map(function (d) {
56 | return {
57 | date: d[0],
58 | close: d[1]
59 | };
60 | });
61 | x.domain(d3.extent(data, function (d) {
62 | return d.date;
63 | }));
64 | y.domain(d3.extent(data, function (d) {
65 | return d.close;
66 | }));
67 |
68 | var svg = d3.select("#score_chart").transition();
69 |
70 | // Make the changes
71 | svg.select(".line") // change the line
72 | .duration(750)
73 | .attr("d", valueline(data));
74 | svg.select(".y.axis") // change the y axis
75 | .duration(750)
76 | .call(yAxis);
77 |
78 | };
79 | this.draw([]);
80 | };
--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Flappy Bird Learning
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "flappy-bird-learning",
3 | "version": "1.0.0",
4 | "description": "",
5 | "main": "index.js",
6 | "scripts": {
7 | "test": "mocha tests/test.js"
8 | },
9 | "repository": {
10 | "type": "git",
11 | "url": "git+https://github.com/gcaaa31928/FlappyBirdLearning.git"
12 | },
13 | "author": "",
14 | "license": "ISC",
15 | "bugs": {
16 | "url": "https://github.com/gcaaa31928/FlappyBirdLearning/issues"
17 | },
18 | "homepage": "https://github.com/gcaaa31928/FlappyBirdLearning#readme",
19 | "dependencies": {
20 | "chai": "^3.5.0",
21 | "istanbul": "^0.4.3",
22 | "mocha": "^2.4.5"
23 | },
24 | "devDependencies": {
25 | "coveralls": "^2.11.9"
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/styles/main.css:
--------------------------------------------------------------------------------
1 | body {
2 | font: 10px sans-serif;
3 | }
4 |
5 | .axis path,
6 | .axis line {
7 | fill: none;
8 | stroke: #000;
9 | shape-rendering: crispEdges;
10 | }
11 |
12 | .x.axis path {
13 | display: none;
14 | }
15 |
16 | .line {
17 | fill: none;
18 | stroke: steelblue;
19 | stroke-width: 1.5px;
20 | }
21 |
22 | rect.bordered {
23 | stroke: #E6E6E6;
24 | stroke-width:2px;
25 | }
26 |
27 | text.mono {
28 | font-size: 9pt;
29 | font-family: Consolas, courier;
30 | fill: #aaa;
31 | }
32 |
33 | text.axis-workweek {
34 | fill: #000;
35 | }
36 |
37 | text.axis-worktime {
38 | fill: #000;
39 | }
--------------------------------------------------------------------------------
/tests/test.js:
--------------------------------------------------------------------------------
1 | var Brain = require('../brain/brain.js');
2 | var expect = require('chai').expect;
3 |
4 | describe('Test Brain', function () {
5 | describe('constructor', function () {
6 | it('should correctly set value', function () {
7 | var brain = new Brain(0, 1, 0, 1, 1, 0, 1);
8 | expect(brain.width_range).to.eql([0, 1]);
9 | expect(brain.height_range).to.eql([0, 1]);
10 | expect(brain.velocity_range).to.eql([0, 1]);
11 | expect(brain.width_dist).to.eql(1);
12 | expect(brain.height_dist).to.eql(1);
13 | expect(brain.velocity_dist).to.eql(1);
14 | });
15 | it('should throw exception if low value lower than high value', function () {
16 | expect(function () {
17 | new Brain(1, 0, 0, 1, 1, 0, 1);
18 | }).to.throw('width low must be lower than high value');
19 | expect(function () {
20 | new Brain(0, 1, 1, 0, 1, 0, 1);
21 | }).to.throw('height low must be lower than high value');
22 | expect(function () {
23 | new Brain(0, 1, 0, 1, -1, 0, 1);
24 | }).to.throw('sky height must be higher than zero');
25 | expect(function () {
26 | new Brain(0, 1, 0, 1, 1, 1, 0);
27 | }).to.throw('velocity low must be lower than high value');
28 | });
29 | });
30 |
31 | describe('QState', function () {
32 | it('should init QState correctly', function () {
33 | var brain = new Brain(0, 1, 0, 1, 1, 0, 1);
34 | brain.initQState();
35 | var expected = [];
36 | for (var i = 0; i <= 0; i++) {
37 | expected[i] = [];
38 | for (var j = 0; j <= 0; j++) {
39 | expected[i][j] = [];
40 | for (var k = 0; k <= 0; k++) {
41 | expected[i][j][k] = [];
42 | for (var v = 0; v <= 0; v++) {
43 | expected[i][j][k][v] = {
44 | 'click': 0,
45 | 'noClick': 0
46 | };
47 | }
48 | }
49 | }
50 | }
51 | expect(brain.QState).to.eql(expected);
52 | });
53 | });
54 |
55 | describe('restart', function () {
56 | it('should restart correctly', function () {
57 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
58 | brain.current_state = [3, 3, 3, 3];
59 | brain.next_state = [4, 4, 4, 4];
60 | brain.restart();
61 | expect(brain.current_state).to.eql([0, 0, 0, 0]);
62 | expect(brain.next_state).to.eql([1, 1, 1, 1]);
63 | expect(brain.action).to.equal('noClick');
64 | expect(brain.next_action).to.equal('noClick');
65 | });
66 | });
67 |
68 | describe('set next state', function () {
69 | it('should set correctly', function () {
70 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
71 | brain.setNextState(1, 1, 1, 1);
72 | expect(brain.next_state).to.eql([1, 1, 1, 1]);
73 | });
74 | });
75 |
76 | describe('bin vertical state', function () {
77 | it('should puts higher vertical state in corrects bins', function () {
78 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
79 | brain.resolution = 2;
80 | expect(brain.binVerticalState(100)).to.equal(2);
81 | });
82 | it('should puts negative vertical state in corrects bins', function () {
83 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
84 | brain.resolution = 2;
85 | expect(brain.binVerticalState(-100)).to.equal(0);
86 | });
87 | it('should puts float vertical state in corrects bins', function () {
88 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
89 | brain.resolution = 2;
90 | expect(brain.binVerticalState(2.02000020202020)).to.equal(1);
91 | });
92 | });
93 |
94 | describe('bin horizontal state', function () {
95 | it('should puts higher horizontal state in corrects bins', function () {
96 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
97 | brain.resolution = 2;
98 | expect(brain.binHorizontalState(100)).to.equal(2);
99 | });
100 | it('should puts negative horizontal state in corrects bins', function () {
101 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
102 | brain.resolution = 2;
103 | expect(brain.binHorizontalState(-100)).to.equal(0);
104 | });
105 | it('should puts float horizontal state in corrects bins', function () {
106 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
107 | brain.resolution = 2;
108 | expect(brain.binHorizontalState(2.02000020202020)).to.equal(1);
109 | });
110 | });
111 |
112 | describe('bin sky state', function () {
113 | it('should puts higher sky state in corrects bins', function () {
114 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
115 | brain.sky_resolution = 2;
116 | expect(brain.binSkyDist(100)).to.equal(2);
117 | });
118 | it('should puts negative vertical state in corrects bins', function () {
119 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
120 | brain.sky_resolution = 2;
121 | expect(brain.binSkyDist(-100)).to.equal(0);
122 | });
123 | it('should puts float vertical state in corrects bins', function () {
124 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
125 | brain.sky_resolution = 2;
126 | expect(brain.binSkyDist(2.02000020202020)).to.equal(1);
127 | });
128 | });
129 | describe('bin velocity state', function () {
130 | it('should puts higher velocity state in corrects bins', function () {
131 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
132 | brain.velocity_grid = 2;
133 | expect(brain.binVelocityState(100)).to.equal(2);
134 | });
135 | it('should puts negative velocity state in corrects bins', function () {
136 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
137 | brain.velocity_grid = 2;
138 | expect(brain.binVelocityState(-100)).to.equal(0);
139 | });
140 | it('should puts float velocity state in corrects bins', function () {
141 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
142 | brain.velocity_grid = 2;
143 | expect(brain.binVelocityState(2.02000020202020)).to.equal(1);
144 | });
145 | });
146 |
147 | describe('update QState', function () {
148 | it('vertical state in others region', function () {
149 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
150 | brain.vertical_bin_offset = 0;
151 | brain.velocity_grid = 1;
152 | brain.resolution = 1;
153 | brain.sky_resolution = 1;
154 | brain.initQState();
155 | var action = brain.updateQState(0, 4, 0, 0, 0, 0, 0, 0, 'noClick', 0);
156 | expect(action).to.equal('noClick');
157 | action = brain.updateQState(0, 0, 0, 0, 0, 0, 0, 0, 0, 'noClick', 0);
158 | expect(action).to.equal('click');
159 | });
160 | it('update correctly information', function () {
161 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
162 | brain.vertical_bin_offset = 0;
163 | brain.velocity_grid = 1;
164 | brain.resolution = 1;
165 | brain.sky_resolution = 1;
166 | brain.initQState();
167 | brain.updateQState(0, 4, 0, 0, 0, 0, 0, 0, 'noClick', 1000);
168 | var state = brain.QState[0][4][0][0]['noClick'];
169 | expect(state).to.equal(700);
170 | brain.updateQState(1, 1, 0, 0, 0, 4, 0, 0, 'click', -1000);
171 | state = brain.QState[1][1][0][0]['click'];
172 | expect(state).to.equal(-210);
173 | brain.updateQState(1, 0, 0, 0, 1, 1, 0, 0, 'click', 0.7);
174 | });
175 | });
176 | describe('update current state', function () {
177 | it('regularly update current state correctly', function () {
178 | var brain = new Brain(0, 4, 0, 4, 4, 0, 4);
179 | brain.action = null;
180 | brain.current_state = [1, 1, 1, 1];
181 | brain.next_action = 'noClick';
182 | brain.next_state = [5, 5, 5, 5];
183 | brain.updateCurrentState();
184 | expect(brain.action).to.equal('noClick');
185 | expect(brain.current_state).to.eql([5, 5, 5, 5]);
186 | });
187 | });
188 | describe('machine learning', function () {
189 | it('learning correctly', function () {
190 | var brain = new Brain(0, 10, 0, 10, 10, 0, 1);
191 | brain.vertical_bin_offset = 0;
192 | brain.velocity_grid = 1;
193 | brain.resolution = 1;
194 | brain.sky_resolution = 1;
195 | brain.initQState();
196 | for (var reward = -3; reward < 3; reward++) {
197 | for (var i = 0; i < 10; i++) {
198 | brain.learning(i, i, i, 0, reward);
199 | }
200 | }
201 | expect(brain.learning(1, 1, 1, 0, -900)).to.equal('noClick');
202 | expect(brain.learning(2, 2, 2, 0, -800)).to.equal('noClick');
203 | expect(brain.QState[0][0][0][0]['click']).to.equal(-0.053297999999999845);
204 | expect(brain.QState[0][0][0][0]['noClick']).to.equal(-3.4999999999999996);
205 | });
206 | });
207 | });
--------------------------------------------------------------------------------