├── .editorconfig ├── .gitignore ├── .gitmodules ├── .travis.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE.md ├── README.md ├── code-of-conduct.md ├── images └── Macao-logo-color.png ├── package-lock.json ├── package.json ├── rollup.config.ts ├── src ├── controller.ts ├── data-store.ts ├── entities.ts ├── macao.ts ├── mcts │ ├── back-propagate │ │ └── back-propagate.ts │ ├── expand │ │ └── expand.ts │ ├── mcts.ts │ ├── select │ │ ├── best-child │ │ │ └── best-child.ts │ │ └── select.ts │ └── simulate │ │ └── simulate.ts └── utils.ts ├── test ├── data-store.test.ts ├── entities.test.ts ├── macao.test.ts ├── mcts.test.ts ├── tic-tac-toe │ ├── playground.ts │ ├── tic-tac-toe.test.ts │ └── tic-tac-toe.ts ├── ultimate-tic-tac-toe │ ├── playground.ts │ ├── ultimate-tic-tac-toe.test.ts │ └── ultimate-tic-tac-toe.ts ├── utils.test.ts └── wondev-woman │ ├── playground.ts │ └── wondev-woman.ts ├── tools ├── gh-pages-publish.ts └── semantic-release-prepare.ts ├── tsconfig.json └── tslint.json /.editorconfig: -------------------------------------------------------------------------------- 1 | #root = true 2 | 3 | [*] 4 | indent_style = space 5 | end_of_line = lf 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | max_line_length = 100 10 | indent_size = 2 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | .nyc_output 4 | .DS_Store 5 | *.log 6 | .vscode 7 | .idea 8 | dist 9 | compiled 10 | .awcache 11 | .rpt2_cache 12 | docs/api 13 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "docs/wiki"] 2 | path = docs/wiki 3 | url = https://github.com/Neoflash1979/macao.wiki.git 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | branches: 3 | only: 4 | - master 5 | - /^greenkeeper/.*$/ 6 | cache: 7 | yarn: true 8 | directories: 9 | - node_modules 10 | notifications: 11 | email: false 12 | node_js: 13 | - node 14 | script: 15 | - npm run test:prod && npm run build 16 | after_success: 17 | - npm run report-coverage 18 | - npm run deploy-docs 19 | - npm run semantic-release 20 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | ## [2.0.1](https://github.com/Neoflash1979/macao/compare/v2.0.0...v2.0.1) (2018-05-22) 3 | 4 | 5 | ### Bug Fixes 6 | 7 | * Trigger new patch release for major documentation update ([75c61d8](https://github.com/Neoflash1979/macao/commit/75c61d8)) 8 | 9 | 10 | # [2.0.0](https://github.com/Neoflash1979/macao/compare/v1.5.0...v2.0.0) (2018-05-17) 11 | 12 | 13 | ### Features 14 | 15 | * **macao.ts:** Add asynchronous getAction method ([a2995a0](https://github.com/Neoflash1979/macao/commit/a2995a0)) 16 | 17 | 18 | ### Performance Improvements 19 | 20 | * **utils.ts:** Modify now() so that it uses performance.now() in Node.js ([324e09a](https://github.com/Neoflash1979/macao/commit/324e09a)) 21 | 22 | 23 | ### BREAKING CHANGES 24 | 25 | * **macao.ts:** Macao.getAction() is now asynchronous and returns a `Promise` that resolves into 26 | and `Action`. 27 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Macao 2 | 3 | 👍🎉 First off, thanks for taking the time to contribute! 🎉👍 4 | 5 | The following is a set of guidelines for contributing to **Macao**. These are mostly guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request. 6 | 7 | **Working on your first Pull Request?** You can learn how from this *free* series [How to Contribute to an Open Source Project on GitHub](https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github) 8 | 9 | ## [Code of Conduct](#code-of-conduct) 10 | 11 | This project and everyone participating in it is governed by the **Macao** [Code of Conduct](code-of-conduct). By participating, you are expected to uphold this code. Please report unacceptable behavior to philippe_vaillancourt@sympatico.ca. -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2019 Philippe Vaillancourt 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | Macao Logo 3 |

4 | 5 | [![Build Status](https://travis-ci.org/Neoflash1979/macao.svg?branch=master)](https://travis-ci.org/Neoflash1979/macao) 6 | [![Coverage Status](https://coveralls.io/repos/github/Neoflash1979/macao/badge.svg)](https://coveralls.io/github/Neoflash1979/macao) 7 | [![npm version](https://badge.fury.io/js/macao.svg)](https://www.npmjs.com/package/macao) 8 | [![PRs welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](http://makeapullrequest.com/) 9 | 10 | **Macao** is a simple framework that allows the addition of powerful game AI to your JavaScript game with minimal configuration and coding. 11 | 12 | ## [Why use **Macao**](#why) 13 | 14 | Perhaps: 15 | 16 | 1. You are programming a simple game like Tic-Tac-Toe, Connect Four, Nim or the likes and would like to make the game playable against "the computer" without having to code a bunch of if statements, state machines and decision trees. 17 | 2. You're programming a more complicated game like chess, Checkers or Go. Unfortunately, programming an AI player that is able to challenge good human players usually requires imparting the AI with the knowledge of expert players, and you are not. You may know the rules of the game, but you are no Grand Master. 18 | 3. You are developing an entirely original game. Obviously, you know the rules of the game since you are the one who invented it, but do you know the best strategies to employ during play? How can you? Unlike with chess, your game hasn't been played by millions of people over centuries so there are no doubt various ways to play your game that even YOU haven't thought about. 19 | 4. You are building a prototype of a game and, eventually you plan to give it a very complicated AI with detailed expert domain knowledge, gained through weeks of machine learning and written in thousands of lines of code. But for now, you'd really just like to figure out if and how the game works and if it's fun to play. 20 | 21 | ## [What Is **Macao**](#what) 22 | 23 | **Macao** is a simple framework that allows the addition of powerful game AI to your JavaScript game with minimal configuration and coding. It is based on the powerful, yet simple, [Monte Carlo Tree Search](https://en.wikipedia.org/wiki/Monte_Carlo_tree_search) algorithm. **Macao** makes it possible for you to add AI to your game, even if you are a bad player and don't understand the strategy. All you need to know are the rules of the game, that's it, and that's all **Macao** needs to know in order to work it's magic. 24 | 25 | Although we plan to quickly expand the types of games it supports, at the moment **Macao** is compatible with two player, turn-based, [perfect information](https://en.wikipedia.org/wiki/Perfect_information), [deterministic](http://www.whatgamesare.com/determinism.html) games. Well, that was a mouthful. Basically this means that it should work with any game similar to Chess, Checkers, Tic-Tac-Toe, Nine Men's Morris etc. 26 | 27 | ## [How to use **Macao**](#how) 28 | 29 | ```javascript 30 | import { Macao } from "macao"; 31 | 32 | // Functions that implement the game's rules. 33 | // These functions are provided by you. 34 | const funcs = { 35 | generateActions, 36 | applyAction, 37 | stateIsTerminal, 38 | calculateReward 39 | }; 40 | 41 | const config = { 42 | duration: 30 43 | // ... 44 | }; 45 | 46 | const macao = new Macao(funcs, config); 47 | 48 | // Somewhere inside your game loop 49 | const action = macao.getAction(state); 50 | ``` 51 | 52 | For more indepth information on how to use **Macao**, please see the [general documentation](https://github.com/Neoflash1979/macao/wiki). 53 | 54 | ## [Installation](#installation) 55 | 56 | ```bash 57 | npm install macao --save 58 | ``` 59 | 60 | ## [General Documentation](#general-documentation) 61 | 62 | You can read the general documenation [here](https://github.com/Neoflash1979/macao/wiki). 63 | 64 | ## [API](#api) 65 | 66 | You can read the API documentation [here](https://neoflash1979.github.io/macao/). 67 | 68 | ## [Contributing](#contributing) 69 | 70 | Please take a look at our [contributing](https://github.com/Neoflash1979/macao/blob/master/CONTRIBUTING.md) guidelines if you're interested in helping! 71 | 72 | ## [Changelog](#changelog) 73 | 74 | See [CHANGELOG.md](https://github.com/Neoflash1979/macao/blob/master/CHANGELOG.md) 75 | 76 | ## [License](#license) 77 | 78 | **Macao** is provided under the [MIT License](https://github.com/Neoflash1979/macao/blob/master/LICENSE). 79 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 26 | * Trolling, insulting/derogatory comments, and personal or political attacks 27 | * Public or private harassment 28 | * Publishing others' private information, such as a physical or electronic 29 | address, without explicit permission 30 | * Other conduct which could reasonably be considered inappropriate in a 31 | professional setting 32 | 33 | ## Our Responsibilities 34 | 35 | Project maintainers are responsible for clarifying the standards of acceptable 36 | behavior and are expected to take appropriate and fair corrective action in 37 | response to any instances of unacceptable behavior. 38 | 39 | Project maintainers have the right and responsibility to remove, edit, or 40 | reject comments, commits, code, wiki edits, issues, and other contributions 41 | that are not aligned to this Code of Conduct, or to ban temporarily or 42 | permanently any contributor for other behaviors that they deem inappropriate, 43 | threatening, offensive, or harmful. 44 | 45 | ## Scope 46 | 47 | This Code of Conduct applies both within project spaces and in public spaces 48 | when an individual is representing the project or its community. Examples of 49 | representing a project or community include using an official project e-mail 50 | address, posting via an official social media account, or acting as an appointed 51 | representative at an online or offline event. Representation of a project may be 52 | further defined and clarified by project maintainers. 53 | 54 | ## Enforcement 55 | 56 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 57 | reported by contacting the project team at philippe_vaillancourt@sympatico.ca. All 58 | complaints will be reviewed and investigated and will result in a response that 59 | is deemed necessary and appropriate to the circumstances. The project team is 60 | obligated to maintain confidentiality with regard to the reporter of an incident. 61 | Further details of specific enforcement policies may be posted separately. 62 | 63 | Project maintainers who do not follow or enforce the Code of Conduct in good 64 | faith may face temporary or permanent repercussions as determined by other 65 | members of the project's leadership. 66 | 67 | ## Attribution 68 | 69 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 70 | available at [http://contributor-covenant.org/version/1/4][version] 71 | 72 | [homepage]: http://contributor-covenant.org 73 | [version]: http://contributor-covenant.org/version/1/4/ 74 | -------------------------------------------------------------------------------- /images/Macao-logo-color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snowfrogdev/macao/bdad30aa05c615d9afa917d2cdba85d97393c4bd/images/Macao-logo-color.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "macao", 3 | "version": "2.0.1", 4 | "description": "A general purpose game playing A.I. framework based on the Monte Carlo tree search algorithm.", 5 | "keywords": [ 6 | "ai", 7 | "artificial intelligence", 8 | "macao", 9 | "macaojs", 10 | "macao.js", 11 | "monte carlo tree search", 12 | "mcts", 13 | "minimax", 14 | "general game playing", 15 | "game ai" 16 | ], 17 | "main": "dist/macao.umd.js", 18 | "module": "dist/macao.es5.js", 19 | "typings": "dist/types/macao.d.ts", 20 | "files": [ 21 | "dist" 22 | ], 23 | "author": "Philippe Vaillancourt ", 24 | "repository": { 25 | "type": "git", 26 | "url": "https://github.com/snowfrogdev/macao.git" 27 | }, 28 | "license": "MIT", 29 | "engines": { 30 | "node": ">=6.0.0" 31 | }, 32 | "scripts": { 33 | "lint": "tslint -t codeFrame 'src/**/*.ts' 'test/**/*.ts'", 34 | "prebuild": "rimraf dist", 35 | "build": "tsc --module commonjs && rollup -c rollup.config.ts && typedoc --out docs/api --target es6 --theme minimal --mode file src", 36 | "start": "rollup -c rollup.config.ts -w", 37 | "test": "jest", 38 | "test:watch": "jest --watch", 39 | "test:prod": "npm run lint && npm run test -- --coverage --no-cache", 40 | "deploy-docs": "ts-node tools/gh-pages-publish", 41 | "report-coverage": "cat ./coverage/lcov.info | coveralls", 42 | "commit": "git-cz", 43 | "semantic-release": "semantic-release", 44 | "semantic-release-prepare": "ts-node tools/semantic-release-prepare", 45 | "precommit": "lint-staged", 46 | "travis-deploy-once": "travis-deploy-once", 47 | "prepush": "npm run test:prod && npm run build", 48 | "commitmsg": "validate-commit-msg" 49 | }, 50 | "lint-staged": { 51 | "{src,test}/**/*.ts": [ 52 | "prettier --write --no-semi --single-quote", 53 | "git add" 54 | ] 55 | }, 56 | "config": { 57 | "commitizen": { 58 | "path": "node_modules/cz-conventional-changelog" 59 | }, 60 | "validate-commit-msg": { 61 | "types": "conventional-commit-types", 62 | "helpMessage": "Use \"npm run commit\" instead, we use conventional-changelog format :) (https://github.com/commitizen/cz-cli)" 63 | } 64 | }, 65 | "jest": { 66 | "transform": { 67 | ".(ts|tsx)": "/node_modules/ts-jest/preprocessor.js" 68 | }, 69 | "testRegex": "(/__tests__/.*|\\.(test|spec))\\.(ts|tsx|js)$", 70 | "moduleFileExtensions": [ 71 | "ts", 72 | "tsx", 73 | "js" 74 | ], 75 | "coveragePathIgnorePatterns": [ 76 | "/node_modules/", 77 | "/test/" 78 | ], 79 | "coverageThreshold": { 80 | "global": { 81 | "branches": 90, 82 | "functions": 95, 83 | "lines": 95, 84 | "statements": 95 85 | } 86 | }, 87 | "collectCoverage": true 88 | }, 89 | "release": { 90 | "verifyConditions": [ 91 | "@semantic-release/changelog", 92 | "@semantic-release/npm", 93 | "@semantic-release/github", 94 | "@semantic-release/git" 95 | ], 96 | "prepare": [ 97 | "@semantic-release/changelog", 98 | "@semantic-release/npm", 99 | "@semantic-release/git" 100 | ] 101 | }, 102 | "devDependencies": { 103 | "@semantic-release/changelog": "^2.0.2", 104 | "@semantic-release/git": "^5.0.0", 105 | "@types/jest": "^22.2.3", 106 | "@types/node": "^9.6.7", 107 | "colors": "^1.2.3", 108 | "commitizen": "^2.9.6", 109 | "coveralls": "^3.0.4", 110 | "cross-env": "^5.1.4", 111 | "cz-conventional-changelog": "^2.1.0", 112 | "husky": "^0.14.0", 113 | "jest": "^22.4.3", 114 | "lint-staged": "^7.0.5", 115 | "lodash.camelcase": "^4.3.0", 116 | "prettier": "^1.12.1", 117 | "prompt": "^1.0.0", 118 | "replace-in-file": "^3.4.0", 119 | "rimraf": "^2.6.1", 120 | "rollup": "^0.57.1", 121 | "rollup-plugin-commonjs": "^9.1.3", 122 | "rollup-plugin-json": "^2.3.0", 123 | "rollup-plugin-node-resolve": "^3.3.0", 124 | "rollup-plugin-sourcemaps": "^0.4.2", 125 | "rollup-plugin-typescript2": "^0.11.1", 126 | "semantic-release": "^15.13.16", 127 | "travis-deploy-once": "^4.4.1", 128 | "ts-jest": "^22.4.4", 129 | "ts-node": "^5.0.1", 130 | "tslint": "^5.8.0", 131 | "tslint-config-prettier": "^1.12.0", 132 | "tslint-config-standard": "^7.0.0", 133 | "typedoc": "^0.14.2", 134 | "typescript": "^2.8.3", 135 | "validate-commit-msg": "^2.12.2" 136 | }, 137 | "dependencies": {} 138 | } 139 | -------------------------------------------------------------------------------- /rollup.config.ts: -------------------------------------------------------------------------------- 1 | import resolve from 'rollup-plugin-node-resolve' 2 | import commonjs from 'rollup-plugin-commonjs' 3 | import sourceMaps from 'rollup-plugin-sourcemaps' 4 | import camelCase from 'lodash.camelcase' 5 | import typescript from 'rollup-plugin-typescript2' 6 | import json from 'rollup-plugin-json' 7 | 8 | const pkg = require('./package.json') 9 | 10 | const libraryName = 'macao' 11 | 12 | export default { 13 | input: `src/${libraryName}.ts`, 14 | output: [ 15 | { file: pkg.main, name: camelCase(libraryName), format: 'umd' }, 16 | { file: pkg.module, format: 'es' }, 17 | ], 18 | sourcemap: true, 19 | // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash') 20 | external: [], 21 | watch: { 22 | include: 'src/**', 23 | }, 24 | plugins: [ 25 | // Allow json resolution 26 | json(), 27 | // Compile TypeScript files 28 | typescript({ useTsconfigDeclarationDir: true }), 29 | // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs) 30 | commonjs(), 31 | // Allow node_modules resolution, so you can use 'external' to control 32 | // which external modules to include in the bundle 33 | // https://github.com/rollup/rollup-plugin-node-resolve#usage 34 | resolve(), 35 | 36 | // Resolve source maps to the original source 37 | sourceMaps(), 38 | ], 39 | } 40 | -------------------------------------------------------------------------------- /src/controller.ts: -------------------------------------------------------------------------------- 1 | import { Collection, HashTable, TranspositionTable } from './data-store' 2 | import { 3 | ApplyAction, 4 | CalculateReward, 5 | GenerateActions, 6 | Playerwise, 7 | StateIsTerminal, 8 | GameRules 9 | } from './entities' 10 | import { DefaultBackPropagate } from './mcts/back-propagate/back-propagate' 11 | import { DefaultExpand } from './mcts/expand/expand' 12 | import { DefaultMCTSFacade, MCTSFacade } from './mcts/mcts' 13 | import { DefaultBestChild, DefaultUCB1, UCB1 } from './mcts/select/best-child/best-child' 14 | import { DefaultSelect } from './mcts/select/select' 15 | import { DefaultSimulate } from './mcts/simulate/simulate' 16 | 17 | /** 18 | * 19 | * @hidden 20 | * @internal 21 | * @export 22 | * @class Controller 23 | * @template State 24 | * @template Action 25 | */ 26 | export class Controller { 27 | private mcts_!: MCTSFacade 28 | private duration_!: number 29 | private explorationParam_!: number 30 | private fpuParam_!: number 31 | private decayingParam_!: number 32 | private transpoTable_!: number | undefined 33 | 34 | constructor( 35 | funcs: GameRules, 36 | config: { 37 | duration: number 38 | explorationParam?: number 39 | fpuParam?: number 40 | decayingParam?: number 41 | transpoTable?: number 42 | } 43 | ) { 44 | this.duration_ = config.duration 45 | this.explorationParam_ = config.explorationParam || 1.414 46 | this.fpuParam_ = config.fpuParam || Infinity 47 | this.decayingParam_ = config.decayingParam || 1 48 | this.transpoTable_ = config.transpoTable 49 | 50 | this.init(funcs) 51 | } 52 | 53 | init(funcs: GameRules) { 54 | // This is where we bootstrap the library according to initialization options. 55 | let data: Collection 56 | if (this.transpoTable_) { 57 | data = new HashTable(this.transpoTable_) 58 | } else { 59 | data = new Map() 60 | } 61 | 62 | const transpositionTable = new TranspositionTable(data) 63 | const ucb1: UCB1 = new DefaultUCB1(this.explorationParam_) 64 | const bestChild = new DefaultBestChild(ucb1) 65 | 66 | const expand = new DefaultExpand(funcs.applyAction, funcs.generateActions, transpositionTable) 67 | 68 | const select = new DefaultSelect(funcs.stateIsTerminal, expand, bestChild, ucb1, this.fpuParam_) 69 | 70 | const simulate = new DefaultSimulate( 71 | funcs.stateIsTerminal, 72 | funcs.generateActions, 73 | funcs.applyAction, 74 | funcs.calculateReward 75 | ) 76 | 77 | const backPropagate = new DefaultBackPropagate(this.decayingParam_) 78 | this.mcts_ = new DefaultMCTSFacade( 79 | select, 80 | expand, 81 | simulate, 82 | backPropagate, 83 | bestChild, 84 | funcs.generateActions, 85 | transpositionTable, 86 | this.duration_, 87 | this.explorationParam_ 88 | ) 89 | } 90 | 91 | getAction(state: State, duration?: number): Promise { 92 | return this.mcts_.getAction(state, duration) 93 | } 94 | 95 | getActionSync(state: State, duration?: number): Action { 96 | return this.mcts_.getActionSync(state, duration) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/data-store.ts: -------------------------------------------------------------------------------- 1 | import { DataGateway } from './mcts/mcts' 2 | import { MCTSState } from './entities' 3 | 4 | /** 5 | * 6 | * @hidden 7 | * @internal 8 | * @template State 9 | * @template Action 10 | */ 11 | export interface Collection { 12 | get(key: string): MCTSState | undefined 13 | set(key: string, value: MCTSState): this 14 | } 15 | 16 | /** 17 | * 18 | * @hidden 19 | * @internal 20 | * @template State 21 | * @template Action 22 | */ 23 | export class TranspositionTable implements DataGateway { 24 | constructor(private data_: Collection) {} 25 | 26 | get(key: State): MCTSState | undefined { 27 | const stringKey = JSON.stringify(key) 28 | return this.data_.get(stringKey) 29 | } 30 | 31 | set(key: State, value: MCTSState): this { 32 | const stringKey = JSON.stringify(key) 33 | this.data_.set(stringKey, value) 34 | return this 35 | } 36 | } 37 | 38 | /** 39 | * 40 | * @hidden 41 | * @internal 42 | * @template Key 43 | * @template Value 44 | */ 45 | export class HashTable implements Collection { 46 | private buckets_: Map>[] = [] 47 | constructor(private bucketCount_: number) { 48 | for (let i = 0; i < this.bucketCount_; i++) { 49 | this.buckets_.push(new Map()) 50 | } 51 | } 52 | 53 | hashFunction_(key: string) { 54 | let hash = 0 55 | if (key.length === 0) return hash 56 | for (let i = 0; i < key.length; i++) { 57 | hash = (hash << 5) - hash 58 | hash = hash + key.charCodeAt(i) 59 | hash = hash & hash // Convert to 32bit integer 60 | } 61 | return Math.abs(hash) 62 | } 63 | 64 | getBucketIndex_(key: string) { 65 | return this.hashFunction_(key) % this.bucketCount_ 66 | } 67 | 68 | getBucket_(key: string) { 69 | return this.buckets_[this.getBucketIndex_(key)] 70 | } 71 | 72 | set(key: string, value: MCTSState): this { 73 | this.getBucket_(key).set(key, value) 74 | return this 75 | } 76 | get(lookupKey: string) { 77 | return this.getBucket_(lookupKey).get(lookupKey) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/entities.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * `Playerwise` is an interface made to extend generic `State` objects used in 3 | * the [[GameRules]] interface. It is meant to insure that, even though the shape 4 | * and implementation of the `State` object is left up to the user, it should 5 | * atleast have a `player` property. 6 | */ 7 | export interface Playerwise { 8 | player: number 9 | } 10 | 11 | /** 12 | * `GenerateActions` is a type of function that you provide that takes in a `State` 13 | * as an argument and returns an `Array` of possible `Action`s. 14 | * 15 | * ### Example 16 | * ```javascript 17 | * function(state) { 18 | * const possibleActions = []; 19 | * 20 | * // Some kind of algortihm that you implement and 21 | * // pushes all possible Action(s) into an array. 22 | * 23 | * return possibleActions; 24 | * } 25 | * ``` 26 | * @param State An object representing the state of the game. 27 | * @param Action An object representing an action in the game. 28 | */ 29 | export interface GenerateActions { 30 | (state: State): Action[] 31 | } 32 | 33 | /** 34 | * `ApplyAction` is a type of function that you provide that takes in a `State` 35 | * and an `Action` as arguments. It applies the `Action` to the `State` and returns 36 | * a new `State`. 37 | * 38 | * **IMPORTANT** 39 | * Make sure that the function indeed returns a NEW State and does not simply 40 | * mutate the provided State. 41 | * 42 | * ### Example 43 | * ```javascript 44 | * function(state, action) { 45 | * let newState; 46 | * 47 | * // Apply the action to state and create a new State object. 48 | * 49 | * return newState; 50 | * } 51 | * ``` 52 | * @param State An object representing the state of the game. 53 | * @param Action An object representing an action in the game. 54 | */ 55 | export interface ApplyAction { 56 | (state: State, action: Action): State 57 | } 58 | 59 | /** 60 | * `StateIsTerminal` is a type of function that you provide that takes in a `State` 61 | * as an argument and returns `true` if the game is over and `false` otherwise. 62 | * 63 | * ### Example 64 | * ```javascript 65 | * function(state) { 66 | * if (gameIsADraw(state) || gamesIsWon(state)) return true; 67 | * 68 | * return false; 69 | * } 70 | * ``` 71 | * @param State An object representing the state of the game. 72 | */ 73 | export interface StateIsTerminal { 74 | (state: State): boolean 75 | } 76 | 77 | /** 78 | * `CalculateReward` is a type of function that you provide that takes in a `State` 79 | * and a `number` representing the player, as arguments. Given the game `State`, 80 | * it calculates a reward for the player and returns that reward as a `number`. 81 | * 82 | * Normaly, you would want a win to return 1, a loss to return -1 and a draw 83 | * to return 0 but you can decide on a different reward scheme. 84 | * 85 | * ### Example 86 | * ```javascript 87 | * function(state, player) { 88 | * if (hasWon(state, player)) return 1; 89 | * 90 | * if (isDraw(state)) return 0; 91 | * 92 | * return -1; 93 | * } 94 | * ``` 95 | * @param State An object representing the state of the game. 96 | */ 97 | export interface CalculateReward { 98 | (state: State, player: number): number 99 | } 100 | 101 | /** 102 | * `GameRules` is an interface for a class containing all the game functions provided 103 | * by the user. 104 | * @hidden 105 | * @internal 106 | * @param State An object representing the state of the game. 107 | * @param Action An object representing an action in the game. 108 | */ 109 | export interface GameRules { 110 | generateActions: GenerateActions 111 | applyAction: ApplyAction 112 | stateIsTerminal: StateIsTerminal 113 | calculateReward: CalculateReward 114 | } 115 | 116 | /** 117 | * The `DefaultGameRules` class implements [[GameRules]] and contains all of the game 118 | * functions supplied by the user. 119 | * @hidden 120 | * @internal 121 | * @param State An object representing the state of the game. 122 | * @param Action An object representing an action in the game. 123 | */ 124 | export class DefaultGameRules 125 | implements GameRules { 126 | private generateActions_!: GenerateActions 127 | private applyAction_!: ApplyAction 128 | private stateIsTerminal_!: StateIsTerminal 129 | private calculateReward_!: CalculateReward 130 | /** 131 | * Creates an instance of DefaultGameRules. 132 | * @param funcs an object containing all the of the game functions. 133 | */ 134 | constructor(funcs: { 135 | generateActions: GenerateActions 136 | applyAction: ApplyAction 137 | stateIsTerminal: StateIsTerminal 138 | calculateReward: CalculateReward 139 | }) { 140 | this.generateActions = funcs.generateActions 141 | this.applyAction = funcs.applyAction 142 | this.stateIsTerminal = funcs.stateIsTerminal 143 | this.calculateReward = funcs.calculateReward 144 | } 145 | 146 | get generateActions(): GenerateActions { 147 | return this.generateActions_ 148 | } 149 | set generateActions(value: GenerateActions) { 150 | if (typeof value !== 'function' || value.length !== 1) { 151 | throw new TypeError('Expected generateActions to be a function that takes one argument.') 152 | } 153 | this.generateActions_ = value 154 | } 155 | 156 | get applyAction(): ApplyAction { 157 | return this.applyAction_ 158 | } 159 | set applyAction(value: ApplyAction) { 160 | if (typeof value !== 'function' || value.length !== 2) { 161 | throw new TypeError('Expected applyAction to be a function that takes two arguments.') 162 | } 163 | this.applyAction_ = value 164 | } 165 | 166 | get stateIsTerminal(): StateIsTerminal { 167 | return this.stateIsTerminal_ 168 | } 169 | set stateIsTerminal(value: StateIsTerminal) { 170 | if (typeof value !== 'function' || value.length !== 1) { 171 | throw new TypeError('Expected stateIsTerminal to be a function that takes one argument.') 172 | } 173 | this.stateIsTerminal_ = value 174 | } 175 | 176 | get calculateReward(): CalculateReward { 177 | return this.calculateReward_ 178 | } 179 | set calculateReward(value: CalculateReward) { 180 | if (typeof value !== 'function' || value.length !== 2) { 181 | throw new TypeError('Expected calculateReward to be a function that takes two arguments.') 182 | } 183 | this.calculateReward_ = value 184 | } 185 | } 186 | 187 | /** 188 | * 189 | * @hidden 190 | * @internal 191 | * @param State An object representing the state of the game. 192 | */ 193 | export class MCTSNode { 194 | private possibleActionsLeftToExpand_: Action[] 195 | private children_: MCTSNode[] = [] 196 | constructor( 197 | private mctsState_: MCTSState, 198 | possibleActions: Action[], 199 | private parent_?: MCTSNode, 200 | private action_?: Action 201 | ) { 202 | this.possibleActionsLeftToExpand_ = possibleActions 203 | } 204 | 205 | get mctsState(): MCTSState { 206 | return this.mctsState_ 207 | } 208 | 209 | get possibleActionsLeftToExpand(): Action[] { 210 | return this.possibleActionsLeftToExpand_ 211 | } 212 | 213 | get action(): Action | undefined { 214 | return this.action_ 215 | } 216 | 217 | get children(): MCTSNode[] { 218 | return this.children_ 219 | } 220 | 221 | get parent(): MCTSNode | undefined { 222 | return this.parent_ 223 | } 224 | 225 | addChild( 226 | mctsState: MCTSState, 227 | possibleActions: Action[], 228 | action: Action 229 | ): MCTSNode { 230 | const node = new MCTSNode(mctsState, possibleActions, this, action) 231 | this.children_.push(node) 232 | return node 233 | } 234 | 235 | isNotFullyExpanded(): boolean { 236 | return this.possibleActionsLeftToExpand_.length > 0 237 | } 238 | } 239 | 240 | /** 241 | * 242 | * @hidden 243 | * @internal 244 | * @param State An object representing the state of the game. 245 | */ 246 | export class MCTSState { 247 | private reward_: number = 0 248 | private visits_: number = 0 249 | constructor(private state_: State) {} 250 | 251 | get reward(): number { 252 | return this.reward_ 253 | } 254 | set reward(value: number) { 255 | this.reward_ = value 256 | } 257 | 258 | get visits(): number { 259 | return this.visits_ 260 | } 261 | set visits(value: number) { 262 | this.visits_ = value 263 | } 264 | 265 | get state(): State { 266 | return this.state_ 267 | } 268 | } 269 | -------------------------------------------------------------------------------- /src/macao.ts: -------------------------------------------------------------------------------- 1 | import { Controller } from './controller' 2 | import { 3 | ApplyAction, 4 | CalculateReward, 5 | GenerateActions, 6 | Playerwise, 7 | StateIsTerminal, 8 | DefaultGameRules 9 | } from './entities' 10 | 11 | /** 12 | * The `Macao` class represents a Monte Carlo tree search that can be easily 13 | * created. It provides, through it's [[getAction]] and [[getActionSync]] methods, the results of running 14 | * the algorithm. 15 | * 16 | * ```javascript 17 | * const funcs = { 18 | * generateActions, 19 | * applyAction, 20 | * stateIsTerminal, 21 | * calculateReward, 22 | * } 23 | * 24 | * const config = { 25 | * duration: 30, 26 | * explorationParam: 1.414 27 | * } 28 | * 29 | * const macao = new Macao(funcs, config); 30 | * 31 | * const action = macao.getActionSync(state); 32 | * ``` 33 | * @param State Generic Type representing a game state object. 34 | * @param Action Generic Type representing an action in the game. 35 | */ 36 | export class Macao { 37 | /** 38 | * @hidden 39 | * @internal 40 | */ 41 | private controller_: Controller 42 | 43 | /** 44 | * Creates an instance of Macao. 45 | * 46 | * ```javascript 47 | * const funcs = { 48 | * generateActions, 49 | * applyAction, 50 | * stateIsTerminal, 51 | * calculateReward, 52 | * } 53 | * 54 | * const config = { 55 | * duration: 30, 56 | * explorationParam: 1.414 57 | * } 58 | * 59 | * const macao = new Macao(funcs, config); 60 | * ``` 61 | * @param {object} funcs - Contains all the functions implementing the game's rules. 62 | * @param {GenerateActions} funcs.generateActions 63 | * @param {ApplyAction} funcs.applyAction 64 | * @param {StateIsTerminal} funcs.stateIsTerminal 65 | * @param {CalculateReward} funcs.calculateReward 66 | * @param {object} config Configuration options 67 | * @param {number} config.duration Run time of the algorithm, in milliseconds. 68 | * @param {number | undefined} config.explorationParam The exploration parameter constant. 69 | * Used in [UCT](https://en.wikipedia.org/wiki/Monte_Carlo_tree_search). Defaults to 1.414. 70 | * @param {number | undefined} config.fpuParam The First play urgency parameter. Used to encourage 71 | * early exploitation. Defaults to `Infinity`. 72 | * See [Exploration exploitation in Go: 73 | * UCT for Monte-Carlo Go](https://hal.archives-ouvertes.fr/hal-00115330/document) 74 | * @param {number | undefined} config.decayingParam The multiplier by which to decay the reward 75 | * in the backpropagtion phase. Defaults to 1. 76 | * @param {number | undefined} config.transpoTable The number of buckets in the Transoposition Hash Table. 77 | */ 78 | constructor( 79 | funcs: { 80 | generateActions: GenerateActions 81 | applyAction: ApplyAction 82 | stateIsTerminal: StateIsTerminal 83 | calculateReward: CalculateReward 84 | }, 85 | config: { 86 | duration: number 87 | explorationParam?: number 88 | fpuParam?: number 89 | decayingParam?: number 90 | /** 91 | * The number of buckets in the Transposition Hash table 92 | */ 93 | transpoTable?: number 94 | } 95 | ) { 96 | const gameRules = new DefaultGameRules(funcs) 97 | this.controller_ = new Controller(gameRules, config) 98 | } 99 | /** 100 | * Runs the Monte Carlo Tree search algorithm synchronously and returns the estimated 101 | * best action given the current state of the game. This method will block the event 102 | * loop and is suitable if you only wish to run the algorithm for a very short amount 103 | * of time (we suggest 33 milliseconds or less) or if you are running the algorithm in 104 | * an application that does not have a UI. For other applications, you should use 105 | * `Macao.getAction()` instead. 106 | * 107 | * ```javascript 108 | * const action = macao.getActionSync(state); 109 | * ``` 110 | * @param {State} state Object representing the game state. 111 | * @param {number | undefined} duration Run time of the algorithm, in milliseconds. 112 | * @returns {Action} 113 | */ 114 | getAction(state: State, duration?: number): Promise { 115 | return this.controller_.getAction(state, duration) 116 | } 117 | 118 | /** 119 | * Runs the Monte Carlo Tree search algorithm asynchronously and returns a promise 120 | * that resolves to the estimated best action given the current state of the game. 121 | * 122 | * ```javascript 123 | * macao.getAction(state) 124 | * .then(action => { 125 | * // Do stuff with the action 126 | * }); 127 | * ``` 128 | * or 129 | * 130 | * ```javascript 131 | * const someAsyncFunction = async() => { 132 | * const action = await macao.getAction(state); 133 | * } 134 | * ``` 135 | * @param {State} state Object representing the game state. 136 | * @param {number | undefined} duration Run time of the algorithm, in milliseconds. 137 | * @returns {Promise} 138 | */ 139 | getActionSync(state: State, duration?: number): Action { 140 | return this.controller_.getActionSync(state, duration) 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/mcts/back-propagate/back-propagate.ts: -------------------------------------------------------------------------------- 1 | import { MCTSNode } from '../../entities' 2 | 3 | /** 4 | * 5 | * @hidden 6 | * @internal 7 | * @export 8 | * @interface BackPropagate 9 | * @template State 10 | * @template Action 11 | */ 12 | export interface BackPropagate { 13 | run: (node: MCTSNode | undefined, score: number) => void 14 | } 15 | 16 | /** 17 | * 18 | * @hidden 19 | * @internal 20 | * @export 21 | * @class DefaultBackPropagate 22 | * @implements {BackPropagate} 23 | * @template State 24 | * @template Action 25 | */ 26 | export class DefaultBackPropagate implements BackPropagate { 27 | constructor(private decayingParam: number) {} 28 | run(node: MCTSNode | undefined, score: number): void { 29 | while (node) { 30 | node.mctsState.visits++ 31 | node.mctsState.reward += score 32 | 33 | // Flip reward 34 | score *= -1 35 | 36 | // Decay reward 37 | score *= this.decayingParam 38 | 39 | node = node.parent 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/mcts/expand/expand.ts: -------------------------------------------------------------------------------- 1 | import { ApplyAction, GenerateActions, MCTSNode, MCTSState, Playerwise } from '../../entities' 2 | import { spliceRandom } from '../../utils' 3 | import { DataGateway } from '../mcts' 4 | 5 | /** 6 | * The Expand interface represents the Expansion part of the Monte Carlo Tree 7 | * Search algorithm. This part of the algorithm deals with adding a children Node 8 | * to a previously selected Node. 9 | * @hidden 10 | * @internal 11 | */ 12 | export interface Expand { 13 | run: (node: MCTSNode) => MCTSNode 14 | } 15 | 16 | /** 17 | * The DefaultExpand class provides the standard Monte Carlo Tree Search algorithm 18 | * with the expansion phase. Through it's [[run]] method, when supplied with a tree 19 | * node, it will expand the tree by adding a children node. 20 | * @hidden 21 | * @internal 22 | */ 23 | export class DefaultExpand implements Expand { 24 | constructor( 25 | private applyAction_: ApplyAction, 26 | private generateActions_: GenerateActions, 27 | private dataStore_: DataGateway 28 | ) {} 29 | 30 | run(node: MCTSNode): MCTSNode { 31 | const action = spliceRandom(node.possibleActionsLeftToExpand) 32 | const state = this.applyAction_(node.mctsState.state, action) 33 | // Check to see if state is already in Map 34 | let mctsState = this.dataStore_.get(state) 35 | // If it isn't, create a new MCTSState and store it in the map 36 | if (!mctsState) { 37 | mctsState = new MCTSState(state) 38 | this.dataStore_.set(state, mctsState) 39 | } 40 | const child = node.addChild(mctsState, this.generateActions_(state), action) 41 | return child 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/mcts/mcts.ts: -------------------------------------------------------------------------------- 1 | import { 2 | MCTSNode, 3 | StateIsTerminal, 4 | Playerwise, 5 | ApplyAction, 6 | MCTSState, 7 | GenerateActions, 8 | CalculateReward 9 | } from '../entities' 10 | import { Select } from './select/select' 11 | import { BestChild } from './select/best-child/best-child' 12 | import { Expand } from './expand/expand' 13 | import { Simulate } from './simulate/simulate' 14 | import { BackPropagate } from './back-propagate/back-propagate' 15 | import { spliceRandom, loopFor, now } from '../utils' 16 | 17 | /** 18 | * 19 | * @hidden 20 | * @internal 21 | * @export 22 | * @interface DataGateway 23 | * @template Key 24 | * @template Value 25 | */ 26 | export interface DataGateway { 27 | get: (key: State) => MCTSState | undefined 28 | set: (key: State, value: MCTSState) => this 29 | } 30 | 31 | /** 32 | * 33 | * @hidden 34 | * @internal 35 | * @export 36 | * @interface MCTSFacade 37 | * @template State 38 | * @template Action 39 | */ 40 | export interface MCTSFacade { 41 | getAction: (state: State, duration?: number) => Promise 42 | getActionSync: (state: State, duration?: number) => Action 43 | } 44 | 45 | /** 46 | * 47 | * @hidden 48 | * @internal 49 | * @export 50 | * @class DefaultMCTSFacade 51 | * @implements {MCTSFacade} 52 | * @template State 53 | * @template Action 54 | */ 55 | export class DefaultMCTSFacade 56 | implements MCTSFacade { 57 | /** 58 | * Creates an instance of DefaultMCTSFacade. 59 | */ 60 | constructor( 61 | private select_: Select, 62 | private expand_: Expand, 63 | private simulate_: Simulate, 64 | private backPropagate_: BackPropagate, 65 | private bestChild_: BestChild, 66 | private generateActions_: GenerateActions, 67 | private dataStore_: DataGateway, 68 | private duration_: number, 69 | private explorationParam_: number 70 | ) {} 71 | 72 | async getAction(state: State, duration?: number): Promise { 73 | const rootNode = this.createRootNode_(state) 74 | const startTime = now() 75 | const simulation = new Promise(resolve => { 76 | const doChunk = () => { 77 | if (now() - startTime >= (duration || this.duration_)) { 78 | return resolve() 79 | } 80 | loopFor(30).milliseconds(() => { 81 | const node = this.select_.run(rootNode) 82 | const score = this.simulate_.run(node.mctsState.state) 83 | this.backPropagate_.run(node, score) 84 | }) 85 | setTimeout(doChunk) 86 | } 87 | doChunk() 88 | }) 89 | 90 | await simulation 91 | 92 | const bestChild = this.bestChild_.run(rootNode, true) 93 | return bestChild!.action as Action 94 | } 95 | 96 | getActionSync(state: State, duration?: number): Action { 97 | const rootNode = this.createRootNode_(state) 98 | loopFor(duration || this.duration_).milliseconds(() => { 99 | const node = this.select_.run(rootNode) 100 | const score = this.simulate_.run(node.mctsState.state) 101 | this.backPropagate_.run(node, score) 102 | }) 103 | const bestChild = this.bestChild_.run(rootNode, true) 104 | return bestChild!.action as Action 105 | } 106 | 107 | private createRootNode_(state: State): MCTSNode { 108 | // Check to see if state is already in DataStore 109 | let mctsState = this.dataStore_.get(state) 110 | // If it isn't, create a new MCTSState and store it 111 | if (!mctsState) { 112 | mctsState = new MCTSState(state) 113 | this.dataStore_.set(state, mctsState) 114 | } 115 | // Create new MCTSNode 116 | const node = new MCTSNode(mctsState, this.generateActions_(state)) 117 | return node 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/mcts/select/best-child/best-child.ts: -------------------------------------------------------------------------------- 1 | import { MCTSNode, MCTSState, Playerwise } from '../../../entities' 2 | 3 | /** 4 | * 5 | * @hidden 6 | * @internal 7 | * @export 8 | * @interface BestChild 9 | * @template State 10 | * @template Action 11 | */ 12 | export interface BestChild { 13 | run: (node: MCTSNode, exploit?: boolean) => MCTSNode | undefined 14 | } 15 | 16 | /** 17 | * 18 | * @hidden 19 | * @internal 20 | * @export 21 | * @class DefaultBestChild 22 | * @implements {BestChild} 23 | * @template State 24 | * @template Action 25 | */ 26 | export class DefaultBestChild 27 | implements BestChild { 28 | /** 29 | * Creates an instance of DefaultBestChild. 30 | * @param {UCB1} UCB1_ 31 | * @memberof DefaultBestChild 32 | */ 33 | constructor(private UCB1_: UCB1) {} 34 | 35 | /** 36 | * 37 | * 38 | * @param {MCTSNode} node 39 | * @param {number} explorationParam 40 | * @returns {MCTSNode} 41 | * @memberof DefaultBestChild 42 | */ 43 | run(node: MCTSNode, exploit = false): MCTSNode | undefined { 44 | if (!node.children.length) { 45 | return undefined 46 | } 47 | const sumChildVisits = node.children.reduce((p, c) => p + c.mctsState.visits, 0) 48 | const selectedNode = node.children.reduce((p, c) => { 49 | return this.UCB1_.run(sumChildVisits, p.mctsState, exploit) > 50 | this.UCB1_.run(sumChildVisits, c.mctsState, exploit) 51 | ? p 52 | : c 53 | }) 54 | 55 | return selectedNode 56 | } 57 | } 58 | 59 | /** 60 | * 61 | * @hidden 62 | * @internal 63 | * @export 64 | * @interface UCB1 65 | * @template State 66 | * @template Action 67 | */ 68 | export interface UCB1 { 69 | run(sumChildVisits: number, child: MCTSState, exploit?: boolean): number 70 | } 71 | 72 | /** 73 | * 74 | * @hidden 75 | * @internal 76 | * @export 77 | * @class DefaultUCB1 78 | * @implements {UCB1} 79 | * @template State 80 | * @template Action 81 | */ 82 | export class DefaultUCB1 implements UCB1 { 83 | constructor(private explorationParam_: number) {} 84 | /** 85 | * 86 | * 87 | * @param {MCTSState} parent 88 | * @param {MCTSState} child 89 | * @param {number} explorationParam 90 | * @returns {number} 91 | * @memberof DefaultUCB1 92 | */ 93 | run(sumChildVisits: number, child: MCTSState, exploit = false): number { 94 | if (exploit) this.explorationParam_ = 0 95 | const exploitationTerm = child.reward / child.visits 96 | const explorationTerm = Math.sqrt(Math.log(sumChildVisits) / child.visits) 97 | return exploitationTerm + this.explorationParam_ * explorationTerm 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/mcts/select/select.ts: -------------------------------------------------------------------------------- 1 | import { MCTSNode, Playerwise, StateIsTerminal } from '../../entities' 2 | import { Expand } from '../expand/expand' 3 | import { BestChild, UCB1 } from './best-child/best-child' 4 | 5 | /** 6 | * The Select interface represents the Selection part of the Monte Carlo Tree 7 | * Search algorithm. This part of the algorithm deals with choosing which node 8 | * in the tree to run a simulation on. 9 | * @hidden 10 | * @internal 11 | */ 12 | export interface Select { 13 | run: (node: MCTSNode) => MCTSNode 14 | } 15 | 16 | /** 17 | * The DefaultSelect class provides the standard Monte Carlo Tree Search algorithm 18 | * with the selection phase. Through it's [[run]] method, when supplied with a tree 19 | * node, it will provide another tree node from which to run a simulation. 20 | * @hidden 21 | * @internal 22 | */ 23 | export class DefaultSelect implements Select { 24 | constructor( 25 | private stateIsTerminal_: StateIsTerminal, 26 | private expand_: Expand, 27 | private bestChild_: BestChild, 28 | private ucb1_: UCB1, 29 | private fpuParam_: number 30 | ) {} 31 | 32 | run(node: MCTSNode): MCTSNode { 33 | while (!this.stateIsTerminal_(node.mctsState.state)) { 34 | const child = this.bestChild_.run(node) 35 | if (!child) return this.expand_.run(node) 36 | if (node.isNotFullyExpanded()) { 37 | const sumChildVisits = node.children.reduce((p, c) => p + c.mctsState.visits, 0) 38 | const ucb1 = this.ucb1_.run(sumChildVisits, child.mctsState) 39 | if (ucb1 < this.fpuParam_) { 40 | return this.expand_.run(node) 41 | } 42 | } 43 | node = child 44 | } 45 | return node 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/mcts/simulate/simulate.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ApplyAction, 3 | CalculateReward, 4 | GenerateActions, 5 | Playerwise, 6 | StateIsTerminal 7 | } from '../../entities' 8 | import { getRandomIntInclusive } from '../../utils' 9 | 10 | /** 11 | * The Simulate interface provides a playthrough of the game and is 12 | * a part of the Monte Carlo Tree Search algorithm. 13 | * @hidden 14 | * @internal 15 | */ 16 | export interface Simulate { 17 | run: (state: State) => number 18 | } 19 | 20 | /** 21 | * The DefaultSimulate class provides, trough it's [[run]] method, 22 | * a standard playthrough of the game where each move is selected entirely at random. 23 | * @hidden 24 | * @internal 25 | */ 26 | export class DefaultSimulate implements Simulate { 27 | /** 28 | * Creates an instance of DefaultSimulate. 29 | * @param {StateIsTerminal} stateIsTerminal_ 30 | * @param {GenerateActions} generateActions_ 31 | * @param {ApplyAction} applyAction_ 32 | * @param {CalculateReward} calculateReward_ 33 | * @memberof DefaultSimulate 34 | */ 35 | constructor( 36 | private stateIsTerminal_: StateIsTerminal, 37 | private generateActions_: GenerateActions, 38 | private applyAction_: ApplyAction, 39 | private calculateReward_: CalculateReward 40 | ) {} 41 | 42 | /** 43 | * The `run` method of the [[DefaultSimulate]] class runs a standard, 44 | * entirely random, simulation of the game and returns a number representing 45 | * the result of the simulation from the perspective of the player who's just 46 | * played a move and is now waiting for his opponent's turn. 47 | * @param {State} state An object representing the state of the game. 48 | * @returns {number} 49 | * @memberof DefaultSimulate 50 | */ 51 | run(state: State): number { 52 | const player = state.player 53 | while (!this.stateIsTerminal_(state)) { 54 | // Generate possible actions 55 | const actions = this.generateActions_(state) 56 | 57 | // Select an action at random 58 | const index = getRandomIntInclusive(0, actions.length - 1) 59 | const action = actions[index] 60 | 61 | // Apply action and create new state 62 | state = this.applyAction_(state, action) 63 | } 64 | return this.calculateReward_(state, player) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * When running in browsers supporting performance.now(), or in Node.js, this function will 3 | * return the time elapsed, in milliseconds, since the time origin. If running 4 | * in environments that don't support performance.now() it will use Date.now(); 5 | * @hidden 6 | * @internal 7 | * @returns {number} - The returned value represents the time elapsed, in milliseconds, since the 8 | * time origin or since the UNIX epoch. 9 | */ 10 | export let now: () => number 11 | try { 12 | if (typeof window !== 'undefined') { 13 | now = performance.now 14 | } else { 15 | const { performance } = require('perf_hooks') 16 | now = performance.now 17 | } 18 | } catch { 19 | now = Date.now 20 | } 21 | 22 | /** 23 | * A function that return a random whole number between 24 | * `min` and `max`. 25 | * @hidden 26 | * @internal 27 | * @param {number} min 28 | * @param {number} max 29 | * @returns {number} 30 | */ 31 | export const getRandomIntInclusive = (min: number, max: number): number => { 32 | min = Math.ceil(min) 33 | max = Math.floor(max) 34 | return Math.floor(Math.random() * (max - min + 1)) + min // The maximum is inclusive and the minimum is inclusive 35 | } 36 | 37 | /** 38 | * A function that removes and returns a random element from an `arrray`. 39 | * @hidden 40 | * @internal 41 | * @template T 42 | * @param {T[]} array 43 | * @returns {T} 44 | */ 45 | export const spliceRandom = (array: T[]): T => { 46 | const index = getRandomIntInclusive(0, array.length - 1) 47 | return array.splice(index, 1)[0] 48 | } 49 | 50 | /** 51 | * A function to make looping for a specific amount of time 52 | * or a specific amount of loops, easier. Simply return `true` 53 | * to break out of the loop. 54 | * 55 | * ### Example 56 | * ```javascript 57 | * // Loop for 2 seconds 58 | * loopFor(2).seconds(() => { 59 | * // Things to do in a loop. 60 | * }); 61 | * 62 | * // Loop for 50 turns 63 | * loopFor(50).turns(() => { 64 | * // Things to do in a loop. 65 | * if (someCondition) { 66 | * // break out of loop 67 | * return true; 68 | * } 69 | * }); 70 | * ``` 71 | * @hidden 72 | * @internal 73 | * @param {number} time 74 | * @returns 75 | */ 76 | export const loopFor = (time: number) => { 77 | return { 78 | milliseconds: (callback: () => any) => { 79 | const start = now() 80 | while (now() - start < time) { 81 | if (callback()) break 82 | } 83 | }, 84 | seconds: (callback: () => any) => { 85 | const start = now() 86 | const t = time * 1000 87 | while (now() - start < t) { 88 | if (callback()) break 89 | } 90 | }, 91 | turns: (callback: () => any) => { 92 | while (time > 0) { 93 | if (callback()) break 94 | time-- 95 | } 96 | } 97 | } 98 | } 99 | 100 | /** 101 | * Function to get the average of a numbers array 102 | * @hidden 103 | * @internal 104 | * @param {number[]} arr 105 | */ 106 | const arrAvg = (arr: number[]) => arr.reduce((a, b) => a + b, 0) / arr.length 107 | -------------------------------------------------------------------------------- /test/data-store.test.ts: -------------------------------------------------------------------------------- 1 | import { HashTable } from '../src/data-store' 2 | import { MCTSState } from '../src/entities' 3 | 4 | describe('The Hashtable instance', () => { 5 | it('stores and retrieves elements', () => { 6 | const table = new HashTable(100) 7 | const mctsState = new MCTSState({ board: '01010101' }) 8 | const key = 'magickey123' 9 | table.set(key, mctsState) 10 | expect(table.get(key)).toBe(mctsState) 11 | expect(table.get('someOtherKey')).not.toBeDefined() 12 | }) 13 | }) 14 | -------------------------------------------------------------------------------- /test/entities.test.ts: -------------------------------------------------------------------------------- 1 | import { DefaultGameRules } from './../src/entities' 2 | describe('The GameRules instance', () => { 3 | describe('when being created', () => { 4 | let funcs 5 | beforeEach(() => { 6 | funcs = { 7 | generateActions: state => state, 8 | applyAction: (state, action) => state, 9 | stateIsTerminal: state => true, 10 | calculateReward: (state, player) => state 11 | } 12 | }) 13 | it('should validate that generateActions is a function that takes one argument', () => { 14 | expect(new DefaultGameRules(funcs).generateActions).toBe(funcs.generateActions) 15 | funcs.generateActions = '' 16 | expect(() => new DefaultGameRules(funcs)).toThrow( 17 | 'Expected generateActions to be a function that takes one argument.' 18 | ) 19 | funcs.generateActions = (state, extraParam) => state 20 | expect(() => new DefaultGameRules(funcs)).toThrow( 21 | 'Expected generateActions to be a function that takes one argument.' 22 | ) 23 | }) 24 | it('should validate that applyAction is a function that takes two arguments', () => { 25 | expect(new DefaultGameRules(funcs).applyAction).toBe(funcs.applyAction) 26 | funcs.applyAction = '' 27 | expect(() => new DefaultGameRules(funcs)).toThrow( 28 | 'Expected applyAction to be a function that takes two arguments.' 29 | ) 30 | funcs.generateActions = state => state 31 | expect(() => new DefaultGameRules(funcs)).toThrow( 32 | 'Expected applyAction to be a function that takes two arguments.' 33 | ) 34 | }) 35 | it('should validate that stateIsTerminal is a function that takes one argument', () => { 36 | expect(new DefaultGameRules(funcs).stateIsTerminal).toBe(funcs.stateIsTerminal) 37 | funcs.stateIsTerminal = '' 38 | expect(() => new DefaultGameRules(funcs)).toThrow( 39 | 'Expected stateIsTerminal to be a function that takes one argument.' 40 | ) 41 | funcs.stateIsTerminal = (state, extraParam) => state 42 | expect(() => new DefaultGameRules(funcs)).toThrow( 43 | 'Expected stateIsTerminal to be a function that takes one argument.' 44 | ) 45 | }) 46 | it('should validate that calculateReward is a function that takes two arguments', () => { 47 | expect(new DefaultGameRules(funcs).calculateReward).toBe(funcs.calculateReward) 48 | funcs.calculateReward = '' 49 | expect(() => new DefaultGameRules(funcs)).toThrow( 50 | 'Expected calculateReward to be a function that takes two arguments.' 51 | ) 52 | funcs.calculateReward = (state, player, extraParam) => state 53 | expect(() => new DefaultGameRules(funcs)).toThrow( 54 | 'Expected calculateReward to be a function that takes two arguments.' 55 | ) 56 | }) 57 | }) 58 | }) 59 | -------------------------------------------------------------------------------- /test/macao.test.ts: -------------------------------------------------------------------------------- 1 | import { Macao } from '../src/macao' 2 | import { ticTacToeFuncs, TicTacToeState } from './tic-tac-toe/tic-tac-toe' 3 | 4 | const macao = new Macao( 5 | { 6 | stateIsTerminal: ticTacToeFuncs.stateIsTerminal, 7 | generateActions: ticTacToeFuncs.generateActions, 8 | applyAction: ticTacToeFuncs.applyAction, 9 | calculateReward: ticTacToeFuncs.calculateReward 10 | }, 11 | { duration: 100 } 12 | ) 13 | describe('The Macao instance', () => { 14 | describe('when created', () => { 15 | it('should be an instance of Macao', () => { 16 | expect(macao).toBeInstanceOf(Macao) 17 | }) 18 | }) 19 | 20 | describe('when calling getActionSync', () => { 21 | it('should return something', () => { 22 | const ticTacToeBoard = [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 23 | const state: TicTacToeState = { 24 | board: ticTacToeBoard, 25 | player: 1 26 | } 27 | expect(macao.getActionSync(state)).toBeDefined() 28 | }) 29 | }) 30 | }) 31 | -------------------------------------------------------------------------------- /test/mcts.test.ts: -------------------------------------------------------------------------------- 1 | import { DataGateway, MCTSFacade, DefaultMCTSFacade } from '../src/mcts/mcts' 2 | import { TranspositionTable } from '../src/data-store' 3 | import { 4 | ticTacToeFuncs, 5 | TicTacToeState, 6 | TicTacToeMove, 7 | ticTacToeBoard 8 | } from './tic-tac-toe/tic-tac-toe' 9 | import { MCTSState, MCTSNode } from '../src/entities' 10 | import { Expand, DefaultExpand } from '../src/mcts/expand/expand' 11 | import { 12 | BestChild, 13 | UCB1, 14 | DefaultUCB1, 15 | DefaultBestChild 16 | } from '../src/mcts/select/best-child/best-child' 17 | import { Select, DefaultSelect } from '../src/mcts/select/select' 18 | import { Simulate, DefaultSimulate } from '../src/mcts/simulate/simulate' 19 | import { BackPropagate, DefaultBackPropagate } from '../src/mcts/back-propagate/back-propagate' 20 | import { loopFor } from '../src/utils' 21 | 22 | let dataStore: DataGateway> 23 | let expand: Expand 24 | let bestChild: BestChild 25 | let select: Select 26 | let simulate: Simulate 27 | let backPropagate: BackPropagate 28 | let mcts: MCTSFacade 29 | let ucb1: UCB1 30 | 31 | beforeEach(() => { 32 | const map = new Map() 33 | dataStore = new TranspositionTable(map) 34 | expand = new DefaultExpand(ticTacToeFuncs.applyAction, ticTacToeFuncs.generateActions, dataStore) 35 | ucb1 = new DefaultUCB1(1.414) 36 | bestChild = new DefaultBestChild(ucb1) 37 | select = new DefaultSelect(ticTacToeFuncs.stateIsTerminal, expand, bestChild, ucb1, Infinity) 38 | simulate = new DefaultSimulate( 39 | ticTacToeFuncs.stateIsTerminal, 40 | ticTacToeFuncs.generateActions, 41 | ticTacToeFuncs.applyAction, 42 | ticTacToeFuncs.calculateReward 43 | ) 44 | backPropagate = new DefaultBackPropagate(1) 45 | mcts = new DefaultMCTSFacade( 46 | select, 47 | expand, 48 | simulate, 49 | backPropagate, 50 | bestChild, 51 | ticTacToeFuncs.generateActions, 52 | dataStore, 53 | 100, 54 | 1.414 55 | ) 56 | }) 57 | 58 | describe('The DefaultSelect instance', () => { 59 | describe('when the current node is terminal', () => { 60 | const ticTacToeBoard = [[1, 1, -1], [1, 0, -1], [-1, 0, -1]] 61 | const state: TicTacToeState = { 62 | board: ticTacToeBoard, 63 | player: 1 64 | } 65 | const mtcsState = new MCTSState(state) 66 | const node = new MCTSNode(mtcsState, ticTacToeFuncs.generateActions(state)) 67 | it('should return the current node', () => { 68 | expect(select.run(node)).toBe(node) 69 | }) 70 | }) 71 | describe('when the current node is not terminal', () => { 72 | const ticTacToeBoard = [[1, 1, -1], [1, 0, -1], [-1, -1, 1]] 73 | const state: TicTacToeState = { 74 | board: ticTacToeBoard, 75 | player: 1 76 | } 77 | 78 | it('should return a node that is not the current node.', () => { 79 | const mtcsState = new MCTSState(state) 80 | const node = new MCTSNode(mtcsState, ticTacToeFuncs.generateActions(state)) 81 | const result = select.run(node) 82 | expect(result).toBeInstanceOf(MCTSNode) 83 | expect(result).not.toBe(node) 84 | }) 85 | }) 86 | }) 87 | 88 | describe('The DefaultUCB1 function', () => { 89 | describe('given a parent with 300 visits and a node with 100 visites and 50 reward ', () => { 90 | it('should return a number close to 0.8377', () => { 91 | const ticTacToeBoard = [[1, 1, -1], [1, 0, -1], [-1, 0, -1]] 92 | const state: TicTacToeState = { 93 | board: ticTacToeBoard, 94 | player: 1 95 | } 96 | 97 | const child = new MCTSState(state) 98 | child.visits = 100 99 | child.reward = 50 100 | expect(ucb1.run(300, child)).toBeCloseTo(0.8377) 101 | }) 102 | }) 103 | }) 104 | 105 | describe('The DefaultBestChild instance', () => { 106 | describe('given a node ', () => { 107 | it('should return the child with the highest UCB1 score', () => { 108 | const ticTacToeBoard = [[1, 0, -1], [-1, -1, 1], [1, 0, 0]] 109 | const state: TicTacToeState = { 110 | board: ticTacToeBoard, 111 | player: 1 112 | } 113 | const parentState = new MCTSState(state) 114 | const child1State = new MCTSState(state) 115 | const child2State = new MCTSState(state) 116 | const child3State = new MCTSState(state) 117 | const parentNode = new MCTSNode(parentState, ticTacToeFuncs.generateActions(state)) 118 | parentNode.addChild(child1State, ticTacToeFuncs.generateActions(state), { 119 | col: 1, 120 | row: 0 121 | }) 122 | parentNode.addChild(child2State, ticTacToeFuncs.generateActions(state), { 123 | col: 1, 124 | row: 0 125 | }) 126 | parentNode.addChild(child3State, ticTacToeFuncs.generateActions(state), { 127 | col: 1, 128 | row: 0 129 | }) 130 | parentState.visits = 300 131 | child1State.visits = 100 132 | child1State.reward = 50 133 | child2State.visits = 150 134 | child2State.reward = 100 135 | child3State.visits = 50 136 | child3State.reward = 25 137 | 138 | expect(bestChild.run(parentNode)).toBe(parentNode.children[2]) 139 | }) 140 | }) 141 | }) 142 | 143 | describe('The DefaultExpand instance', () => { 144 | describe('when the current node is not fully expanded', () => { 145 | const ticTacToeBoard = [[1, 1, -1], [1, 0, -1], [-1, 0, -1]] 146 | const state: TicTacToeState = { 147 | board: ticTacToeBoard, 148 | player: 1 149 | } 150 | const mtcsState = new MCTSState(state) 151 | let node 152 | beforeEach(() => { 153 | node = new MCTSNode(mtcsState, ticTacToeFuncs.generateActions(state)) 154 | }) 155 | 156 | it('should create a new child node', () => { 157 | expand.run(node) 158 | expect(node.children[0]).toBeInstanceOf(MCTSNode) 159 | }) 160 | it('should return a new child node', () => { 161 | expect(expand.run(node)).toBe(node.children[0]) 162 | }) 163 | }) 164 | }) 165 | 166 | describe('The DefaultSimulate instance', () => { 167 | it('returns a number that is either 1, 0 or -1', () => { 168 | const ticTacToeBoard = [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 169 | const state: TicTacToeState = { 170 | board: ticTacToeBoard, 171 | player: 1 172 | } 173 | expect(simulate.run(state)).toBeGreaterThanOrEqual(-1) 174 | expect(simulate.run(state)).toBeLessThanOrEqual(1) 175 | }) 176 | }) 177 | 178 | describe('The DefaultMCTSFacade instance', () => { 179 | describe('when calling getActionSync', () => { 180 | it('returns an action', () => { 181 | const ticTacToeBoard = [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 182 | const state: TicTacToeState = { 183 | board: ticTacToeBoard, 184 | player: 1 185 | } 186 | expect(mcts.getActionSync(state)).toBeDefined() 187 | }) 188 | }) 189 | describe('when calling getAction', () => { 190 | it('returns an action', async () => { 191 | const ticTacToeBoard = [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 192 | const state: TicTacToeState = { 193 | board: ticTacToeBoard, 194 | player: 1 195 | } 196 | const data = await mcts.getAction(state) 197 | expect(data).toBeDefined() 198 | }) 199 | }) 200 | }) 201 | -------------------------------------------------------------------------------- /test/tic-tac-toe/playground.ts: -------------------------------------------------------------------------------- 1 | import { Macao } from '../../src/macao' 2 | import { ticTacToeFuncs, TicTacToeState, TicTacToeMove } from './tic-tac-toe' 3 | import { loopFor } from '../../src/utils' 4 | 5 | let draws = 0 6 | let wins = 0 7 | let losses = 0 8 | 9 | let player = -1 10 | let gamesPlayed = 0 11 | let gamesLeft = 100 12 | const simStartTime = Date.now() 13 | let averageGameTime = 0 14 | 15 | const game = async () => { 16 | while (gamesLeft) { 17 | const gameStartTime = Date.now() 18 | const ticTacToeBoard = [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 19 | 20 | let state: TicTacToeState = { 21 | board: ticTacToeBoard, 22 | player: player 23 | } 24 | 25 | /***************************************************************************/ 26 | const player1 = new Macao( 27 | { 28 | stateIsTerminal: ticTacToeFuncs.stateIsTerminal, 29 | generateActions: ticTacToeFuncs.generateActions, 30 | applyAction: ticTacToeFuncs.applyAction, 31 | calculateReward: ticTacToeFuncs.calculateReward 32 | }, 33 | { duration: 91 } 34 | ) 35 | /***************************************************************************/ 36 | const player2 = new Macao( 37 | { 38 | stateIsTerminal: ticTacToeFuncs.stateIsTerminal, 39 | generateActions: ticTacToeFuncs.generateActions, 40 | applyAction: ticTacToeFuncs.applyAction, 41 | calculateReward: ticTacToeFuncs.calculateReward 42 | }, 43 | { duration: 91 } 44 | ) 45 | 46 | /***************************************************************************/ 47 | 48 | while (!ticTacToeFuncs.stateIsTerminal(state)) { 49 | // Player 1 50 | let action!: TicTacToeMove 51 | if (state.player === -1) { 52 | action = await player1.getActionSync(state) 53 | } 54 | 55 | // Player -1 56 | if (state.player === 1) { 57 | action = await player2.getActionSync(state) 58 | } 59 | 60 | if (!action) throw new Error('Looks like both players were skipped.') 61 | // Apply action to state 62 | state = ticTacToeFuncs.applyAction(state, action) 63 | } 64 | 65 | // When game is over update cumulative results and switch player's turn 66 | const result = ticTacToeFuncs.calculateReward(state, 1) 67 | player *= -1 68 | 69 | switch (result) { 70 | case 0: 71 | draws++ 72 | break 73 | case 1: 74 | wins++ 75 | break 76 | case -1: 77 | losses++ 78 | break 79 | } 80 | 81 | // Time calculations 82 | // Print cumulative results every 5 games. 83 | gamesPlayed++ 84 | gamesLeft-- 85 | const lastGameTime = (Date.now() - gameStartTime) / 1000 / 60 86 | if (gamesPlayed <= 5) averageGameTime += lastGameTime / 5 87 | if (gamesPlayed % 5 === 0) { 88 | console.log({ wins, draws, losses }) 89 | const simElapsedTime = (Date.now() - simStartTime) / 1000 / 60 90 | averageGameTime = (lastGameTime - averageGameTime) * 0.33 + averageGameTime 91 | const estimatedTimeLeft = averageGameTime * gamesLeft 92 | console.log( 93 | `Elapsed Time: ${Math.round(simElapsedTime)} minutes. Estimated time left: ${Math.round( 94 | estimatedTimeLeft 95 | )} minutes.` 96 | ) 97 | } 98 | } 99 | } 100 | 101 | game().then(() => { 102 | // When simulation is over play system beep 103 | process.stdout.write('\x07') 104 | }) 105 | -------------------------------------------------------------------------------- /test/tic-tac-toe/tic-tac-toe.test.ts: -------------------------------------------------------------------------------- 1 | import { Macao } from '../../src/macao' 2 | import { ticTacToeFuncs, TicTacToeState, TicTacToeMove } from './tic-tac-toe' 3 | import { loopFor } from '../../src/utils' 4 | 5 | xdescribe('The Macao instance', () => { 6 | describe('when used to simulate 50 Tic Tac Toe games', () => { 7 | describe('given 100 ms per turn and an exploration param of 1.414', () => { 8 | it('should end in a draw', () => { 9 | let draws = 0 10 | let wins = 0 11 | let losses = 0 12 | 13 | let player = -1 14 | let gamesPlayed = 0 15 | let gamesLeft = 50 16 | 17 | loopFor(gamesLeft).turns(() => { 18 | const ticTacToeBoard = [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 19 | 20 | let state: TicTacToeState = { 21 | board: ticTacToeBoard, 22 | player: player 23 | } 24 | 25 | /***************************************************************************/ 26 | const player1 = new Macao( 27 | { 28 | stateIsTerminal: ticTacToeFuncs.stateIsTerminal, 29 | generateActions: ticTacToeFuncs.generateActions, 30 | applyAction: ticTacToeFuncs.applyAction, 31 | calculateReward: ticTacToeFuncs.calculateReward 32 | }, 33 | { duration: 100, explorationParam: 1.414 } 34 | ) 35 | /***************************************************************************/ 36 | const player2 = new Macao( 37 | { 38 | stateIsTerminal: ticTacToeFuncs.stateIsTerminal, 39 | generateActions: ticTacToeFuncs.generateActions, 40 | applyAction: ticTacToeFuncs.applyAction, 41 | calculateReward: ticTacToeFuncs.calculateReward 42 | }, 43 | { duration: 100, explorationParam: 1.414 } 44 | ) 45 | 46 | /***************************************************************************/ 47 | 48 | while (!ticTacToeFuncs.stateIsTerminal(state)) { 49 | // Player 1 50 | let action!: TicTacToeMove 51 | if (state.player === -1) { 52 | action = player1.getActionSync(state) 53 | } 54 | 55 | // Player -1 56 | if (state.player === 1) { 57 | action = player2.getActionSync(state) 58 | } 59 | 60 | if (!action) throw new Error('Looks like both players were skipped.') 61 | // Apply action to state 62 | state = ticTacToeFuncs.applyAction(state, action) 63 | } 64 | 65 | // When game is over update cumulative results and switch player's turn 66 | const result = ticTacToeFuncs.calculateReward(state, 1) 67 | player *= -1 68 | 69 | switch (result) { 70 | case 0: 71 | draws++ 72 | break 73 | case 1: 74 | wins++ 75 | break 76 | case -1: 77 | losses++ 78 | break 79 | } 80 | }) 81 | expect(draws).toBeGreaterThanOrEqual(40) 82 | }) 83 | }) 84 | }) 85 | }) 86 | -------------------------------------------------------------------------------- /test/tic-tac-toe/tic-tac-toe.ts: -------------------------------------------------------------------------------- 1 | export interface TicTacToeMove { 2 | row: number 3 | col: number 4 | } 5 | 6 | export interface TicTacToeState { 7 | board: number[][] 8 | player: number 9 | } 10 | 11 | export const ticTacToeBoard = [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 12 | 13 | export function possibleMovesTicTacToe(state: TicTacToeState): TicTacToeMove[] { 14 | const result: TicTacToeMove[] = [] 15 | state.board.forEach((rowArray, row) => { 16 | rowArray.forEach((value, col) => { 17 | if (value === 0) result.push({ row, col }) 18 | }) 19 | }) 20 | return result 21 | } 22 | 23 | // Be careful not to mutate the board but to return a new one 24 | export function playMoveTicTacToe(state: TicTacToeState, move: TicTacToeMove): TicTacToeState { 25 | const jSONBoard = JSON.stringify(state.board) 26 | const newBoard = JSON.parse(jSONBoard) 27 | newBoard[move.row][move.col] = state.player * -1 28 | const newState: TicTacToeState = { 29 | board: newBoard, 30 | player: state.player * -1 31 | } 32 | return newState 33 | } 34 | 35 | export function stateIsTerminalTicTacToe(state: TicTacToeState): boolean { 36 | for (let i = 0; i < 3; i++) { 37 | // check rows to see if there is a winner 38 | if ( 39 | state.board[i][0] === state.board[i][1] && 40 | state.board[i][1] === state.board[i][2] && 41 | state.board[i][0] !== 0 42 | ) { 43 | return true 44 | } 45 | 46 | // check cols to see if there is a winner 47 | if ( 48 | state.board[0][i] === state.board[1][i] && 49 | state.board[1][i] === state.board[2][i] && 50 | state.board[0][i] !== 0 51 | ) { 52 | return true 53 | } 54 | } 55 | 56 | // check diags to see if there is a winner 57 | if ( 58 | state.board[0][0] === state.board[1][1] && 59 | state.board[1][1] === state.board[2][2] && 60 | state.board[0][0] !== 0 61 | ) { 62 | return true 63 | } 64 | 65 | if ( 66 | state.board[0][2] === state.board[1][1] && 67 | state.board[1][1] === state.board[2][0] && 68 | state.board[0][2] !== 0 69 | ) { 70 | return true 71 | } 72 | 73 | // check to see if the board is full and therefore a draw 74 | const flattenBoard = state.board.reduce((p, c) => p.concat(c)) 75 | if (flattenBoard.every(value => value !== 0)) return true 76 | 77 | return false 78 | } 79 | 80 | export function calculateRewardTicTacToe(state: TicTacToeState, player: number): number { 81 | for (let i = 0; i < 3; i++) { 82 | // check rows to see if there is a winner 83 | if (state.board[i][0] === state.board[i][1] && state.board[i][1] === state.board[i][2]) { 84 | if (state.board[i][0] === player) return 1 85 | 86 | return -1 87 | } 88 | 89 | // check cols to see if there is a winner 90 | if (state.board[0][i] === state.board[1][i] && state.board[1][i] === state.board[2][i]) { 91 | if (state.board[0][i] === player) return 1 92 | 93 | return -1 94 | } 95 | } 96 | 97 | // check diags to see if there is a winner 98 | if (state.board[0][0] === state.board[1][1] && state.board[1][1] === state.board[2][2]) { 99 | if (state.board[0][0] === player) return 1 100 | 101 | return -1 102 | } 103 | 104 | if (state.board[0][2] === state.board[1][1] && state.board[1][1] === state.board[2][0]) { 105 | if (state.board[0][2] === player) return 1 106 | 107 | return -1 108 | } 109 | 110 | return 0 111 | } 112 | 113 | export const ticTacToeFuncs = { 114 | generateActions: possibleMovesTicTacToe, 115 | applyAction: playMoveTicTacToe, 116 | stateIsTerminal: stateIsTerminalTicTacToe, 117 | calculateReward: calculateRewardTicTacToe 118 | } 119 | -------------------------------------------------------------------------------- /test/ultimate-tic-tac-toe/playground.ts: -------------------------------------------------------------------------------- 1 | import { Macao } from '../../src/macao' 2 | import { uTicTacToeFuncs, UTicTacToeMove } from './ultimate-tic-tac-toe' 3 | import { loopFor, spliceRandom } from '../../src/utils' 4 | import { DefaultSimulate } from '../../src/mcts/simulate/simulate' 5 | 6 | /** 7 | * After one million random playouts, its seems that the odds, for the first player, 8 | * are 50.9% win, 7.2% draw, 41.9% loss. 9 | */ 10 | 11 | let draws = 0 12 | let wins = 0 13 | let losses = 0 14 | 15 | let player = -1 16 | let gamesPlayed = 0 17 | let gamesLeft = 5000 18 | const simStartTime = Date.now() 19 | let averageGameTime = 0 20 | 21 | loopFor(gamesLeft).turns(() => { 22 | const gameStartTime = Date.now() 23 | const uTicTacToeBoard = [ 24 | [ 25 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 26 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 27 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 28 | ], 29 | [ 30 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 31 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 32 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 33 | ], 34 | [ 35 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 36 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 37 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 38 | ] 39 | ] 40 | let state = { 41 | board: uTicTacToeBoard, 42 | player: player, 43 | previousAction: { bigRow: -1, bigCol: -1, smallRow: -1, smallCol: -1 } 44 | } 45 | 46 | /***************************************************************************/ 47 | const player1 = new Macao( 48 | { 49 | stateIsTerminal: uTicTacToeFuncs.stateIsTerminal, 50 | generateActions: uTicTacToeFuncs.generateActions, 51 | applyAction: uTicTacToeFuncs.applyAction, 52 | calculateReward: uTicTacToeFuncs.calculateReward 53 | }, 54 | { duration: 91, decayingParam: 0.9 } 55 | ) 56 | /***************************************************************************/ 57 | const player2 = new Macao( 58 | { 59 | stateIsTerminal: uTicTacToeFuncs.stateIsTerminal, 60 | generateActions: uTicTacToeFuncs.generateActions, 61 | applyAction: uTicTacToeFuncs.applyAction, 62 | calculateReward: uTicTacToeFuncs.calculateReward 63 | }, 64 | { duration: 91, decayingParam: 0.95 } 65 | ) 66 | 67 | /***************************************************************************/ 68 | 69 | while (!uTicTacToeFuncs.stateIsTerminal(state)) { 70 | // Player 1 71 | let action!: UTicTacToeMove 72 | if (state.player === -1) { 73 | action = player1.getActionSync(state) 74 | } 75 | 76 | // Player -1 77 | if (state.player === 1) { 78 | action = player2.getActionSync(state) 79 | } 80 | 81 | if (!action) throw new Error('Looks like both players were skipped.') 82 | // Apply action to state 83 | state = uTicTacToeFuncs.applyAction(state, action) 84 | } 85 | 86 | // When game is over update cumulative results and switch player's turn 87 | const result = uTicTacToeFuncs.calculateReward(state, 1) 88 | player *= -1 89 | 90 | switch (result) { 91 | case 0: 92 | draws++ 93 | break 94 | case 1: 95 | wins++ 96 | break 97 | case -1: 98 | losses++ 99 | break 100 | } 101 | 102 | // Time calculations 103 | // Print cumulative results every 5 games. 104 | gamesPlayed++ 105 | gamesLeft-- 106 | const lastGameTime = (Date.now() - gameStartTime) / 1000 / 60 107 | if (gamesPlayed <= 5) averageGameTime += lastGameTime / 5 108 | if (gamesPlayed % 5 === 0) { 109 | console.log({ wins, draws, losses }) 110 | const simElapsedTime = (Date.now() - simStartTime) / 1000 / 60 111 | averageGameTime = (lastGameTime - averageGameTime) * 0.33 + averageGameTime 112 | const estimatedTimeLeft = averageGameTime * gamesLeft 113 | console.log( 114 | `Elapsed Time: ${Math.round(simElapsedTime)} minutes. Estimated time left: ${Math.round( 115 | estimatedTimeLeft 116 | )} minutes.` 117 | ) 118 | } 119 | }) 120 | 121 | // When simulation is over play system beep 122 | process.stdout.write('\x07') 123 | -------------------------------------------------------------------------------- /test/ultimate-tic-tac-toe/ultimate-tic-tac-toe.test.ts: -------------------------------------------------------------------------------- 1 | import { UTicTacToeState, UTicTacToeMove, uTicTacToeFuncs } from './ultimate-tic-tac-toe' 2 | import { Macao } from '../../src/macao' 3 | import { loopFor } from '../../src/utils' 4 | 5 | xdescribe('The Macao instance', () => { 6 | let uTicTacToeBoard: number[][][][] 7 | let state: UTicTacToeState 8 | let mcts: Macao 9 | 10 | describe('when used to simulate 100 Ultimate Tic Tac Toe games', () => { 11 | describe('given 85 ms per turn and an exploration param of 1.414', () => { 12 | it('should end in a draw 95% of the time or better', () => { 13 | let results = 0 14 | loopFor(100).turns(() => { 15 | mcts = new Macao( 16 | { 17 | stateIsTerminal: uTicTacToeFuncs.stateIsTerminal, 18 | generateActions: uTicTacToeFuncs.generateActions, 19 | applyAction: uTicTacToeFuncs.applyAction, 20 | calculateReward: uTicTacToeFuncs.calculateReward 21 | }, 22 | { duration: 85 } 23 | ) 24 | uTicTacToeBoard = [ 25 | [ 26 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 27 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 28 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 29 | ], 30 | [ 31 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 32 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 33 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 34 | ], 35 | [ 36 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 37 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 38 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 39 | ] 40 | ] 41 | state = { 42 | board: uTicTacToeBoard, 43 | player: -1, 44 | previousAction: { bigRow: -1, bigCol: -1, smallRow: -1, smallCol: -1 } 45 | } 46 | while (!uTicTacToeFuncs.stateIsTerminal(state)) { 47 | const action = mcts.getActionSync(state) 48 | state = uTicTacToeFuncs.applyAction(state, action) 49 | } 50 | 51 | results += uTicTacToeFuncs.calculateReward(state, 1) === 0 ? 1 : 0 52 | }) 53 | expect(results).toBeGreaterThan(95) 54 | }) 55 | }) 56 | }) 57 | }) 58 | -------------------------------------------------------------------------------- /test/ultimate-tic-tac-toe/ultimate-tic-tac-toe.ts: -------------------------------------------------------------------------------- 1 | import { Macao } from '../../src/macao' 2 | 3 | export interface UTicTacToeMove { 4 | bigRow: number 5 | bigCol: number 6 | smallRow: number 7 | smallCol: number 8 | } 9 | 10 | export interface UTicTacToeState { 11 | board: number[][][][] 12 | player: number 13 | previousAction: UTicTacToeMove 14 | } 15 | 16 | export interface TicTacToeState { 17 | board: (number | string)[][] 18 | } 19 | 20 | export function convertToMove(row: number, col: number): UTicTacToeMove { 21 | const bigRow = Math.floor(row / 3) 22 | const bigCol = Math.floor(col / 3) 23 | const smallRow = Math.floor(row % 3) 24 | const smallCol = Math.floor(col % 3) 25 | return { bigRow, bigCol, smallRow, smallCol } 26 | } 27 | 28 | export function convertFromMove(move: UTicTacToeMove): { row: number; col: number } { 29 | const row = Math.floor(move.bigRow * 3) + Math.floor(move.smallRow % 3) 30 | const col = Math.floor(move.bigCol * 3) + Math.floor(move.smallCol % 3) 31 | return { row, col } 32 | } 33 | 34 | export const uTicTacToeBoard = [ 35 | [ 36 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 37 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 38 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 39 | ], 40 | [ 41 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 42 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 43 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 44 | ], 45 | [ 46 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 47 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 48 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 49 | ] 50 | ] 51 | 52 | export function possibleMovesUTicTacToe(state: UTicTacToeState): UTicTacToeMove[] { 53 | const result: UTicTacToeMove[] = [] 54 | if (state.previousAction.bigRow !== -1) { 55 | const bigRow = state.previousAction.smallRow 56 | const bigCol = state.previousAction.smallCol 57 | const innerState = { board: state.board[bigRow][bigCol] } 58 | 59 | // Check if the inner board square the previous player played into is not terminal 60 | if (!stateIsTerminalTicTacToe(innerState)) { 61 | // Only check for moves in the big board square 62 | innerState.board.forEach((smallRowArray, smallRow) => { 63 | smallRowArray.forEach((value, smallCol) => { 64 | if (value === 0) result.push({ bigRow, bigCol, smallRow, smallCol }) 65 | }) 66 | }) 67 | return result 68 | } 69 | } 70 | 71 | // If that inner board is Terminal, we have to check all other inner boards 72 | state.board.forEach((bigRowArray, bigRow) => { 73 | bigRowArray.forEach((innerSquare, bigCol) => { 74 | const innerState = { board: innerSquare } 75 | // Check if inner board is not Terminal 76 | if (!stateIsTerminalTicTacToe(innerState)) { 77 | // Push all possible moves to result array 78 | innerState.board.forEach((smallRowArray, smallRow) => { 79 | smallRowArray.forEach((value, smallCol) => { 80 | if (value === 0) { 81 | result.push({ bigRow, bigCol, smallRow, smallCol }) 82 | } 83 | }) 84 | }) 85 | } 86 | }) 87 | }) 88 | return result 89 | } 90 | 91 | // Be careful not to mutate the board but to return a new one 92 | export function playMoveUTicTacToe(state: UTicTacToeState, move: UTicTacToeMove): UTicTacToeState { 93 | const jSONBoard = JSON.stringify(state.board) 94 | const newBoard = JSON.parse(jSONBoard) 95 | 96 | newBoard[move.bigRow][move.bigCol][move.smallRow][move.smallCol] = state.player * -1 97 | const newState: UTicTacToeState = { 98 | board: newBoard, 99 | player: state.player * -1, 100 | previousAction: move 101 | } 102 | return newState 103 | } 104 | 105 | export function stateIsTerminalTicTacToe(state: TicTacToeState): boolean { 106 | for (let i = 0; i < 3; i++) { 107 | // check rows to see if there is a winner 108 | if ( 109 | state.board[i][0] === state.board[i][1] && 110 | state.board[i][1] === state.board[i][2] && 111 | state.board[i][0] !== 0 && 112 | state.board[i][0] !== 'D' 113 | ) { 114 | return true 115 | } 116 | 117 | // check cols to see if there is a winner 118 | if ( 119 | state.board[0][i] === state.board[1][i] && 120 | state.board[1][i] === state.board[2][i] && 121 | state.board[0][i] !== 0 && 122 | state.board[0][i] !== 'D' 123 | ) { 124 | return true 125 | } 126 | } 127 | 128 | // check diags to see if there is a winner 129 | if ( 130 | state.board[0][0] === state.board[1][1] && 131 | state.board[1][1] === state.board[2][2] && 132 | state.board[0][0] !== 0 && 133 | state.board[0][0] !== 'D' 134 | ) { 135 | return true 136 | } 137 | 138 | if ( 139 | state.board[0][2] === state.board[1][1] && 140 | state.board[1][1] === state.board[2][0] && 141 | state.board[0][2] !== 0 && 142 | state.board[0][2] !== 'D' 143 | ) { 144 | return true 145 | } 146 | 147 | // check to see if the board is full and therefore a draw 148 | const flattenBoard = state.board.reduce((p, c) => p.concat(c)) 149 | if (flattenBoard.every(value => value !== 0)) return true 150 | 151 | return false 152 | } 153 | 154 | export function stateIsTerminalUTicTacToe(state: UTicTacToeState): boolean { 155 | let metaboard: (number | string)[][] = [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 156 | 157 | state.board.forEach((bigRow, bigRowIndex) => { 158 | bigRow.forEach((innerBoard, bigColIndex) => { 159 | const innerState = { board: innerBoard } 160 | if (stateIsTerminalTicTacToe(innerState)) { 161 | const score = calculateRewardTicTacToe(innerState, 1) 162 | metaboard[bigRowIndex][bigColIndex] = score === 0 ? 'D' : score 163 | } 164 | }) 165 | }) 166 | 167 | return stateIsTerminalTicTacToe({ board: metaboard }) 168 | } 169 | 170 | export function calculateRewardTicTacToe(state: TicTacToeState, player: number): number { 171 | for (let i = 0; i < 3; i++) { 172 | // check rows to see if there is a winner 173 | if ( 174 | state.board[i][0] === state.board[i][1] && 175 | state.board[i][1] === state.board[i][2] && 176 | state.board[i][0] !== 0 177 | ) { 178 | if (state.board[i][0] === player) return 1 179 | 180 | return -1 181 | } 182 | 183 | // check cols to see if there is a winner 184 | if ( 185 | state.board[0][i] === state.board[1][i] && 186 | state.board[1][i] === state.board[2][i] && 187 | state.board[0][i] !== 0 188 | ) { 189 | if (state.board[0][i] === player) return 1 190 | 191 | return -1 192 | } 193 | } 194 | 195 | // check diags to see if there is a winner 196 | if ( 197 | state.board[0][0] === state.board[1][1] && 198 | state.board[1][1] === state.board[2][2] && 199 | state.board[0][0] !== 0 200 | ) { 201 | if (state.board[0][0] === player) return 1 202 | 203 | return -1 204 | } 205 | 206 | if ( 207 | state.board[0][2] === state.board[1][1] && 208 | state.board[1][1] === state.board[2][0] && 209 | state.board[0][2] !== 0 210 | ) { 211 | if (state.board[0][2] === player) return 1 212 | 213 | return -1 214 | } 215 | 216 | return 0 217 | } 218 | 219 | export function calculateRewardUTicTacToe(state: UTicTacToeState, player: number): number { 220 | for (let i = 0; i < 3; i++) { 221 | // check rows to see if there is a winner 222 | if ( 223 | calculateRewardTicTacToe({ board: state.board[i][0] }, player) === 224 | calculateRewardTicTacToe({ board: state.board[i][1] }, player) && 225 | calculateRewardTicTacToe({ board: state.board[i][1] }, player) === 226 | calculateRewardTicTacToe({ board: state.board[i][2] }, player) 227 | ) { 228 | if (calculateRewardTicTacToe({ board: state.board[i][0] }, player) === 1) return 1 229 | 230 | return -1 231 | } 232 | 233 | // check cols to see if there is a winner 234 | if ( 235 | calculateRewardTicTacToe({ board: state.board[0][i] }, player) === 236 | calculateRewardTicTacToe({ board: state.board[1][i] }, player) && 237 | calculateRewardTicTacToe({ board: state.board[1][i] }, player) === 238 | calculateRewardTicTacToe({ board: state.board[2][i] }, player) 239 | ) { 240 | if (calculateRewardTicTacToe({ board: state.board[0][i] }, player) === 1) return 1 241 | 242 | return -1 243 | } 244 | } 245 | 246 | // check diags to see if there is a winner 247 | if ( 248 | calculateRewardTicTacToe({ board: state.board[0][0] }, player) === 249 | calculateRewardTicTacToe({ board: state.board[1][1] }, player) && 250 | calculateRewardTicTacToe({ board: state.board[1][1] }, player) === 251 | calculateRewardTicTacToe({ board: state.board[2][2] }, player) 252 | ) { 253 | if (calculateRewardTicTacToe({ board: state.board[0][0] }, player) === 1) return 1 254 | 255 | return -1 256 | } 257 | 258 | if ( 259 | calculateRewardTicTacToe({ board: state.board[0][2] }, player) === 260 | calculateRewardTicTacToe({ board: state.board[1][1] }, player) && 261 | calculateRewardTicTacToe({ board: state.board[1][1] }, player) === 262 | calculateRewardTicTacToe({ board: state.board[2][0] }, player) 263 | ) { 264 | if (calculateRewardTicTacToe({ board: state.board[0][2] }, player) === 1) return 1 265 | 266 | return -1 267 | } 268 | 269 | // If there is no 3 in a row, the winner is whoever has won the most small boards 270 | let player1 = 0 271 | let player2 = 0 272 | 273 | for (const row of state.board) { 274 | for (const col of row) { 275 | const result = calculateRewardTicTacToe({ board: col }, player) 276 | switch (result) { 277 | case 1: 278 | player1++ 279 | break 280 | case -1: 281 | player2++ 282 | break 283 | } 284 | } 285 | } 286 | 287 | if (player1 > player2) return 1 288 | if (player2 > player1) return -1 289 | 290 | return 0 291 | } 292 | 293 | export const uTicTacToeFuncs = { 294 | generateActions: possibleMovesUTicTacToe, 295 | applyAction: playMoveUTicTacToe, 296 | stateIsTerminal: stateIsTerminalUTicTacToe, 297 | calculateReward: calculateRewardUTicTacToe 298 | } 299 | 300 | const testBoard = [ 301 | [ 302 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 303 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 304 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 305 | ], 306 | [ 307 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 308 | [[0, 0, 0], [-1, 1, 0], [0, 0, 0]], 309 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 310 | ], 311 | [ 312 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 313 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]], 314 | [[0, 0, 0], [0, 0, 0], [0, 0, 0]] 315 | ] 316 | ] 317 | const testState = { 318 | board: testBoard, 319 | player: -1, 320 | previousAction: { 321 | bigRow: 1, 322 | bigCol: 1, 323 | smallRow: 1, 324 | smallCol: 0 325 | } 326 | } 327 | 328 | // const mcts = new Macao(ticTacToeFuncs, {duration: 2000}); 329 | // mcts.getActionSync(testState); //? 330 | 331 | // possibleMovesUTicTacToe(testState) //? 332 | // stateIsTerminalUTicTacToe(testState) //? 333 | // calculateRewardUTicTacToe(testState, 1) 334 | // playMoveUTicTacToe(testState, {bigRow:0, bigCol:0, smallCol:0, smallRow:0}).board[0][0] 335 | -------------------------------------------------------------------------------- /test/utils.test.ts: -------------------------------------------------------------------------------- 1 | import { getRandomIntInclusive, spliceRandom, loopFor, now } from '../src/utils' 2 | describe('The getRandomIntInclusive function', () => { 3 | describe('when given the numbers 15 and 62', () => { 4 | it('should return a number between 15 and 62, inclusively.', () => { 5 | expect(getRandomIntInclusive(15, 62)).toBeGreaterThanOrEqual(15) 6 | expect(getRandomIntInclusive(15, 62)).toBeLessThanOrEqual(62) 7 | }) 8 | }) 9 | }) 10 | 11 | describe('The spliceRandom function', () => { 12 | describe('when given an array of numbers', () => { 13 | it('should remove a random item from it.', () => { 14 | const array = [0, 1, 2, 3, 4, 5] 15 | spliceRandom(array) 16 | expect(array).toHaveLength(5) 17 | }) 18 | it('should return a number', () => { 19 | const array = [0, 1, 2, 3, 4, 5] 20 | expect(spliceRandom(array)).toBeGreaterThanOrEqual(0) 21 | expect(spliceRandom(array)).toBeLessThanOrEqual(5) 22 | }) 23 | }) 24 | }) 25 | 26 | describe('The loopFor function', () => { 27 | describe('when called with 0.1 seconds', () => { 28 | it('should loop for 0.1 seconds', () => { 29 | const start = now() 30 | loopFor(0.1).seconds(() => { 31 | // 32 | }) 33 | const time = now() - start 34 | expect(time).toBeGreaterThanOrEqual(75) 35 | expect(time).toBeLessThanOrEqual(125) 36 | }) 37 | describe('and returning true', () => { 38 | it('should break out of the loop', () => { 39 | const start = now() 40 | loopFor(0.1).seconds(() => { 41 | return true 42 | }) 43 | const time = now() - start 44 | expect(time).toBeLessThanOrEqual(50) 45 | }) 46 | }) 47 | }) 48 | describe('when called with 100 milliseconds', () => { 49 | it('should loop for 100 milliseconds', () => { 50 | const start = now() 51 | loopFor(100).milliseconds(() => { 52 | // 53 | }) 54 | const time = now() - start 55 | expect(time).toBeGreaterThanOrEqual(75) 56 | expect(time).toBeLessThanOrEqual(125) 57 | }) 58 | describe('and returning true', () => { 59 | it('should break out of the loop', () => { 60 | const start = now() 61 | loopFor(100).milliseconds(() => { 62 | return true 63 | }) 64 | const time = now() - start 65 | expect(time).toBeLessThanOrEqual(50) 66 | }) 67 | }) 68 | }) 69 | describe('when called with 10 turns', () => { 70 | it('should loop for 10 turns', () => { 71 | let turns = 0 72 | loopFor(10).turns(() => { 73 | turns++ 74 | }) 75 | 76 | expect(turns).toBeCloseTo(10, 0) 77 | }) 78 | describe('and returning true', () => { 79 | it('should break out of the loop', () => { 80 | let turns = 0 81 | loopFor(10).turns(() => { 82 | turns++ 83 | if (turns === 5) { 84 | return true 85 | } 86 | }) 87 | 88 | expect(turns).toBeCloseTo(5, 0) 89 | }) 90 | }) 91 | }) 92 | }) 93 | -------------------------------------------------------------------------------- /test/wondev-woman/playground.ts: -------------------------------------------------------------------------------- 1 | import { loopFor } from '../../src/utils' 2 | import { WondevSquare, WondevState, wondevFuncs, WondevAction } from './wondev-woman' 3 | import { Macao } from '../../src/macao' 4 | 5 | let draws = 0 6 | let wins = 0 7 | let losses = 0 8 | 9 | let player = -1 10 | let gamesPlayed = 0 11 | let gamesLeft = 5000 12 | const simStartTime = Date.now() 13 | let averageGameTime = 0 14 | let marginOfError = 0.15 15 | 16 | loopFor(gamesLeft).turns(() => { 17 | const gameStartTime = Date.now() 18 | const boardSize = 5 19 | const board: WondevSquare[][] = [] 20 | 21 | for (let i = 0; i < boardSize; i++) { 22 | const row: WondevSquare[] = [] 23 | for (let j = 0; j < boardSize; j++) { 24 | row.push({ height: 0, unit: undefined }) 25 | } 26 | board.push(row) 27 | } 28 | 29 | let state: WondevState = { 30 | board, 31 | player: -1, 32 | playerUnitsPos: [{ x: 0, y: 0 }], 33 | opponentUnitsPos: [{ x: 4, y: 4 }], 34 | playerScore: 0, 35 | opponentScore: 0, 36 | playerEnded: false, 37 | opponentEnded: false 38 | } 39 | state.board[0][0].unit = 1 40 | state.board[4][4].unit = -1 41 | 42 | /***************************************************************************/ 43 | const player1 = new Macao( 44 | { 45 | stateIsTerminal: wondevFuncs.stateIsTerminal, 46 | generateActions: wondevFuncs.generateActions, 47 | applyAction: wondevFuncs.applyAction, 48 | calculateReward: wondevFuncs.calculateReward 49 | }, 50 | { duration: 40, decayingParam: 0.9 } 51 | ) 52 | /***************************************************************************/ 53 | const player2 = new Macao( 54 | { 55 | stateIsTerminal: wondevFuncs.stateIsTerminal, 56 | generateActions: wondevFuncs.generateActions, 57 | applyAction: wondevFuncs.applyAction, 58 | calculateReward: wondevFuncs.calculateReward 59 | }, 60 | { duration: 40 } 61 | ) 62 | 63 | /***************************************************************************/ 64 | let isPlayerOneFirstTurn = true 65 | let isPlayerTwoFirstTurn = true 66 | while (!wondevFuncs.stateIsTerminal(state)) { 67 | // Player 1 68 | let action!: WondevAction 69 | if (state.player === -1) { 70 | action = isPlayerOneFirstTurn 71 | ? player1.getActionSync(state, 950) 72 | : player1.getActionSync(state) 73 | isPlayerOneFirstTurn = false 74 | } 75 | 76 | // Player -1 77 | if (state.player === 1) { 78 | action = isPlayerTwoFirstTurn 79 | ? player2.getActionSync(state, 950) 80 | : player2.getActionSync(state) 81 | isPlayerTwoFirstTurn = false 82 | } 83 | 84 | // if (!action) throw new Error("Looks like both players were skipped."); 85 | // Apply action to state 86 | state = wondevFuncs.applyAction(state, action) 87 | } 88 | 89 | // When game is over update cumulative results and switch player's turn 90 | const result = wondevFuncs.calculateReward(state, 1) 91 | player *= -1 92 | 93 | switch (result) { 94 | case 0: 95 | draws++ 96 | break 97 | case 1: 98 | wins++ 99 | break 100 | case -1: 101 | losses++ 102 | break 103 | } 104 | 105 | // Time calculations 106 | // Print cumulative results every 5 games. 107 | gamesPlayed++ 108 | gamesLeft-- 109 | const lastGameTime = (Date.now() - gameStartTime) / 1000 / 60 110 | if (gamesPlayed <= 5) averageGameTime += lastGameTime / 5 111 | if (gamesPlayed % 5 === 0) { 112 | console.log({ wins, draws, losses }) 113 | const simElapsedTime = (Date.now() - simStartTime) / 1000 / 60 114 | averageGameTime = (lastGameTime - averageGameTime) * 0.33 + averageGameTime 115 | 116 | // Statitics 117 | const z = 1.96 118 | const sampleSize = wins + losses 119 | const p = 0.5 120 | const neededSampleSize = z * z * p * (1 - p) / (marginOfError * marginOfError) 121 | const aP = wins / sampleSize 122 | const c = z * Math.sqrt(aP * (1 - aP) / sampleSize) 123 | 124 | const estimatedTimeLeft = averageGameTime * gamesLeft 125 | console.log( 126 | `Elapsed Time: ${Math.round(simElapsedTime)} minutes. Estimated time left: ${Math.round( 127 | estimatedTimeLeft 128 | )} minutes.` 129 | ) 130 | 131 | if (sampleSize >= neededSampleSize) { 132 | // If actual margin of error is smaller than the difference between half the win% and 50% 133 | if (c < Math.abs(aP - 0.5) / 2) { 134 | if (aP > 0.5) { 135 | console.log(`All done. 95% confidence that Player1 is better than Player2`) 136 | return true 137 | } 138 | if (aP < 0.5) { 139 | console.log(`All done. 95% confidence that Player2 is better than Player1`) 140 | return true 141 | } 142 | console.log(`All done. 95% confidence that both players are of equal strength.`) 143 | return true 144 | } 145 | marginOfError = c 146 | } 147 | } 148 | }) 149 | 150 | // When simulation is over play system beep 151 | process.stdout.write('\x07') 152 | console.log('WondevWoman test over.') 153 | -------------------------------------------------------------------------------- /test/wondev-woman/wondev-woman.ts: -------------------------------------------------------------------------------- 1 | import { GenerateActions, ApplyAction, StateIsTerminal, CalculateReward } from '../../src/entities' 2 | 3 | export interface WondevSquare { 4 | height: number 5 | unit: number | undefined 6 | } 7 | 8 | export interface Point { 9 | x: number 10 | y: number 11 | } 12 | export interface WondevState { 13 | board: WondevSquare[][] 14 | player: number 15 | playerUnitsPos: Point[] 16 | opponentUnitsPos: Point[] 17 | playerScore: number 18 | opponentScore: number 19 | playerEnded: boolean 20 | opponentEnded: boolean 21 | } 22 | 23 | export interface WondevAction { 24 | moveFrom: Point 25 | moveTo: Point 26 | moveDirection: string 27 | buildTo: Point 28 | buildDirection: string 29 | } 30 | 31 | interface Map { 32 | [key: string]: T 33 | } 34 | const WONDEVDIRECTIONS: Map = { 35 | N: { x: 0, y: -1 }, 36 | NE: { x: 1, y: -1 }, 37 | E: { x: 1, y: 0 }, 38 | SE: { x: 1, y: 1 }, 39 | S: { x: 0, y: 1 }, 40 | SW: { x: -1, y: 1 }, 41 | W: { x: -1, y: 0 }, 42 | NW: { x: -1, y: -1 } 43 | } 44 | 45 | const generateActions: GenerateActions = ( 46 | state: WondevState 47 | ): WondevAction[] => { 48 | let actions: WondevAction[] = [] 49 | const unitsPos = state.player === -1 ? state.playerUnitsPos : state.opponentUnitsPos 50 | 51 | for (const unit of unitsPos) { 52 | for (const direction in WONDEVDIRECTIONS) { 53 | const x = unit.x + WONDEVDIRECTIONS[direction].x 54 | const y = unit.y + WONDEVDIRECTIONS[direction].y 55 | const unitSquare = state.board[unit.y][unit.x] 56 | let targetSquare: WondevSquare | undefined 57 | try { 58 | targetSquare = state.board[y][x] 59 | } catch (e) { 60 | targetSquare = undefined 61 | } 62 | // Check if square is accessible from unit position 63 | if ( 64 | !targetSquare || 65 | targetSquare.height > 3 || 66 | targetSquare.height < 0 || 67 | targetSquare.height - unitSquare.height > 1 68 | ) { 69 | continue 70 | } 71 | 72 | // Check if there is another unit there 73 | if (targetSquare.unit) continue 74 | 75 | // Check if you can build somewhere around the target 76 | for (const buildDir in WONDEVDIRECTIONS) { 77 | const buildX = x + WONDEVDIRECTIONS[buildDir].x 78 | const buildY = y + WONDEVDIRECTIONS[buildDir].y 79 | let buildTargetSquare: WondevSquare | undefined 80 | try { 81 | buildTargetSquare = state.board[buildY][buildX] || undefined 82 | } catch (e) { 83 | buildTargetSquare = undefined 84 | } 85 | 86 | // Check if square is buildable 87 | if (!buildTargetSquare || buildTargetSquare.height > 3 || buildTargetSquare.height < 0) { 88 | continue 89 | } 90 | 91 | // Check if there is another unit there 92 | if (buildTargetSquare.unit) continue 93 | 94 | const action: WondevAction = { 95 | moveFrom: unit, 96 | moveTo: { x, y }, 97 | buildTo: { x: buildX, y: buildY }, 98 | moveDirection: direction, 99 | buildDirection: buildDir 100 | } 101 | 102 | actions.push(action) 103 | } 104 | } 105 | } 106 | return actions 107 | } 108 | 109 | const applyAction: ApplyAction = ( 110 | state: WondevState, 111 | action: WondevAction 112 | ): WondevState => { 113 | const stringifiedState = JSON.stringify(state) 114 | const newState = JSON.parse(stringifiedState) as WondevState 115 | // If there are no possible actions, the player has lost, update score accordingly 116 | if (!action) { 117 | newState.player *= -1 118 | if (newState.player === 1) { 119 | newState.playerEnded = true 120 | return newState 121 | } 122 | newState.opponentEnded = true 123 | return newState 124 | } 125 | 126 | // Move the unit 127 | if (state.player === -1) { 128 | newState.board[action.moveFrom.y][action.moveFrom.x].unit = 0 129 | newState.board[action.moveTo.y][action.moveTo.x].unit = 1 130 | // If unit moves into a level 3 square, update score 131 | if (newState.board[action.moveTo.y][action.moveTo.x].height === 3) newState.playerScore++ 132 | // Update unit position 133 | for (const playerUnit of newState.playerUnitsPos) { 134 | if (playerUnit.x === action.moveFrom.x && playerUnit.y === action.moveFrom.y) { 135 | playerUnit.x = action.moveTo.x 136 | playerUnit.y = action.moveTo.y 137 | } 138 | } 139 | } 140 | 141 | if (state.player === 1) { 142 | newState.board[action.moveFrom.y][action.moveFrom.x].unit = 0 143 | newState.board[action.moveTo.y][action.moveTo.x].unit = -1 144 | // If unit moves into a level 3 square, update score 145 | if (newState.board[action.moveTo.y][action.moveTo.x].height === 3) newState.opponentScore++ 146 | // Update unit position 147 | for (const opponentUnit of newState.opponentUnitsPos) { 148 | if (opponentUnit.x === action.moveFrom.x && opponentUnit.y === action.moveFrom.y) { 149 | opponentUnit.x = action.moveTo.x 150 | opponentUnit.y = action.moveTo.y 151 | } 152 | } 153 | } 154 | 155 | // Build 156 | newState.board[action.buildTo.y][action.buildTo.x].height++ 157 | 158 | newState.player *= -1 159 | 160 | return newState 161 | } 162 | 163 | const stateIsTerminal: StateIsTerminal = (state: WondevState) => { 164 | if (state.playerEnded && state.opponentEnded) return true 165 | 166 | return false 167 | } 168 | 169 | const calculateReward: CalculateReward = (state: WondevState, player: number) => { 170 | if (player === 1) { 171 | if (state.playerScore > state.opponentScore) return 1 172 | if (state.playerScore < state.opponentScore) return -1 173 | } 174 | if (player === -1) { 175 | if (state.opponentScore > state.playerScore) return 1 176 | if (state.opponentScore < state.playerScore) return -1 177 | } 178 | return 0 179 | } 180 | 181 | export const wondevFuncs = { 182 | generateActions, 183 | applyAction, 184 | stateIsTerminal, 185 | calculateReward 186 | } 187 | -------------------------------------------------------------------------------- /tools/gh-pages-publish.ts: -------------------------------------------------------------------------------- 1 | const { cd, exec, echo, touch } = require("shelljs") 2 | const { readFileSync } = require("fs") 3 | const url = require("url") 4 | 5 | let repoUrl 6 | let pkg = JSON.parse(readFileSync("package.json") as any) 7 | if (typeof pkg.repository === "object") { 8 | if (!pkg.repository.hasOwnProperty("url")) { 9 | throw new Error("URL does not exist in repository section") 10 | } 11 | repoUrl = pkg.repository.url 12 | } else { 13 | repoUrl = pkg.repository 14 | } 15 | 16 | let parsedUrl = url.parse(repoUrl) 17 | let repository = (parsedUrl.host || "") + (parsedUrl.path || "") 18 | let ghToken = process.env.GH_TOKEN 19 | 20 | echo("Deploying docs!!!") 21 | cd("docs/api") 22 | touch(".nojekyll") 23 | exec("git init") 24 | exec("git add .") 25 | exec('git config user.name "Philippe Vaillancourt"') 26 | exec('git config user.email "philippe_vaillancourt@sympatico.ca"') 27 | exec('git commit -m "docs(docs): update gh-pages"') 28 | exec( 29 | `git push --force --quiet "https://${ghToken}@${repository}" master:gh-pages` 30 | ) 31 | echo("Docs deployed!!") 32 | -------------------------------------------------------------------------------- /tools/semantic-release-prepare.ts: -------------------------------------------------------------------------------- 1 | const path = require("path") 2 | const { fork } = require("child_process") 3 | const colors = require("colors") 4 | 5 | const { readFileSync, writeFileSync } = require("fs") 6 | const pkg = JSON.parse( 7 | readFileSync(path.resolve(__dirname, "..", "package.json")) 8 | ) 9 | 10 | pkg.scripts.prepush = "npm run test:prod && npm run build" 11 | pkg.scripts.commitmsg = "validate-commit-msg" 12 | 13 | writeFileSync( 14 | path.resolve(__dirname, "..", "package.json"), 15 | JSON.stringify(pkg, null, 2) 16 | ) 17 | 18 | // Call husky to set up the hooks 19 | fork(path.resolve(__dirname, "..", "node_modules", "husky", "bin", "install")) 20 | 21 | console.log() 22 | console.log(colors.green("Done!!")) 23 | console.log() 24 | 25 | if (pkg.repository.url.trim()) { 26 | console.log(colors.cyan("Now run:")) 27 | console.log(colors.cyan(" npm install -g semantic-release-cli")) 28 | console.log(colors.cyan(" semantic-release-cli setup")) 29 | console.log() 30 | console.log( 31 | colors.cyan('Important! Answer NO to "Generate travis.yml" question') 32 | ) 33 | console.log() 34 | console.log( 35 | colors.gray( 36 | 'Note: Make sure "repository.url" in your package.json is correct before' 37 | ) 38 | ) 39 | } else { 40 | console.log( 41 | colors.red( 42 | 'First you need to set the "repository.url" property in package.json' 43 | ) 44 | ) 45 | console.log(colors.cyan("Then run:")) 46 | console.log(colors.cyan(" npm install -g semantic-release-cli")) 47 | console.log(colors.cyan(" semantic-release-cli setup")) 48 | console.log() 49 | console.log( 50 | colors.cyan('Important! Answer NO to "Generate travis.yml" question') 51 | ) 52 | } 53 | 54 | console.log() 55 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "moduleResolution": "node", 4 | "target": "es5", 5 | "module":"commonjs", 6 | "lib": ["es2015", "es2016", "es2017", "dom"], 7 | "strict": true, 8 | "sourceMap": true, 9 | "declaration": true, 10 | "stripInternal": true, 11 | "allowSyntheticDefaultImports": true, 12 | "experimentalDecorators": true, 13 | "emitDecoratorMetadata": true, 14 | "declarationDir": "dist/types", 15 | "outDir": "dist/lib", 16 | "typeRoots": [ 17 | "node_modules/@types" 18 | ] 19 | }, 20 | "include": [ 21 | "src" 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "tslint-config-standard", 4 | "tslint-config-prettier" 5 | ], 6 | "rules": { 7 | "variable-name": [true, "ban-keywords", "check-format", "allow-trailing-underscore"] 8 | } 9 | } --------------------------------------------------------------------------------