├── .env.example ├── .github └── ISSUE_TEMPLATE ├── .gitignore ├── .travis.yml ├── CHANGELOG ├── LICENSE ├── README.md ├── db └── .keep ├── docs ├── .keep └── server.md ├── example.js ├── examples └── server-lowdb.js ├── index.js ├── lib ├── bt │ ├── constants.js │ ├── context.js │ ├── data_feed.js │ ├── data_stream.js │ ├── fetch_data.js │ ├── request_semaphore.js │ ├── seed_candles.js │ ├── stream_data.js │ └── sync_data.js ├── cmds │ ├── delete_all_bts.js │ ├── delete_bt.js │ ├── exec_bt.js │ ├── exec_strategy.js │ ├── get_bt_history.js │ ├── get_bts.js │ ├── get_candles.js │ ├── get_trades.js │ ├── set_bt_favorite.js │ ├── stop_bt.js │ └── submit_bt.js ├── db │ ├── bt_dao.js │ └── sqlite_db.js ├── errors.js ├── server.js ├── util │ ├── delay.js │ ├── get_derivatives_config.js │ ├── parse_exec_msg.js │ └── validate_bt_args.js └── wss │ ├── send.js │ └── send_error.js ├── package.json └── test └── unit ├── bt └── request_semaphore.js └── db ├── bt_dao.js └── sqlite_db.js /.env.example: -------------------------------------------------------------------------------- 1 | DB_FILENAME=db/dev.json 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE: -------------------------------------------------------------------------------- 1 | #### Issue type 2 | - [ ] bug 3 | - [ ] missing functionality 4 | - [ ] performance 5 | - [ ] feature request 6 | 7 | #### Brief description 8 | 9 | #### Steps to reproduce 10 | - 11 | 12 | ##### Additional Notes: 13 | - 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | dist/* 3 | node_modules 4 | npm-debug.log 5 | .vscode 6 | *.swo 7 | *.swp 8 | .DS_Store 9 | db/*.sql 10 | .env 11 | scripts/* 12 | !db/.keep 13 | db/* 14 | package-lock.json 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | 3 | language: node_js 4 | node_js: 5 | - "12" 6 | 7 | install: 8 | - npm install 9 | 10 | script: 11 | - npm run lint 12 | - npm run unit 13 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | # 2.0.0 2 | - use new backtesting lib, fix memory leak, do not simulate trades for candles 3 | 4 | # 1.5.0 5 | - allow different sources for backtesting 6 | 7 | # 1.4.1 8 | - feature: added support for server side backtesting 9 | 10 | # 1.4.0 11 | - feature: adds meta field to exec.bt command 12 | 13 | # 1.3.5 14 | - meta: bump bitfinex-api-node to v4 15 | 16 | # 1.3.4 17 | - fix: further sync key refactoring 18 | 19 | # 1.3.3 20 | - fix: sync key parsing and reporting 21 | 22 | # 1.3.2 23 | - bump knex to version v0.19.5 for security fix 24 | 25 | # 1.3.1 26 | - docs: create/update 27 | 28 | # 1.3.0 29 | - manifest: bump deps 30 | - meta: add github issue/pr templates 31 | - meta: standardize travis config 32 | - meta: add placeholder npm test 33 | - meta: add changelog 34 | - refactor: manual call to open() now required to start wss server 35 | 36 | # 1.2.1 37 | - refactor: move socket open logic out of DataServer constructor 38 | - fix: throw error on redundant socket close 39 | 40 | # 1.2.0 41 | - refactor: do not send market data on initial connect 42 | - refactor: send version number on initial connect 43 | 44 | # 1.1.0 45 | - refactor: buffer bfx proxy prior to socket open 46 | - refactor: update to use new bfx-hf-models API 47 | - refactor: update to accept new exchange field on exec.bt command 48 | - feature: split start commands into lowdb & sql variants 49 | - feature: optimize candle sync logic by taking active syncs into account 50 | - feature: add meta field to get.candles call for custom client data 51 | 52 | # 1.0.0 53 | - fix: exec.bt cmd trade key access typo 54 | - feature: add bfx API proxy support 55 | - refactor: move logic into DataServer class 56 | - refactor: switch from bfx-hf-db to bfx-hf-models 57 | - manfiest: rm private flag 58 | - meta: rm knexfile.js 59 | 60 | # 1.0.0-alpha 61 | - initial version 62 | 63 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Bitfinex Honey Framework Data Server for Node.JS 2 | 3 | [![Build Status](https://travis-ci.org/bitfinexcom/bfx-hf-data-server.svg?branch=master)](https://travis-ci.org/bitfinexcom/bfx-hf-data-server) 4 | 5 | The HF data server runs backtests for the HF UI Electron App. It syncs candle and trade data and then executes the strategy on it for backtesting. 6 | 7 | The DB backend is implemented by a plugin: 8 | * [bfx-hf-models-adapter-lowdb](https://github.com/bitfinexcom/bfx-hf-models-adapter-lowdb) 9 | 10 | Regardless of the backend, a schema must be specified (providing exchange-specific API methods). The official Bitfinex schema is [bfx-hf-ext-plugin-bitfinex](https://github.com/bitfinexcom/bfx-hf-ext-plugin-bitfinex). 11 | 12 | ### Installation 13 | 14 | For standalone usage: 15 | ```bash 16 | git clone https://github.com/bitfinexcom/bfx-hf-data-server 17 | cd bfx-hf-data-server 18 | npm i 19 | 20 | cp .env.example .env 21 | 22 | npm run start-lowdb 23 | ``` 24 | 25 | For usage/extension within an existing project: 26 | ```bash 27 | npm i --save bfx-hf-data-server 28 | ``` 29 | 30 | ### Quickstart 31 | 32 | Follow the installation instructions, and run `npm run start-lowdb`. 33 | 34 | ### Docs 35 | 36 | For executable examples, [refer to `examples/`](/examples) 37 | 38 | ### Example 39 | 40 | ```js 41 | const DataServer = require('bfx-hf-data-server') 42 | const HFDB = require('bfx-hf-models') 43 | const HFDBLowDBAdapter = require('bfx-hf-models-adapter-lowdb') 44 | const { schema: HFDBBitfinexSchema } = require('bfx-hf-ext-plugin-bitfinex') 45 | 46 | const db = new HFDB({ 47 | schema: HFDBBitfinexSchema, 48 | adapter: HFDBLowDBAdapter({ 49 | dbPath: './SOME_DB_PATH.json', 50 | schema: HFDBBitfinexSchema 51 | }) 52 | }) 53 | 54 | const ds = new DataServer({ 55 | port: 8899, 56 | db 57 | }) 58 | 59 | ds.open() 60 | 61 | // data server ready to receive commands 62 | ``` 63 | 64 | ### Contributing 65 | 66 | 1. Fork it 67 | 2. Create your feature branch (`git checkout -b my-new-feature`) 68 | 3. Commit your changes (`git commit -am 'Add some feature'`) 69 | 4. Push to the branch (`git push origin my-new-feature`) 70 | 5. Create a new Pull Request 71 | 72 | ### Note 73 | 74 | This package will be maintained only via github, please use latest relases from github instead of npm. 75 | 76 | Example on how to install specific version from github: 77 | ``` 78 | npm i --save-prod https://github.com/bitfinexcom/bfx-hf-data-server.git#v3.0.4 79 | ``` 80 | 81 | Example on how to install it latest version from github: 82 | ``` 83 | npm i --save-prod https://github.com/bitfinexcom/bfx-hf-data-server.git 84 | ``` 85 | -------------------------------------------------------------------------------- /db/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bitfinexcom/bfx-hf-data-server/95e599cec4aa4671cbb029f36d2f204a494819ba/db/.keep -------------------------------------------------------------------------------- /docs/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bitfinexcom/bfx-hf-data-server/95e599cec4aa4671cbb029f36d2f204a494819ba/docs/.keep -------------------------------------------------------------------------------- /docs/server.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## DataServer 4 | **Kind**: global class 5 | 6 | * [DataServer](#DataServer) 7 | * [new DataServer(args, db, port)](#new_DataServer_new) 8 | * [.open()](#DataServer+open) 9 | * [.close()](#DataServer+close) 10 | * [.getRunningSyncRanges()](#DataServer+getRunningSyncRanges) ⇒ Array.<Object> 11 | * [.expectSync(range)](#DataServer+expectSync) ⇒ Promise 12 | * [.optimizeSyncRange(range)](#DataServer+optimizeSyncRange) ⇒ Object 13 | * [.notifySyncStart(args)](#DataServer+notifySyncStart) 14 | * [.notifySyncEnd(args)](#DataServer+notifySyncEnd) 15 | 16 | 17 | 18 | ### new DataServer(args, db, port) 19 | 20 | | Param | Type | Description | 21 | | --- | --- | --- | 22 | | args | Object | | 23 | | db | Object | bfx-hf-models DB instance | 24 | | port | number | websocket server port | 25 | 26 | 27 | 28 | ### dataServer.open() 29 | Spawns the WebSocket API server; throws an error if it is already open 30 | 31 | **Kind**: instance method of [DataServer](#DataServer) 32 | 33 | 34 | ### dataServer.close() 35 | Closes the WebSocket API server; throws an error if it is not open 36 | 37 | **Kind**: instance method of [DataServer](#DataServer) 38 | 39 | 40 | ### dataServer.getRunningSyncRanges() ⇒ Array.<Object> 41 | Returns an array of active sync ranges 42 | 43 | **Kind**: instance method of [DataServer](#DataServer) 44 | **Returns**: Array.<Object> - ranges 45 | 46 | 47 | ### dataServer.expectSync(range) ⇒ Promise 48 | Returns a promise that resolves when a sync covering the specified range 49 | finishes. If no such sync is active, this is a no-op. 50 | 51 | **Kind**: instance method of [DataServer](#DataServer) 52 | **Returns**: Promise - p - resolves on sync completion 53 | 54 | | Param | Type | 55 | | --- | --- | 56 | | range | Object | 57 | | range.start | number | 58 | | range.end | number | 59 | | range.exchange | string | 60 | | range.symbol | string | 61 | | range.tf | string | 62 | 63 | 64 | 65 | ### dataServer.optimizeSyncRange(range) ⇒ Object 66 | Returns a sync range that takes into account active syncs, to prevent 67 | overlapping sync tasks. 68 | 69 | **Kind**: instance method of [DataServer](#DataServer) 70 | **Returns**: Object - optimalRange - null if sync not required at all 71 | 72 | | Param | Type | 73 | | --- | --- | 74 | | range | Object | 75 | | range.exchange | string | 76 | | range.symbol | string | 77 | | range.tf | string | 78 | | range.start | number | 79 | | range.end | number | 80 | 81 | 82 | 83 | ### dataServer.notifySyncStart(args) 84 | Notify the server that a sync is running for the specified range/market 85 | 86 | **Kind**: instance method of [DataServer](#DataServer) 87 | 88 | | Param | Type | 89 | | --- | --- | 90 | | args | Object | 91 | | args.exchange | string | 92 | | args.symbol | string | 93 | | args.tf | string | 94 | | args.start | number | 95 | | args.end | number | 96 | 97 | 98 | 99 | ### dataServer.notifySyncEnd(args) 100 | Notify the server that a sync has finished for the specified range/market 101 | 102 | **Kind**: instance method of [DataServer](#DataServer) 103 | 104 | | Param | Type | 105 | | --- | --- | 106 | | args | Object | 107 | | args.exchange | string | 108 | | args.symbol | string | 109 | | args.tf | string | 110 | | args.start | number | 111 | | args.end | number | 112 | 113 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | // node examples/server-lowdb.js 4 | 5 | const Ws = require('ws') 6 | const ws = new Ws('ws://localhost:8899') 7 | 8 | ws.on('open', () => { 9 | console.log('open') 10 | 11 | ws.send( 12 | JSON.stringify(msg) 13 | ) 14 | }) 15 | 16 | ws.on('message', (data) => { 17 | console.log(data) 18 | }) 19 | 20 | /* eslint-disable */ 21 | const strategy = { 22 | defineIndicators: '(I) => {\n' + 23 | ' const indicators = {\n' + 24 | " \temaL: new I.EMA([100, 'high']),\n" + 25 | " \temaS: new I.EMA([10, 'low']),\n" + 26 | ' }\n' + 27 | ' \n' + 28 | " indicators.emaL.color = '#00ff00'\n" + 29 | " indicators.emaS.color = '#ff0000'\n" + 30 | ' \n' + 31 | ' return indicators\n' + 32 | '}', 33 | onEnter: '({ HFS, _, HFU }) => async (state = {}, update = {}) => {\n' + 34 | ' if (HFS.getNumCandles(state) < 2) { // 2 price points needed\n' + 35 | ' return state\n' + 36 | ' }\n' + 37 | '\n' + 38 | ' const { price, mts } = update\n' + 39 | ' const i = HFS.indicators(state)\n' + 40 | ' const iv = HFS.indicatorValues(state)\n' + 41 | ' const { emaS } = i\n' + 42 | ' const l = iv.emaL\n' + 43 | ' const s = iv.emaS\n' + 44 | ' const amount = 1\n' + 45 | ' \n' + 46 | ' if (emaS.crossed(l)) {\n' + 47 | ' if (s > l) {\n' + 48 | ' return HFS.openLongPositionMarket(state, {\n' + 49 | ' mtsCreate: mts,\n' + 50 | ' amount,\n' + 51 | ' price,\n' + 52 | " label: 'enter long',\n" + 53 | ' })\n' + 54 | ' } else {\n' + 55 | ' return HFS.openShortPositionMarket(state, {\n' + 56 | ' mtsCreate: mts,\n' + 57 | ' amount,\n' + 58 | ' price,\n' + 59 | " label: 'enter short',\n" + 60 | ' })\n' + 61 | ' }\n' + 62 | ' }\n' + 63 | '\n' + 64 | ' return state\n' + 65 | '}', 66 | onUpdateLong: '({ HFS, HFU }) => async (state = {}, update = {}) => {\n' + 67 | ' const { price, mts } = update\n' + 68 | ' const i = HFS.indicators(state)\n' + 69 | ' const iv = HFS.indicatorValues(state)\n' + 70 | ' const { emaS } = i\n' + 71 | ' const l = iv.emaL\n' + 72 | ' const s = iv.emaS\n' + 73 | ' \n' + 74 | ' if (emaS.crossed(l) && s < l) {\n' + 75 | ' return HFS.closePositionMarket(state, {\n' + 76 | ' price,\n' + 77 | ' mtsCreate: mts,\n' + 78 | " label: 'close long',\n" + 79 | ' })\n' + 80 | ' }\n' + 81 | ' \n' + 82 | ' return state\n' + 83 | '}', 84 | onUpdateShort: '({ HFS, HFU }) => async (state = {}, update = {}) => {\n' + 85 | ' const { price, mts } = update\n' + 86 | ' const i = HFS.indicators(state)\n' + 87 | ' const iv = HFS.indicatorValues(state)\n' + 88 | ' const { emaS } = i\n' + 89 | ' const l = iv.emaL\n' + 90 | ' const s = iv.emaS\n' + 91 | ' \n' + 92 | ' if (emaS.crossed(l) && s > l) {\n' + 93 | ' return HFS.closePositionMarket(state, {\n' + 94 | ' price,\n' + 95 | ' mtsCreate: mts,\n' + 96 | " label: 'close short',\n" + 97 | ' })\n' + 98 | ' }\n' + 99 | ' \n' + 100 | ' return state\n' + 101 | '}', 102 | id: null 103 | } 104 | 105 | const msg = [ 106 | 'exec.str', 107 | [ 108 | 'bitfinex', 109 | 1610029361708, 110 | 1610114861708, 111 | 'tBTCUSD', 112 | '15m', 113 | true, 114 | false, 115 | true, 116 | strategy, 117 | '1610115118349-tBTCUSD-15m-1610029361708-1610114861708' 118 | ] 119 | ] 120 | 121 | const bugMsg = [ 122 | 'exec.str', 123 | [ 124 | 'bitfinex', 125 | 1610029109969, 126 | 1610114609969, 127 | 'tADABTC', 128 | '15m', 129 | true, 130 | false, 131 | true, 132 | strategy, 133 | '1610115118342-tADABTC-15m-1610029109969-1610114609969' 134 | ] 135 | ] 136 | -------------------------------------------------------------------------------- /examples/server-lowdb.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const path = require('path') 4 | 5 | process.env.DEBUG = 'bfx:hf:*' 6 | 7 | require('bfx-hf-util/lib/catch_uncaught_errors') 8 | require('dotenv').config() 9 | 10 | const HFDB = require('bfx-hf-models') 11 | const HFDBLowDBAdapter = require('bfx-hf-models-adapter-lowdb') 12 | const { schema: HFDBBitfinexSchema } = require('bfx-hf-ext-plugin-bitfinex') 13 | const DataServer = require('../lib/server') 14 | 15 | const { DB_FILENAME = 'test-db.json' } = process.env 16 | 17 | const db = new HFDB({ 18 | schema: HFDBBitfinexSchema, 19 | adapter: HFDBLowDBAdapter({ 20 | dbPath: path.join(__dirname, '..', DB_FILENAME), 21 | schema: HFDBBitfinexSchema 22 | }) 23 | }) 24 | 25 | const ds = new DataServer({ 26 | port: 8899, 27 | db 28 | }) 29 | 30 | ds.open() 31 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = require('./lib/server') 4 | -------------------------------------------------------------------------------- /lib/bt/constants.js: -------------------------------------------------------------------------------- 1 | const MINUTE = 60 * 1000 2 | 3 | const MAX_ITEMS_PER_REQUEST = 10000 4 | const MAX_REQUESTS_PER_MINUTE = 90 5 | const DEFAULT_ITEMS_LIMIT = 1000 6 | const CANDLE_FETCH_SECTION = 'hist' 7 | 8 | module.exports = { 9 | MINUTE, 10 | MAX_ITEMS_PER_REQUEST, 11 | MAX_REQUESTS_PER_MINUTE, 12 | DEFAULT_ITEMS_LIMIT, 13 | CANDLE_FETCH_SECTION 14 | } 15 | -------------------------------------------------------------------------------- /lib/bt/context.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events') 2 | 3 | class ExecutionContext extends EventEmitter { 4 | constructor () { 5 | super() 6 | this.done = false 7 | } 8 | 9 | close () { 10 | if (this.done) return 11 | this.done = true 12 | this.emit('done') 13 | this.removeAllListeners() 14 | } 15 | } 16 | 17 | module.exports = ExecutionContext 18 | -------------------------------------------------------------------------------- /lib/bt/data_feed.js: -------------------------------------------------------------------------------- 1 | const Heap = require('heap') 2 | const EventEmitter = require('events') 3 | 4 | /** 5 | * @typedef {{mts: number}} DataPoint 6 | */ 7 | 8 | const compareDataPoints = (a, b) => { 9 | return a.mts - b.mts 10 | } 11 | 12 | class DataPointFeed extends EventEmitter { 13 | constructor () { 14 | super() 15 | this.heap = new Heap(compareDataPoints) 16 | this.closed = false 17 | this.streams = [] 18 | this.windowLowerBound = -1 19 | } 20 | 21 | /** 22 | * @param {DataPoint} item 23 | */ 24 | push (item) { 25 | if (this.closed) { 26 | return 27 | } 28 | 29 | this.heap.push(item) 30 | } 31 | 32 | peek () { 33 | return this.heap.peek() 34 | } 35 | 36 | pop () { 37 | return this.heap.pop() 38 | } 39 | 40 | size () { 41 | return this.heap.size() 42 | } 43 | 44 | /** 45 | * @param {DataPointStream} stream 46 | */ 47 | addStream (stream) { 48 | this.streams.push(stream) 49 | } 50 | 51 | /** 52 | * @param {ExecutionContext} ctx 53 | * @param {number} start 54 | * @param {number} end 55 | * @return {Promise} 56 | */ 57 | start (ctx, start, end) { 58 | const streamsInProgress = this.streams.map((stream) => 59 | stream.fetchPaginated(this, ctx, start, end) 60 | ) 61 | 62 | ctx.once('done', () => this.close()) 63 | 64 | return Promise.all(streamsInProgress) 65 | .then(() => this.close()) 66 | .catch((err) => this.emit('error', err)) 67 | } 68 | 69 | requestDrain () { 70 | // each stream works as a data window 71 | // allow consumers to drain until the lower bound that is common to all windows 72 | // this way we don't send data out of sync 73 | this.windowLowerBound = Math.min( 74 | ...this.streams.map(stream => stream.offset) 75 | ) 76 | 77 | if (this.windowLowerBound > 0) { 78 | this.emit('drain') 79 | } 80 | } 81 | 82 | close () { 83 | if (this.closed) return 84 | this.closed = true 85 | this.emit('close') 86 | this.removeAllListeners() 87 | } 88 | } 89 | 90 | module.exports = DataPointFeed 91 | -------------------------------------------------------------------------------- /lib/bt/data_stream.js: -------------------------------------------------------------------------------- 1 | class DataPointStream { 2 | /** 3 | * @param {function(start: number, end: number): Promise} fetchDataPoints 4 | */ 5 | constructor (fetchDataPoints) { 6 | this.offset = -1 7 | this.fetchDataPoints = fetchDataPoints 8 | } 9 | 10 | /** 11 | * @param {DataPointFeed} feed 12 | * @param {ExecutionContext} ctx 13 | * @param {number} start 14 | * @param {number} end 15 | */ 16 | async fetchPaginated (feed, ctx, start, end) { 17 | this.offset = start 18 | 19 | while (this.offset < end) { 20 | if (ctx.done) { 21 | return 22 | } 23 | 24 | const dataPoints = await this.fetchDataPoints(this.offset, end) 25 | 26 | if (dataPoints.length === 0) { 27 | break 28 | } 29 | 30 | dataPoints.forEach(dp => feed.push(dp)) 31 | const lastDataPoint = dataPoints[dataPoints.length - 1] 32 | this.offset = lastDataPoint.mts + 1 33 | 34 | feed.requestDrain() 35 | } 36 | 37 | this.offset = Math.max(this.offset, end) 38 | } 39 | } 40 | 41 | module.exports = DataPointStream 42 | -------------------------------------------------------------------------------- /lib/bt/fetch_data.js: -------------------------------------------------------------------------------- 1 | const { DEFAULT_ITEMS_LIMIT, CANDLE_FETCH_SECTION } = require('./constants') 2 | 3 | /** 4 | * @param {RESTv2} rest 5 | * @param {RequestSemaphore} semaphore 6 | * @param {string} symbol 7 | * @param {string} timeframe 8 | * @return {function(*, *): *} 9 | */ 10 | const fetchCandles = (rest, semaphore, { symbol, timeframe }) => { 11 | return async (offset, end, limit = DEFAULT_ITEMS_LIMIT) => { 12 | const candles = await semaphore.add( 13 | rest.candles.bind(rest, { 14 | timeframe, 15 | symbol, 16 | section: CANDLE_FETCH_SECTION, 17 | query: { 18 | limit, 19 | start: offset, 20 | end, 21 | sort: 1 22 | } 23 | }) 24 | ) 25 | 26 | candles.forEach(candle => { 27 | candle.tf = timeframe 28 | candle.symbol = symbol 29 | }) 30 | 31 | return candles 32 | } 33 | } 34 | 35 | /** 36 | * @param {RESTv2} rest 37 | * @param {RequestSemaphore} semaphore 38 | * @param {string} symbol 39 | * @return {function(*, *): *} 40 | */ 41 | const fetchTrades = (rest, semaphore, { symbol }) => { 42 | return (offset, end, limit = DEFAULT_ITEMS_LIMIT, sort = 1) => { 43 | return semaphore.add( 44 | rest.trades.bind(rest, symbol, offset, end, limit, sort) 45 | ) 46 | } 47 | } 48 | 49 | module.exports = { 50 | fetchTrades, 51 | fetchCandles 52 | } 53 | -------------------------------------------------------------------------------- /lib/bt/request_semaphore.js: -------------------------------------------------------------------------------- 1 | const delay = require('../util/delay') 2 | const { MINUTE, MAX_REQUESTS_PER_MINUTE } = require('./constants') 3 | 4 | const MAX_RETRIES = 3 5 | 6 | class RequestSemaphore { 7 | constructor ({ 8 | maxRequests = MAX_REQUESTS_PER_MINUTE, 9 | maxTries = MAX_RETRIES, 10 | interval = MINUTE 11 | } = {}) { 12 | this.queue = [] 13 | this.capacity = 0 14 | this.isSuspended = false 15 | 16 | this.maxRequests = maxRequests 17 | this.maxTries = maxTries 18 | this.interval = interval 19 | 20 | this.dequeue = this.dequeue.bind(this) 21 | } 22 | 23 | add (req) { 24 | return new Promise((resolve, reject) => { 25 | this.queue.push({ req, resolve, reject }) 26 | this.dequeue() 27 | }) 28 | } 29 | 30 | dequeue () { 31 | if (this.queue.length === 0) { 32 | return 33 | } 34 | if (!this.timeSpan || Date.now() > this.timeSpan) { 35 | this.reset() 36 | } 37 | if (this.capacity === 0 || this.isSuspended) { 38 | setTimeout(this.dequeue, this.interval) 39 | return 40 | } 41 | 42 | this.capacity-- 43 | const { req, resolve, reject, retries = 0 } = this.queue.shift() 44 | 45 | req() 46 | .then(resolve) 47 | .catch((err) => { 48 | if (err.message.includes('ratelimit')) { 49 | this._rateLimitReached() 50 | } 51 | if (retries >= this.maxTries) { 52 | return reject(err) 53 | } 54 | this.queue.push({ req, resolve, reject, retries: retries + 1 }) 55 | }) 56 | .finally(this.dequeue) 57 | } 58 | 59 | reset () { 60 | this.timeSpan = Date.now() + this.interval 61 | this.capacity = this.maxRequests 62 | } 63 | 64 | _rateLimitReached () { 65 | this.isSuspended = true 66 | 67 | delay(this.interval) 68 | .then(() => { 69 | this.isSuspended = false 70 | this.dequeue() 71 | }) 72 | } 73 | } 74 | 75 | module.exports = RequestSemaphore 76 | -------------------------------------------------------------------------------- /lib/bt/seed_candles.js: -------------------------------------------------------------------------------- 1 | const { candleWidth } = require('bfx-hf-util') 2 | const { onSeedCandle } = require('bfx-hf-strategy') 3 | 4 | module.exports = ({ symbol, timeframe, fetchCandles, start, candleSeed }) => { 5 | return async (state) => { 6 | const cWidth = candleWidth(timeframe) 7 | const seedStart = start - (candleSeed * cWidth) 8 | const candles = await fetchCandles(seedStart, start, candleSeed) 9 | 10 | for (const candle of candles) { 11 | candle.tf = timeframe 12 | candle.symbol = symbol 13 | 14 | state = await onSeedCandle(state, candle) 15 | } 16 | 17 | return state 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /lib/bt/stream_data.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const _isEmpty = require('lodash/isEmpty') 4 | const send = require('../wss/send') 5 | 6 | module.exports = (ws, start, end, trades = [], candles = []) => { 7 | send(ws, ['bt.start', '', '', start, end, null, trades.length, candles.length]) 8 | 9 | if (_isEmpty(trades)) { // no trades, send only candles 10 | candles.forEach(c => { send(ws, ['bt.candle', '', '', c]) }) 11 | } else if (_isEmpty(candles)) { // no candles, send only trades 12 | trades.forEach(t => { send(ws, ['bt.trade', '', t]) }) 13 | } else { // mixed response 14 | let tradeI = 0 15 | 16 | // go through candles, advancing through trades as needed 17 | candles.forEach(c => { 18 | while (tradeI < trades.length && trades[tradeI].mts < c.mts) { 19 | send(ws, ['bt.trade', '', trades[tradeI]]) 20 | tradeI++ 21 | } 22 | 23 | send(ws, ['bt.candle', '', '', c]) 24 | }) 25 | } 26 | 27 | send(ws, ['bt.end', '', '', start, end]) 28 | } 29 | -------------------------------------------------------------------------------- /lib/bt/sync_data.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('debug')('bfx:hf:data-server:bt:sync-data') 4 | const send = require('../wss/send') 5 | 6 | module.exports = async (db, ws, btArgs) => { 7 | const { 8 | exchange, symbol, tf, start, end, includeTrades, includeCandles 9 | } = btArgs 10 | 11 | const { Trade, Candle } = db 12 | 13 | debug('syncing data...') 14 | send(ws, ['data.sync.start', symbol, tf, start, end]) 15 | 16 | const syncTasks = [] 17 | if (includeCandles) { 18 | syncTasks.push(Candle.syncRange({ exchange, symbol, tf }, { start, end })) 19 | } 20 | 21 | if (includeTrades) { 22 | syncTasks.push(Trade.syncRange({ exchange, symbol }, { start, end })) 23 | } 24 | 25 | await Promise.all(syncTasks) 26 | 27 | send(ws, ['data.sync.end', symbol, tf, start, end]) 28 | debug('sync ended') 29 | } 30 | -------------------------------------------------------------------------------- /lib/cmds/delete_all_bts.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { deleteAllBts } = require('../db/bt_dao') 4 | const send = require('../wss/send') 5 | 6 | module.exports = async (ds, ws, msg) => { 7 | const [, strategyId] = msg 8 | 9 | await deleteAllBts(strategyId) 10 | send(ws, ['data.bt.history.all.deleted', strategyId]) 11 | } 12 | -------------------------------------------------------------------------------- /lib/cmds/delete_bt.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { deleteBt } = require('../db/bt_dao') 4 | const send = require('../wss/send') 5 | 6 | module.exports = async (ds, ws, msg) => { 7 | const [, executionId] = msg 8 | 9 | await deleteBt(executionId) 10 | send(ws, ['data.bt.history.deleted', executionId]) 11 | } 12 | -------------------------------------------------------------------------------- /lib/cmds/exec_bt.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('debug')('bfx:hf:data-server:cmds:exec-bt') 4 | const { rangeString } = require('bfx-hf-util') 5 | 6 | const validateBTArgs = require('../util/validate_bt_args') 7 | const parseExecMsg = require('../util/parse_exec_msg') 8 | const sendError = require('../wss/send_error') 9 | const syncBTData = require('../bt/sync_data') 10 | const send = require('../wss/send') 11 | 12 | module.exports = async (ds, ws, msg) => { 13 | const { db } = ds 14 | const err = validateBTArgs(msg) 15 | 16 | if (err !== null) { 17 | return sendError(ws, err) 18 | } 19 | 20 | const { Candle, Trade } = db 21 | const btArgs = parseExecMsg(msg) 22 | const { 23 | exchange, sync, symbol, tf, includeTrades, includeCandles, start, end, meta 24 | } = btArgs 25 | 26 | if (sync) { 27 | await syncBTData(db, ws, btArgs) 28 | } 29 | 30 | debug( 31 | 'running backtest for %s:%s [%s]', 32 | symbol, tf, rangeString(start, end) 33 | ) 34 | 35 | const candleData = await Candle.getInRange([ 36 | ['exchange', '=', exchange], 37 | ['symbol', '=', symbol], 38 | ['tf', '=', tf] 39 | ], { 40 | key: 'mts', 41 | start, 42 | end 43 | }, { 44 | orderBy: 'mts', 45 | orderDirection: 'asc' 46 | }) 47 | 48 | debug('loaded %d candles', candleData.length) 49 | debug('streaming data...') 50 | 51 | let sentTrades = 0 52 | let sentCandles = 0 53 | 54 | send(ws, ['bt.start', '', '', start, end,, 0, candleData.length, meta]) // eslint-disable-line 55 | 56 | if (!includeTrades) { 57 | candleData.forEach((c) => { 58 | send(ws, ['bt.candle', '', '', c, meta]) 59 | sentCandles += 1 60 | }) 61 | 62 | send(ws, ['bt.end', '', '', start, end, meta]) 63 | debug('stream complete [%d candles]', sentCandles) 64 | return // NOTE: hard return 65 | } 66 | 67 | const trades = await Trade.getInRange([ 68 | ['exchange', '=', exchange], 69 | ['symbol', '=', symbol] 70 | ], { 71 | key: 'mts', 72 | start, 73 | end 74 | }, { 75 | orderBy: 'mts', 76 | orderDirection: 'asc' 77 | }) 78 | 79 | trades.sort((a, b) => a.mts - b.mts) 80 | debug('loaded %d trades', trades.length) 81 | 82 | let trade 83 | let candleI = 0 84 | 85 | for (let i = 0; i < trades.length; i += 1) { 86 | trade = trades[i] 87 | 88 | if (includeCandles) { 89 | while ( 90 | (candleI < candleData.length) && 91 | (candleData[candleI].mts < trade.mts) 92 | ) { 93 | send(ws, ['bt.candle', '', '', candleData[candleI], meta]) 94 | candleI++ 95 | sentCandles++ 96 | } 97 | } 98 | 99 | send(ws, ['bt.trade', '', trade, meta]) 100 | sentTrades++ 101 | } 102 | 103 | send(ws, ['bt.end', '', '', start, end, meta]) 104 | debug('stream complete [%d candles, %d trades]', sentCandles, sentTrades) 105 | } 106 | -------------------------------------------------------------------------------- /lib/cmds/exec_strategy.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('debug')('bfx:hf:data-server:cmds:exec-bt') 4 | 5 | const { rangeString } = require('bfx-hf-util') 6 | const { execOffline } = require('bfx-hf-backtest') 7 | const HFS = require('bfx-hf-strategy') 8 | const Indicators = require('bfx-hf-indicators') 9 | const { PriceFeed, PerformanceManager, StartWatchers: startPerformanceWatchers } = require('bfx-hf-strategy-perf') 10 | const { RESTv2 } = require('bfx-api-node-rest') 11 | 12 | const validateBTArgs = require('../util/validate_bt_args') 13 | const sendError = require('../wss/send_error') 14 | const send = require('../wss/send') 15 | const DataPointFeed = require('../bt/data_feed') 16 | const DataPointStream = require('../bt/data_stream') 17 | const { 18 | fetchCandles: fetchCandlesFactory, 19 | fetchTrades: fetchTradesFactory 20 | } = require('../bt/fetch_data') 21 | const ExecutionContext = require('../bt/context') 22 | const RequestSemaphore = require('../bt/request_semaphore') 23 | const seedCandlesFactory = require('../bt/seed_candles') 24 | const generateResults = require('bfx-hf-strategy/lib/util/generate_strategy_results') 25 | const parseStrategy = require('bfx-hf-strategy/lib/util/parse_strategy') 26 | const btDao = require('../db/bt_dao') 27 | const getDerivativesConfig = require('../util/get_derivatives_config') 28 | 29 | /** 30 | * @param {DataServer} ds 31 | * @param {WebSocket} ws 32 | * @param {Array} msg 33 | * @return {Promise} 34 | */ 35 | module.exports = async (ds, ws, msg) => { 36 | const err = validateBTArgs(msg) 37 | 38 | if (err !== null) { 39 | return sendError(ws, err) 40 | } 41 | const [ 42 | // eslint-disable-next-line no-unused-vars 43 | exchange, strategyId, start, end, symbol, timeframe, includeCandles, includeTrades, candleSeed, sync = true, margin = false, strategyContent, executionId, constraints = {}, leverageSettings = {}, stopOrderSettings = {} 44 | ] = msg[1] || [] 45 | const { capitalAllocation, stopLossPerc, maxDrawdownPerc } = constraints 46 | const { useMaxLeverage = false, increaseLeverage = false, leverage = 0 } = leverageSettings 47 | const { addStopOrder = false, stopOrderPercent = 0 } = stopOrderSettings 48 | 49 | let strategy 50 | try { 51 | strategy = parseStrategy(strategyContent) 52 | } catch (e) { 53 | console.log(e) 54 | return send(ws, ['bt.btresult', { error: 'Strategy could not get parsed - parse error' }, executionId]) 55 | } 56 | 57 | const priceFeed = new PriceFeed() 58 | const perfManager = new PerformanceManager(priceFeed, { allocation: capitalAllocation }) 59 | 60 | try { 61 | const indicators = strategy.defineIndicators 62 | ? strategy.defineIndicators(Indicators) 63 | : {} 64 | 65 | strategy = HFS.define({ 66 | ...strategy, 67 | tf: timeframe, 68 | symbol, 69 | indicators, 70 | priceFeed, 71 | perfManager 72 | }) 73 | } catch (e) { 74 | return send(ws, ['bt.btresult', { error: 'Strategy is invalid' }, executionId]) 75 | } 76 | 77 | const context = new ExecutionContext() 78 | const dataPointFeed = new DataPointFeed() 79 | const rest = new RESTv2({ transform: true }) 80 | const requestSemaphore = new RequestSemaphore() 81 | let fetchCandles, seedCandles 82 | 83 | debug( 84 | 'running backtest for %s:%s [%s]', 85 | symbol, timeframe, rangeString(start, end) 86 | ) 87 | 88 | if (includeCandles) { 89 | fetchCandles = fetchCandlesFactory(rest, requestSemaphore, { symbol, timeframe }) 90 | const stream = new DataPointStream(fetchCandles) 91 | dataPointFeed.addStream(stream) 92 | } 93 | 94 | if (includeTrades) { 95 | const fetchStrategy = fetchTradesFactory(rest, requestSemaphore, { symbol }) 96 | const stream = new DataPointStream(fetchStrategy) 97 | dataPointFeed.addStream(stream) 98 | } 99 | 100 | if (includeCandles && candleSeed) { 101 | seedCandles = seedCandlesFactory({ 102 | symbol, 103 | timeframe, 104 | fetchCandles, 105 | start, 106 | candleSeed 107 | }) 108 | } 109 | 110 | ds.activeBacktests.set(strategy.gid, context) 111 | 112 | let executionError 113 | const reportError = (err) => { 114 | console.error(err) 115 | executionError = err.message 116 | sendError(ws, { code: 600, res: err.message }) 117 | } 118 | 119 | let count = 0 120 | let progressSent = 0 121 | const reportProgress = (mts) => { 122 | count++ 123 | 124 | if (includeCandles || (count % 50) === 0) { 125 | const progress = Math.round((mts - start) / (end - start) * 100) 126 | if (progress > progressSent) { 127 | progressSent = progress 128 | send(ws, ['bt.progress', progress, executionId]) 129 | } 130 | } 131 | } 132 | 133 | // returns null if not derivatives symbol 134 | const symbolConfig = await getDerivativesConfig(symbol) 135 | const isDerivative = !!symbolConfig 136 | const maxLeverage = symbolConfig ? symbolConfig.maxLeverage : 0 137 | 138 | const args = { 139 | start, 140 | end, 141 | includeTrades, 142 | includeCandles, 143 | candleSeed, 144 | seedCandles, 145 | priceFeed, 146 | perfManager, 147 | startPerformanceWatchers, 148 | constraints: { 149 | maxDrawdown: maxDrawdownPerc, 150 | percStopLoss: stopLossPerc 151 | }, 152 | context, 153 | dataPointFeed, 154 | reportError, 155 | reportProgress, 156 | margin, 157 | isDerivative, 158 | maxLeverage, 159 | useMaxLeverage, 160 | increaseLeverage, 161 | leverage, 162 | addStopOrder, 163 | stopOrderPercent 164 | } 165 | 166 | send(ws, ['bt.started', strategy.gid]) 167 | 168 | return execOffline(strategy, args) 169 | .then(async (btState = {}) => { 170 | // check if error received during execution 171 | if (executionError) { 172 | send(ws, ['bt.btresult', { error: executionError }, executionId]) 173 | return 174 | } 175 | 176 | const { nCandles, nTrades, strategy = {} } = btState 177 | const res = generateResults(perfManager, { 178 | ...strategy, 179 | nCandles, 180 | nTrades 181 | }) 182 | 183 | send(ws, ['bt.btresult', res, executionId]) 184 | 185 | // save to bt history db 186 | const savedBt = await btDao.saveBt(msg[1], res) 187 | send(ws, ['data.bt.saved', executionId, savedBt]) 188 | }) 189 | .catch(reportError) 190 | .finally(() => { 191 | send(ws, ['bt.stopped', strategy.gid]) 192 | ds.activeBacktests.delete(strategy.gid) 193 | context.close() 194 | priceFeed.close() 195 | perfManager.close() 196 | }) 197 | } 198 | -------------------------------------------------------------------------------- /lib/cmds/get_bt_history.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { getBtHistory } = require('../db/bt_dao') 4 | const send = require('../wss/send') 5 | 6 | module.exports = async (ds, ws, msg) => { 7 | const [, strategyId] = msg 8 | const btHistory = await getBtHistory(strategyId) 9 | 10 | send(ws, ['data.bt.history.list', strategyId, btHistory]) 11 | } 12 | -------------------------------------------------------------------------------- /lib/cmds/get_bts.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const send = require('../wss/send') 4 | 5 | module.exports = async (ds, ws) => { 6 | const { db } = ds 7 | const { Backtest } = db 8 | 9 | const bts = await Backtest.getAll() 10 | 11 | send(ws, ['data.bts', bts]) 12 | } 13 | -------------------------------------------------------------------------------- /lib/cmds/get_candles.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('debug')('bfx:hf:data-server:cmds:get-candles') 4 | const send = require('../wss/send') 5 | 6 | module.exports = async (ds, ws, msg) => { 7 | const [, exchange, symbol, tf, type, start, end, meta] = msg 8 | const { db } = ds 9 | const { Candle } = db 10 | 11 | let optimizedRange = ds.optimizeSyncRange({ 12 | exchange, 13 | symbol, 14 | tf, 15 | start, 16 | end 17 | }) 18 | 19 | if (optimizedRange) { // null if no sync required 20 | let syncRequired = true 21 | let futureSync = ds.futureSyncFor(optimizedRange) 22 | 23 | // Notify even if sync is later redundant; we notify end even if redundant 24 | ds.notifySyncStart({ exchange, symbol, tf, start, end }) 25 | send(ws, ['data.sync.start', exchange, symbol, tf, start, end, meta]) 26 | 27 | while (futureSync) { 28 | debug( 29 | 'waiting for future sync to complete (%d - %d)', 30 | futureSync.start, futureSync.end 31 | ) 32 | 33 | await ds.expectSync(futureSync) 34 | 35 | // Optimise range again 36 | optimizedRange = ds.optimizeSyncRange({ 37 | exchange, 38 | symbol, 39 | tf, 40 | start, 41 | end 42 | }) 43 | 44 | if (!optimizedRange) { 45 | syncRequired = false 46 | break 47 | } 48 | 49 | futureSync = ds.futureSyncFor(optimizedRange) // check again 50 | } 51 | 52 | if (syncRequired) { 53 | await Candle.syncRange({ 54 | exchange, 55 | symbol, 56 | tf 57 | }, { 58 | start, 59 | end 60 | }) 61 | } 62 | 63 | ds.notifySyncEnd({ exchange, symbol, tf, start, end }) 64 | send(ws, ['data.sync.end', exchange, symbol, tf, start, end, meta]) 65 | } 66 | 67 | const candles = await Candle.getInRange([ 68 | ['exchange', '=', exchange], 69 | ['symbol', '=', symbol], 70 | ['tf', '=', tf] 71 | ], { 72 | key: 'mts', 73 | start, 74 | end 75 | }, { 76 | orderBy: 'mts', 77 | orderDirection: 'asc' 78 | }) 79 | 80 | debug( 81 | 'responding with %d candles for range %d - %d [%s %s]', 82 | candles.length, start, end, symbol, tf 83 | ) 84 | 85 | send(ws, ['data.candles', exchange, symbol, tf, type, start, end, meta, candles]) 86 | 87 | return candles 88 | } 89 | -------------------------------------------------------------------------------- /lib/cmds/get_trades.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const send = require('../wss/send') 4 | 5 | module.exports = async (ds, ws, msg) => { 6 | // NOTE: 'type' is currently unused, but will be used to differentiate between 7 | // funding & normal trades 8 | const [, symbol,, from, to] = msg 9 | const { db } = ds 10 | const { Trade } = db 11 | 12 | const trades = await Trade.getInRange([['symbol', '=', symbol]], { 13 | key: 'mts', 14 | start: from, 15 | end: to 16 | }) 17 | 18 | send(ws, ['data.trades', symbol, from, to, trades]) 19 | 20 | return trades 21 | } 22 | -------------------------------------------------------------------------------- /lib/cmds/set_bt_favorite.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { setBtFavorite } = require('../db/bt_dao') 4 | const send = require('../wss/send') 5 | 6 | module.exports = async (ds, ws, msg) => { 7 | const [, executionId, isFavorite] = msg 8 | 9 | await setBtFavorite(executionId, isFavorite) 10 | send(ws, ['data.bt.history.favorite', executionId, !!isFavorite]) 11 | } 12 | -------------------------------------------------------------------------------- /lib/cmds/stop_bt.js: -------------------------------------------------------------------------------- 1 | const send = require('../wss/send') 2 | 3 | /** 4 | * @param {DataServer} ds 5 | * @param {WebSocket} ws 6 | * @param {Array} msg 7 | * @return {Promise} 8 | */ 9 | module.exports = async (ds, ws, msg) => { 10 | const [, gid] = msg 11 | 12 | if (ds.activeBacktests.has(gid)) { 13 | const context = ds.activeBacktests.get(gid) 14 | context.close() 15 | } 16 | 17 | send(ws, ['bt.stopped', gid]) 18 | } 19 | -------------------------------------------------------------------------------- /lib/cmds/submit_bt.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('debug')('bfx:hf:data-server:cmds:submit-bt') 4 | const _isFinite = require('lodash/isFinite') 5 | 6 | const ERRORS = require('../errors') 7 | const send = require('../wss/send') 8 | const sendError = require('../wss/send_error') 9 | 10 | module.exports = async (ds, ws, msg) => { 11 | const [ 12 | btID, strategyID, indicators, trades, symbol, tf, from, to 13 | ] = msg[1] 14 | 15 | const { db } = ds 16 | const { Backtest } = db 17 | 18 | if (!_isFinite(btID)) { 19 | return sendError(ws, ERRORS.BACKTEST.BT_ID_REQUIRED) 20 | } else if (!_isFinite(strategyID)) { 21 | return sendError(ws, ERRORS.BACKTEST.ST_ID_REQUIRED) 22 | } 23 | 24 | const existingBT = await Backtest.get({ btID, strategyID }) 25 | 26 | if (existingBT) { 27 | return sendError(ws, ERRORS.BACKTEST.DUPLICATE) 28 | } 29 | 30 | debug('creating backtest %s', btID) 31 | 32 | const bt = await Backtest.create({ 33 | btID, strategyID, indicators, symbol, trades, from, to, tf 34 | }) 35 | 36 | send(ws, ['data.bt', bt]) 37 | } 38 | -------------------------------------------------------------------------------- /lib/db/bt_dao.js: -------------------------------------------------------------------------------- 1 | const sqliteDb = require('./sqlite_db') 2 | 3 | const saveBt = async (args, btResult) => { 4 | const [ 5 | exchange, strategyId, start, end, symbol, timeframe, 6 | includeCandles, includeTrades, candleSeed, 7 | sync = true, margin, , executionId, 8 | { capitalAllocation, stopLossPerc, maxDrawdownPerc }, 9 | { useMaxLeverage, increaseLeverage, leverage }, 10 | { addStopOrder, stopOrderPercent } 11 | ] = args 12 | 13 | const isFavorite = 0 14 | const timestamp = Date.now() 15 | 16 | const query = 'insert into bt_history (exchange, strategyId, start, end, symbol, timeframe, includeCandles, includeTrades, candleSeed, sync, margin, executionId, capitalAllocation, stopLossPerc, maxDrawdownPerc, useMaxLeverage, increaseLeverage, leverage, addStopOrder, stopOrderPercent, isFavorite, timestamp, btResult) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)' 17 | const values = [ 18 | exchange, strategyId, start, end, symbol, timeframe, 19 | includeCandles, includeTrades, candleSeed, 20 | sync, margin, executionId, 21 | capitalAllocation, stopLossPerc, maxDrawdownPerc, 22 | useMaxLeverage, increaseLeverage, leverage, 23 | addStopOrder, stopOrderPercent, 24 | isFavorite, timestamp, JSON.stringify(btResult) 25 | ] 26 | 27 | await sqliteDb.createData(query, values) 28 | 29 | const savedBt = { 30 | exchange, 31 | strategyId, 32 | start, 33 | end, 34 | symbol, 35 | timeframe, 36 | includeCandles, 37 | includeTrades, 38 | candleSeed, 39 | sync, 40 | executionId, 41 | capitalAllocation, 42 | stopLossPerc, 43 | maxDrawdownPerc, 44 | margin, 45 | useMaxLeverage, 46 | increaseLeverage, 47 | leverage, 48 | addStopOrder, 49 | stopOrderPercent, 50 | isFavorite: false, 51 | timestamp, 52 | btResult 53 | } 54 | 55 | return savedBt 56 | } 57 | 58 | const getBtHistory = async (strategyId) => { 59 | const query = 'SELECT * FROM bt_history where strategyId=?' 60 | const values = [strategyId] 61 | 62 | const btHistory = await sqliteDb.queryData(query, values) 63 | if (btHistory.length === 0) { 64 | return btHistory 65 | } 66 | const normalizedBtHistory = btHistory.map((bt) => { 67 | const parsedResult = bt?.btResult ? JSON.parse(bt.btResult) : {} 68 | return { 69 | ...bt, 70 | btResult: parsedResult, 71 | // Transform binary values to boolean type after SQLite 72 | includeCandles: !!bt.includeCandles, 73 | includeTrades: !!bt.includeTrades, 74 | sync: !!bt.sync, 75 | isFavorite: !!bt.isFavorite, 76 | margin: !!bt.margin, 77 | useMaxLeverage: !!bt.useMaxLeverage, 78 | increaseLeverage: !!bt.increaseLeverage, 79 | addStopOrder: !!bt.addStopOrder 80 | } 81 | }) 82 | return normalizedBtHistory 83 | } 84 | 85 | const setBtFavorite = async (executionId, isFavorite) => { 86 | const query = 'update bt_history set isFavorite=? where executionId=?' 87 | const values = [isFavorite, executionId] 88 | await sqliteDb.executeQuery(query, values) 89 | } 90 | 91 | const deleteBt = async (executionId) => { 92 | const query = 'delete from bt_history where executionId=?' 93 | const values = [executionId] 94 | await sqliteDb.executeQuery(query, values) 95 | } 96 | 97 | const deleteAllBts = async (strategyId) => { 98 | const query = 'delete from bt_history where strategyId=?' 99 | const values = [strategyId] 100 | await sqliteDb.executeQuery(query, values) 101 | } 102 | 103 | module.exports = { saveBt, getBtHistory, setBtFavorite, deleteBt, deleteAllBts } 104 | -------------------------------------------------------------------------------- /lib/db/sqlite_db.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('debug')('bfx:hf:data-server:db:sqlite_db') 4 | const Sqlite = require('bfx-facs-db-sqlite') 5 | let sqliteDb 6 | 7 | // connect to db 8 | const connectDb = async (sqlitePath) => { 9 | const opts = { name: 'hf_ds', label: '', dbPathAbsolute: sqlitePath } 10 | const sqlite = new Sqlite(this, opts, {}) 11 | 12 | return new Promise((resolve, reject) => { 13 | sqlite.start(async () => { 14 | if (!sqlite.db) reject(new Error('sqlite connection failed')) 15 | 16 | debug('sqlite connected') 17 | sqliteDb = sqlite.db 18 | // create tables after db connected 19 | await createTables() 20 | 21 | resolve(true) 22 | }) 23 | }) 24 | } 25 | 26 | const createTables = async () => { 27 | const createBTHistory = 'create table if not exists bt_history (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, exchange VARCHAR(20), strategyId VARCHAR(50), start INTEGER, end INTEGER, symbol VARCHAR(20), timeframe VARCHAR(10), includeCandles TINYINT(1), includeTrades TINYINT(1), candleSeed INTEGER, sync TINYINT(1), executionId VARCHAR(50), capitalAllocation DOUBLE, stopLossPerc DOUBLE, maxDrawdownPerc DOUBLE, margin TINYINT(1), useMaxLeverage TINYINT(1), increaseLeverage TINYINT(1), leverage INTEGER, addStopOrder TINYINT(1), stopOrderPercent DOUBLE, isFavorite TINYINT(1), timestamp INTEGER, btResult TEXT)' 28 | await executeQuery(createBTHistory) 29 | 30 | // add new columns with alter query to support previous version table 31 | await addNewColumns() 32 | } 33 | 34 | const addNewColumns = async () => { 35 | const addedColumns = await queryData('pragma table_info(bt_history)') 36 | if (addedColumns.length === 24) { // expected columns count 37 | debug('all table columns added') 38 | return 39 | } 40 | 41 | const alterQueries = [ 42 | 'alter table bt_history add column margin TINYINT(1) DEFAULT 0', 43 | 'alter table bt_history add column useMaxLeverage TINYINT(1) DEFAULT 0', 44 | 'alter table bt_history add column increaseLeverage TINYINT(1) DEFAULT 0', 45 | 'alter table bt_history add column leverage INTEGER DEFAULT 0', 46 | 'alter table bt_history add column addStopOrder TINYINT(1) DEFAULT 0', 47 | 'alter table bt_history add column stopOrderPercent DOUBLE DEFAULT 0' 48 | ] 49 | 50 | const queryPromises = [] 51 | for (const query of alterQueries) { 52 | queryPromises.push(executeQuery(query)) 53 | } 54 | 55 | await Promise.all(queryPromises) 56 | debug('new columns added') 57 | } 58 | 59 | const createData = async (query, values = []) => { 60 | await executeQuery(query, values) 61 | } 62 | 63 | const executeQuery = async (query, values = []) => { 64 | if (!sqliteDb) return 65 | 66 | return new Promise((resolve, reject) => { 67 | sqliteDb.run(query, values, (err, data) => _handleDbCallback(err, data, resolve, reject)) 68 | }) 69 | } 70 | 71 | const getData = async (query, values = []) => { 72 | if (!sqliteDb) return 73 | 74 | return new Promise((resolve, reject) => { 75 | sqliteDb.get(query, values, (err, data) => _handleDbCallback(err, data, resolve, reject)) 76 | }) 77 | } 78 | 79 | const queryData = async (query, values = []) => { 80 | if (!sqliteDb) return 81 | 82 | return new Promise((resolve, reject) => { 83 | sqliteDb.all(query, values, (err, data) => _handleDbCallback(err, data, resolve, reject)) 84 | }) 85 | } 86 | 87 | const _handleDbCallback = (error, data, resolve, reject) => { 88 | if (error) { 89 | console.error(error) 90 | reject(new Error(error.toString())) 91 | } 92 | resolve(data) 93 | } 94 | 95 | module.exports = { connectDb, executeQuery, createData, getData, queryData } 96 | -------------------------------------------------------------------------------- /lib/errors.js: -------------------------------------------------------------------------------- 1 | // NOTE: error codes should never change; if removing an error, do NOT re-use 2 | // the code (better, leave a note reminding the code is taken) 3 | // 4 | // NOTE: Errors are objects to leave room for future expansion 5 | module.exports = { 6 | GENERIC: { // 1xx 7 | BFX_REST_ERROR: { res: 'bitfinex API error', code: 100 }, 8 | MSG_NOT_ARRAY: { res: 'client message not an array', code: 101 }, 9 | UNKNOWN_COMMAND: { res: 'unrecognized command', code: 102 }, 10 | INTERNAL: { res: 'internal error', code: 103 } 11 | }, 12 | 13 | BFX_PROXY: { // 4xx 14 | UNAVAILABLE: { res: 'cannot proxy message, no bfx proxy available', code: 400 } 15 | }, 16 | 17 | BACKTEST: { // 5xx 18 | BT_ID_REQUIRED: { res: 'backtest ID required', code: 500 }, 19 | ST_ID_REQUIRED: { res: 'strategy ID required', code: 502 }, 20 | 21 | INVALID_START: { res: 'invalid start time', code: 503 }, 22 | INVALID_END: { res: 'invalid end time', code: 504 }, 23 | INVALID_TF: { res: 'invalid timeframe', code: 505 }, 24 | INVALID_INCLUDE_CANDLES: { res: 'invalid includeCandles flag', code: 506 }, 25 | INVALID_INCLUDE_TRADES: { res: 'invalid includeTrades flag', code: 507 }, 26 | INVALID_SYNC: { res: 'invalid sync flag', code: 508 }, 27 | 28 | SYMBOL_NOT_STRING: { res: 'symbol not a string', code: 509 }, 29 | START_BEFORE_END: { res: 'start is before end', code: 510 }, 30 | DUPLICATE: { res: 'backtest already exists', code: 511 }, 31 | REQ_EMPTY: { res: 'requested empty backtest (no trades or candles)', code: 512 }, 32 | EXCHANGE_NOT_STRING: { res: 'exchange not a string', code: 513 }, 33 | 34 | INVALID_ALLOCATION: { res: 'invalid allocation', code: 514 }, 35 | INVALID_MAX_POSITION_SIZE: { res: 'invalid max position size', code: 515 }, 36 | INVALID_MAX_DRAWDOWN: { res: 'invalid max drawdown', code: 516 }, 37 | INVALID_ABS_STOP_LOSS: { res: 'invalid absolute stop loss', code: 517 }, 38 | INVALID_PERC_STOP_LOSS: { res: 'invalid percentage stop loss', code: 518 }, 39 | INVALID_EXIT_MODE: { res: 'invalid exit position mode', code: 519 }, 40 | INVALID_CANDLE_SEED: { res: 'invalid candle seed count', code: 520 }, 41 | INVALID_MARGIN: { res: 'invalid margin flag', code: 521 }, 42 | INVALID_USE_MAX_LEVERAGE: { res: 'invalid useMaxLeverage flag', code: 522 }, 43 | INVALID_LEVERAGE: { res: 'invalid leverage value', code: 523 }, 44 | INVALID_INCREASE_LEVERAGE: { res: 'invalid increaseLeverage flag', code: 524 }, 45 | INVALID_ADD_STOP_ORDER: { res: 'invalid addStopOrder flag', code: 525 }, 46 | INVALID_STOP_ORDER_PERCENT: { res: 'invalid stopOrderPercent percent', code: 526 } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /lib/server.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('debug')('bfx:hf:data-server') 4 | const _isFunction = require('lodash/isFunction') 5 | const { nonce } = require('bfx-api-node-util') 6 | const { Server } = require('ws') 7 | const sqliteDb = require('./db/sqlite_db') 8 | 9 | const { version } = require('../package.json') 10 | const getCandles = require('./cmds/get_candles') 11 | const getTrades = require('./cmds/get_trades') 12 | const getBTs = require('./cmds/get_bts') 13 | const execBT = require('./cmds/exec_bt') 14 | const execStr = require('./cmds/exec_strategy') 15 | const submitBT = require('./cmds/submit_bt') 16 | const stopBT = require('./cmds/stop_bt') 17 | const sendError = require('./wss/send_error') 18 | const send = require('./wss/send') 19 | const ERRORS = require('./errors') 20 | const getBtHistory = require('./cmds/get_bt_history') 21 | const setBtFavorite = require('./cmds/set_bt_favorite') 22 | const deleteBt = require('./cmds/delete_bt') 23 | const deleteAllBts = require('./cmds/delete_all_bts') 24 | 25 | const COMMANDS = { 26 | 'exec.str': execStr, 27 | 'exec.bt': execBT, 28 | 'get.bts': getBTs, 29 | 'get.candles': getCandles, 30 | 'get.trades': getTrades, 31 | 'submit.bt': submitBT, 32 | 'stop.bt': stopBT, 33 | 'get.bt.history.list': getBtHistory, 34 | 'set.bt.history.favorite': setBtFavorite, 35 | 'delete.bt.history': deleteBt, 36 | 'delete.bt.history.all': deleteAllBts 37 | } 38 | 39 | class DataServer { 40 | /** 41 | * @param {Object} args 42 | * @param {Object} db - bfx-hf-models DB instance 43 | * @param {number} port - websocket server port 44 | * @param {string} sqlitePath - dir to save sqlite db 45 | */ 46 | constructor ({ 47 | db, 48 | port, 49 | sqlitePath 50 | } = {}) { 51 | this.db = db 52 | this.activeSyncs = [] // sync ranges 53 | this.syncExpectants = [] // promises awaiting sync end, keyed by sync range 54 | this.wssClients = {} 55 | this.activeBacktests = new Map() 56 | 57 | this.port = port 58 | this.wss = null 59 | this.sqlitePath = sqlitePath 60 | } 61 | 62 | /** 63 | * Spawns the WebSocket API server; throws an error if it is already open 64 | */ 65 | async open () { 66 | // connect to sqlite db before starting the ws server 67 | await this.connectSqlite() 68 | 69 | if (this.wss) { 70 | throw new Error('already open') 71 | } 72 | 73 | this.wss = new Server({ 74 | clientTracking: true, 75 | port: this.port 76 | }) 77 | 78 | this.wss.on('connection', this.onWSConnected.bind(this)) 79 | 80 | debug('websocket API open on port %d', this.port) 81 | } 82 | 83 | /** 84 | * Connects to sqlite db; throws an error if connection fails 85 | */ 86 | async connectSqlite () { 87 | if (!this.sqlitePath) throw new Error('sqlitePath is missing') 88 | 89 | try { 90 | // connect to sqlite db 91 | await sqliteDb.connectDb(this.sqlitePath) 92 | } catch (err) { 93 | debug('error connecting to sqlite', err) 94 | throw new Error(err.message) 95 | } 96 | } 97 | 98 | /** 99 | * Closes the WebSocket API server; throws an error if it is not open 100 | */ 101 | close () { 102 | if (!this.wss) { 103 | throw new Error('already closed') 104 | } 105 | 106 | this.wss.close() 107 | this.wss = null 108 | } 109 | 110 | /** 111 | * @private 112 | */ 113 | static getSyncKeyForRange ({ exchange, symbol, tf, start, end } = {}) { 114 | return `${exchange}-${symbol}-${tf}-${start}-${end}` 115 | } 116 | 117 | /** 118 | * Returns an array of active sync ranges 119 | * 120 | * @return {Object[]} ranges 121 | */ 122 | getRunningSyncRanges () { 123 | return Object.keys(this.activeSyncs).map(k => { 124 | const [exchange, symbol, tf, start, end] = k.split('-') 125 | 126 | return { 127 | exchange, 128 | symbol, 129 | tf, 130 | start: +start, 131 | end: +end 132 | } 133 | }) 134 | } 135 | 136 | /** 137 | * @private 138 | */ 139 | futureSyncFor ({ exchange, symbol, tf, start }) { 140 | const runningSyncs = this.getRunningSyncRanges().filter(s => ( 141 | s.exchange === exchange && s.symbol === symbol && s.tf === tf 142 | )) 143 | 144 | if (runningSyncs.length === 0) { 145 | return false 146 | } 147 | 148 | return runningSyncs.find((sync) => { 149 | return sync.start > start 150 | }) 151 | } 152 | 153 | /** 154 | * Returns a promise that resolves when a sync covering the specified range 155 | * finishes. If no such sync is active, this is a no-op. 156 | * 157 | * @param {Object} range 158 | * @param {number} range.start 159 | * @param {number} range.end 160 | * @param {string} range.exchange 161 | * @param {string} range.symbol 162 | * @param {string} range.tf 163 | * @return {Promise} p - resolves on sync completion 164 | */ 165 | expectSync ({ exchange, symbol, tf, start, end }) { 166 | const key = DataServer.getSyncKeyForRange({ 167 | exchange, symbol, tf, start, end 168 | }) 169 | 170 | if (!this.activeSyncs[key]) { 171 | const msg = `error: tried to expect non-existent sync (${key})` 172 | debug(msg) 173 | return Promise.reject(msg) 174 | } 175 | 176 | return new Promise((resolve) => { 177 | this.syncExpectants[key].push(resolve) 178 | }) 179 | } 180 | 181 | /** 182 | * Returns a sync range that takes into account active syncs, to prevent 183 | * overlapping sync tasks. 184 | * 185 | * @param {Object} range 186 | * @param {string} range.exchange 187 | * @param {string} range.symbol 188 | * @param {string} range.tf 189 | * @param {number} range.start 190 | * @param {number} range.end 191 | * @return {Object} optimalRange - null if sync not required at all 192 | */ 193 | optimizeSyncRange ({ exchange, symbol, tf, start, end }) { 194 | const runningSyncs = this.getRunningSyncRanges().filter(s => ( 195 | s.exchange === exchange && s.tf === tf && s.symbol === symbol 196 | )) 197 | 198 | if (runningSyncs.length === 0) { 199 | return { exchange, symbol, tf, start, end } 200 | } 201 | 202 | runningSyncs.sort((a, b) => b.start - a.start) 203 | 204 | let sync 205 | let optimalStart = start 206 | let optimalEnd = end 207 | 208 | for (let i = 0; i < runningSyncs.length; i += 1) { 209 | sync = runningSyncs[i] 210 | 211 | if (optimalStart >= sync.start && optimalEnd <= sync.end) { // engulfed 212 | return null 213 | } 214 | 215 | if (optimalStart >= sync.start && optimalStart <= sync.end) { // start already covered 216 | optimalStart = sync.end 217 | } else if (optimalEnd >= sync.start && optimalEnd <= sync.end) { // end already covered 218 | optimalEnd = sync.start 219 | } 220 | } 221 | 222 | if (optimalStart !== start || optimalEnd !== end) { 223 | debug( 224 | 'optimised sync (%d -> %d) - (%d -> %d)', 225 | start, optimalStart, end, optimalEnd 226 | ) 227 | } 228 | 229 | return { 230 | exchange, 231 | symbol, 232 | tf, 233 | start: optimalStart, 234 | end: optimalEnd 235 | } 236 | } 237 | 238 | /** 239 | * Notify the server that a sync is running for the specified range/market 240 | * 241 | * @param {Object} args 242 | * @param {string} args.exchange 243 | * @param {string} args.symbol 244 | * @param {string} args.tf 245 | * @param {number} args.start 246 | * @param {number} args.end 247 | */ 248 | notifySyncStart ({ exchange, symbol, tf, start, end }) { 249 | const key = DataServer.getSyncKeyForRange({ 250 | exchange, symbol, tf, start, end 251 | }) 252 | 253 | if (this.activeSyncs[key]) { 254 | debug('error: notified start of sync that is already running (%s)', key) 255 | return 256 | } 257 | 258 | debug('sync started: %s', key) 259 | 260 | this.activeSyncs[key] = true 261 | this.syncExpectants[key] = [] 262 | } 263 | 264 | /** 265 | * Notify the server that a sync has finished for the specified range/market 266 | * 267 | * @param {Object} args 268 | * @param {string} args.exchange 269 | * @param {string} args.symbol 270 | * @param {string} args.tf 271 | * @param {number} args.start 272 | * @param {number} args.end 273 | */ 274 | notifySyncEnd ({ exchange, symbol, tf, start, end }) { 275 | const key = DataServer.getSyncKeyForRange({ 276 | exchange, symbol, tf, start, end 277 | }) 278 | 279 | if (!this.activeSyncs[key]) { 280 | debug('error: notified end of unknown sync (%s)', key) 281 | return 282 | } 283 | 284 | debug('sync ended: %s', key) 285 | 286 | this.syncExpectants[key].forEach(resolve => resolve()) 287 | 288 | delete this.syncExpectants[key] 289 | delete this.activeSyncs[key] 290 | } 291 | 292 | /** 293 | * @private 294 | */ 295 | onWSConnected (ws) { 296 | debug('ws client connected') 297 | 298 | const clientID = nonce() 299 | 300 | this.wssClients[clientID] = ws 301 | 302 | ws.on('message', this.onWSMessage.bind(this, clientID)) 303 | ws.on('close', this.onWSDisconnected.bind(this, clientID)) 304 | 305 | send(ws, ['connected', version]) 306 | } 307 | 308 | /** 309 | * @private 310 | */ 311 | onWSDisconnected (clientID) { 312 | debug('ws client %s disconnected', clientID) 313 | 314 | delete this.wssClients[clientID] 315 | } 316 | 317 | /** 318 | * @private 319 | */ 320 | onWSMessage (clientID, msgJSON = '') { 321 | const ws = this.wssClients[clientID] 322 | 323 | let msg 324 | 325 | try { 326 | msg = JSON.parse(msgJSON) 327 | } catch (e) { 328 | debug('error reading ws client msg: %s', msgJSON) 329 | } 330 | 331 | if (!Array.isArray(msg)) { 332 | return sendError(ws, ERRORS.GENERIC.MSG_NOT_ARRAY) 333 | } 334 | 335 | const [cmd] = msg 336 | const handler = COMMANDS[cmd] 337 | 338 | if (!_isFunction(handler)) { 339 | return sendError(ws, ERRORS.GENERIC.UNKNOWN_COMMAND) 340 | } 341 | 342 | handler(this, ws, msg, clientID).catch((err) => { 343 | debug('error processing message: %s', err.stack) 344 | return sendError(ws, ERRORS.GENERIC.INTERNAL) 345 | }) 346 | } 347 | } 348 | 349 | module.exports = DataServer 350 | -------------------------------------------------------------------------------- /lib/util/delay.js: -------------------------------------------------------------------------------- 1 | module.exports = (ms) => { 2 | return new Promise(resolve => setTimeout(resolve, ms)) 3 | } 4 | -------------------------------------------------------------------------------- /lib/util/get_derivatives_config.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { RESTv2 } = require('bfx-api-node-rest') 4 | const _ = require('lodash') 5 | const BN = require('bignumber.js') 6 | const TRADING_PAIR_PREFIX = 't' 7 | 8 | const getFuturesPairs = _.memoize(async () => { 9 | const rest = new RESTv2({ transform: true }) 10 | 11 | const [pairsList, pairsInfo] = await rest.conf({ 12 | keys: [ 13 | 'pub:list:pair:futures', 14 | 'pub:info:pair:futures' 15 | ] 16 | }) 17 | 18 | const futuresPairs = {} 19 | pairsInfo.forEach(([pair, [,,,,,,,, initialMargin]]) => { 20 | if (pairsList.includes(pair)) { 21 | const pairSymbol = TRADING_PAIR_PREFIX + pair 22 | futuresPairs[pairSymbol] = { maxLeverage: new BN(1).dividedBy(initialMargin).toNumber() } 23 | } 24 | }) 25 | 26 | // ex: {tBTCF0:USTF0': { maxLeverage: 100 }, ...} 27 | return futuresPairs 28 | }) 29 | 30 | module.exports = async (symbol) => { 31 | const futuresPairs = await getFuturesPairs() 32 | return futuresPairs[symbol] 33 | } 34 | -------------------------------------------------------------------------------- /lib/util/parse_exec_msg.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = (msg = []) => { 4 | const [ 5 | exchange, start, end, symbol, tf, includeCandles, includeTrades, sync = true, meta 6 | ] = msg[1] || [] 7 | 8 | return { 9 | exchange, start, end, symbol, tf, includeCandles, includeTrades, sync, meta 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /lib/util/validate_bt_args.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const _isString = require('lodash/isString') 4 | const _isFinite = require('lodash/isFinite') 5 | const _isBoolean = require('lodash/isBoolean') 6 | const { candleWidth } = require('bfx-hf-util') 7 | 8 | const ERRORS = require('../errors') 9 | 10 | /** 11 | * @param {*[]} msg - bt exec ws request 12 | * @return {{res: string, code: number}|null} error 13 | */ 14 | module.exports = (msg = []) => { 15 | const [ 16 | , , start, end, symbol, tf, includeCandles, includeTrades, candleSeed, sync, margin, , , constraints = {}, leverageSettings = {}, stopOrderSettings = {} 17 | ] = msg[1] || [] 18 | const { capitalAllocation, maxDrawdownPerc, stopLossPerc } = constraints 19 | const { useMaxLeverage, increaseLeverage, leverage } = leverageSettings 20 | const { addStopOrder, stopOrderPercent } = stopOrderSettings 21 | 22 | if (!_isFinite(start)) { 23 | return ERRORS.BACKTEST.INVALID_START 24 | } else if (!_isFinite(end)) { 25 | return ERRORS.BACKTEST.INVALID_END 26 | } else if (start > end) { 27 | return ERRORS.BACKTEST.START_BEFORE_END 28 | } else if (!_isFinite(candleWidth(tf))) { 29 | return ERRORS.BACKTEST.INVALID_TF 30 | } else if (!_isString(symbol)) { 31 | return ERRORS.BACKTEST.SYMBOL_NOT_STRING 32 | } else if (!_isBoolean(includeCandles)) { 33 | return ERRORS.BACKTEST.INVALID_INCLUDE_CANDLES 34 | } else if (!_isBoolean(includeTrades)) { 35 | return ERRORS.BACKTEST.INVALID_INCLUDE_TRADES 36 | } else if (!_isBoolean(sync)) { 37 | return ERRORS.BACKTEST.INVALID_SYNC 38 | } else if (!includeCandles && !includeTrades) { 39 | return ERRORS.BACKTEST.REQ_EMPTY 40 | } else if (!_isFinite(capitalAllocation)) { 41 | return ERRORS.BACKTEST.INVALID_ALLOCATION 42 | } else if (maxDrawdownPerc && !_isFinite(maxDrawdownPerc)) { 43 | return ERRORS.BACKTEST.INVALID_MAX_DRAWDOWN 44 | } else if (stopLossPerc && !_isFinite(stopLossPerc)) { 45 | return ERRORS.BACKTEST.INVALID_PERC_STOP_LOSS 46 | } else if (!_isFinite(candleSeed)) { 47 | return ERRORS.BACKTEST.INVALID_CANDLE_SEED 48 | } else if (!_isBoolean(margin)) { 49 | return ERRORS.BACKTEST.INVALID_MARGIN 50 | } else if (margin && !_isBoolean(useMaxLeverage)) { 51 | return ERRORS.BACKTEST.INVALID_USE_MAX_LEVERAGE 52 | } else if (useMaxLeverage === false && (!_isFinite(leverage) || !leverage)) { 53 | return ERRORS.BACKTEST.INVALID_LEVERAGE 54 | } else if (useMaxLeverage === false && !_isBoolean(increaseLeverage)) { 55 | return ERRORS.BACKTEST.INVALID_INCREASE_LEVERAGE 56 | } else if (margin && !_isBoolean(addStopOrder)) { 57 | return ERRORS.BACKTEST.INVALID_ADD_STOP_ORDER 58 | } else if (addStopOrder && (!_isFinite(stopOrderPercent) || !stopOrderPercent)) { 59 | return ERRORS.BACKTEST.INVALID_STOP_ORDER_PERCENT 60 | } 61 | 62 | return null 63 | } 64 | -------------------------------------------------------------------------------- /lib/wss/send.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('debug')('bfx:hf:data-server:wss:send') 4 | const WS = require('ws') 5 | 6 | module.exports = (ws, msg = []) => { 7 | if (ws.readyState === WS.OPEN) { 8 | ws.send(JSON.stringify(msg)) 9 | } else { 10 | debug('ws not open (%d), refusing: %j', ws.readyState, msg) 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /lib/wss/send_error.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const debug = require('debug')('bfx:hf:data-server:wss:send-error') 4 | const _isFinite = require('lodash/isFinite') 5 | const _isString = require('lodash/isString') 6 | const _isEmpty = require('lodash/isEmpty') 7 | 8 | const send = require('./send') 9 | 10 | // Errors are thrown here since the err object must be well-formed to be sent 11 | // out as a ws message. Logging the error is not sufficient, since it would 12 | // result in a malformed ws error packet (and cause problems in the client) 13 | module.exports = (ws, err) => { 14 | if (!_isFinite(err.code)) { 15 | throw new Error(`invalid error code: ${err.code}`) 16 | } else if (!_isString(err.res)) { 17 | throw new Error(`invalid error response: ${err.res}`) 18 | } else if (_isEmpty(err.res)) { 19 | throw new Error(`empty error response for code: ${err.code}`) 20 | } 21 | 22 | debug('%s (%d)', err.res, err.code) 23 | send(ws, ['error', `${err.res} (code: ${err.code})`]) 24 | } 25 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bfx-hf-data-server", 3 | "version": "5.0.0", 4 | "description": "HF data server module", 5 | "main": "index.js", 6 | "directories": { 7 | "lib": "lib" 8 | }, 9 | "author": "Bitfinex", 10 | "contributors": [ 11 | "Cris Mihalache (https://www.bitfinex.com)", 12 | "Paolo Ardoino (https://www.bitfinex.com)", 13 | "Jacob Plaster (https://www.bitfinex.com)", 14 | "Anton Nazarenko " 15 | ], 16 | "license": "Apache-2.0", 17 | "repository": { 18 | "type": "git", 19 | "url": "https://github.com/bitfinexcom/bfx-hf-data-server.git" 20 | }, 21 | "bugs": { 22 | "url": "https://github.com/bitfinexcom/bfx-hf-data-server/issues" 23 | }, 24 | "scripts": { 25 | "lint": "standard", 26 | "test": "npm run lint && npm run unit", 27 | "unit": "NODE_ENV=test mocha -R spec -b --recursive test/unit", 28 | "start-lowdb": "node examples/server-lowdb.js", 29 | "server_docs": "node_modules/jsdoc-to-markdown/bin/cli.js lib/server.js > docs/server.md", 30 | "docs": "npm run server_docs" 31 | }, 32 | "keywords": [ 33 | "honey framework", 34 | "bitfinex", 35 | "bitcoin", 36 | "BTC" 37 | ], 38 | "dependencies": { 39 | "bfx-api-node-rest": "5.5.0", 40 | "bfx-api-node-util": "^1.0.9", 41 | "bfx-facs-db-sqlite": "git+https://github.com/bitfinexcom/bfx-facs-db-sqlite.git", 42 | "bfx-hf-backtest": "git+https://github.com/bitfinexcom/bfx-hf-backtest#v3.0.0", 43 | "bfx-hf-indicators": "git+https://github.com/bitfinexcom/bfx-hf-indicators.git#v2.2.0", 44 | "bfx-hf-models": "git+https://github.com/bitfinexcom/bfx-hf-models.git#v4.0.0", 45 | "bfx-hf-strategy": "git+https://github.com/bitfinexcom/bfx-hf-strategy#v3.0.0", 46 | "bfx-hf-strategy-perf": "git+https://github.com/bitfinexcom/bfx-hf-strategy-perf#v3.1.0", 47 | "bfx-hf-util": "git+https://github.com/bitfinexcom/bfx-hf-util#v1.0.12", 48 | "debug": "^4.3.1", 49 | "heap": "^0.2.7", 50 | "lodash": "^4.17.10" 51 | }, 52 | "devDependencies": { 53 | "bfx-hf-ext-plugin-bitfinex": "git+https://github.com/bitfinexcom/bfx-hf-ext-plugin-bitfinex#v1.0.13", 54 | "bfx-hf-models-adapter-lowdb": "git+https://github.com/bitfinexcom/bfx-hf-models-adapter-lowdb.git#v1.0.5", 55 | "chai": "^4.3.6", 56 | "dotenv": "^6.2.0", 57 | "jsdoc-to-markdown": "^5.0.1", 58 | "mocha": "^6.2.0", 59 | "sinon": "^14.0.0", 60 | "standard": "^14.2.0", 61 | "ws": "^8.2.1", 62 | "proxyquire": "^2.1.3" 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /test/unit/bt/request_semaphore.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-unused-expressions */ 2 | /* eslint-env mocha */ 3 | 'use strict' 4 | 5 | const { stub, assert } = require('sinon') 6 | const { expect } = require('chai') 7 | 8 | const RequestSemaphore = require('../../../lib/bt/request_semaphore') 9 | 10 | describe('Request Semaphore', () => { 11 | const maxRequests = 5 12 | const maxTries = 1 13 | const interval = 1000 14 | 15 | const semaphore = new RequestSemaphore({ maxRequests, maxTries, interval }) 16 | 17 | afterEach(() => { 18 | semaphore.reset() 19 | }) 20 | 21 | it('enqueue', async () => { 22 | const req = stub().resolves(1) 23 | 24 | const response = await semaphore.add(req) 25 | 26 | assert.calledOnce(req) 27 | expect(response).to.eq(1) 28 | }) 29 | 30 | it('delay queue when full', async () => { 31 | const req = stub().resolves() 32 | const n = 2 * maxRequests 33 | const pending = [] 34 | 35 | for (let i = 0; i < n; i++) { 36 | pending.push(semaphore.add(req)) 37 | } 38 | 39 | await Promise.all(pending) 40 | expect(req.callCount).to.eq(n) 41 | }) 42 | 43 | it('retry', async () => { 44 | const req = stub() 45 | req.onCall(0).rejects() 46 | req.onCall(1).resolves(1) 47 | 48 | const response = await semaphore.add(req) 49 | 50 | assert.calledTwice(req) 51 | expect(response).to.eq(1) 52 | }) 53 | 54 | it('max-retries', async () => { 55 | const err = new Error() 56 | const req = stub().rejects(err) 57 | 58 | try { 59 | await semaphore.add(req) 60 | assert.fail() 61 | } catch (e) { 62 | expect(e).to.be.eq(err) 63 | } 64 | }) 65 | 66 | it('ratelimit', async () => { 67 | const err = new Error('429 - ["error",11010,"ratelimit: error"]') 68 | const req = stub() 69 | req.onCall(0).rejects(err) 70 | req.onCall(1).resolves() 71 | req.onCall(2).resolves() 72 | 73 | await Promise.all([ 74 | semaphore.add(req), // activates rate limit 75 | semaphore.add(req) // is suspended 76 | ]) 77 | 78 | expect(req.callCount).to.be.eq(3) 79 | }) 80 | }) 81 | -------------------------------------------------------------------------------- /test/unit/db/bt_dao.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | /* eslint-disable no-unused-expressions */ 3 | 'use strict' 4 | 5 | const proxyquire = require('proxyquire') 6 | const sinon = require('sinon') 7 | const { expect } = require('chai') 8 | let btDao 9 | const testBt = { 10 | exchange: 'bitfinex', 11 | strategyId: '123456', 12 | start: 1691491641045, 13 | end: 1691577141045, 14 | symbol: 'TESTBTC:TESTUSD', 15 | timeframe: '5m', 16 | includeCandles: true, 17 | includeTrades: false, 18 | candleSeed: 10, 19 | sync: true, 20 | executionId: 'd80adf68-f672-4f78-a111-5d72585a2c43', 21 | capitalAllocation: 13, 22 | stopLossPerc: 56, 23 | maxDrawdownPerc: 76, 24 | isFavorite: 0, 25 | timestamp: Date.now() 26 | } 27 | 28 | describe('Backtest DAO', () => { 29 | const executeQueryStub = sinon.stub() 30 | const createDataStub = sinon.stub() 31 | const getDataStub = sinon.stub() 32 | const queryDataStub = sinon.stub() 33 | 34 | before(() => { 35 | getDataStub.returns(testBt) 36 | queryDataStub.returns([testBt]) 37 | 38 | btDao = proxyquire('../../../lib/db/bt_dao', 39 | { 40 | './sqlite_db': { 41 | executeQuery: executeQueryStub, 42 | createData: createDataStub, 43 | getData: getDataStub, 44 | queryData: queryDataStub 45 | } 46 | }) 47 | }) 48 | 49 | describe('#saveBt', () => { 50 | it('save backtest', async () => { 51 | const args = [ 52 | 'bitfinex', '123456', 1691491641045, 1691577141045, 'TESTBTC:TESTUSD', 53 | '5m', true, false, 10, true, false, {}, 'd80adf68-f672-4f78-a111-5d72585a2c43', 54 | { capitalAllocation: 13, stopLossPerc: 56, maxDrawdownPerc: 76 }, 55 | { useMaxLeverage: false, increaseLeverage: false, leverage: 0 }, 56 | { addStopOrder: false, stopOrderPercent: 0 } 57 | ] 58 | const savedBt = await btDao.saveBt(args, {}) 59 | expect(savedBt).to.be.an('object') 60 | expect(savedBt.strategyId).to.be.equal('123456') 61 | expect(createDataStub.calledOnce).to.be.true 62 | }) 63 | }) 64 | 65 | describe('#getBtHistory', () => { 66 | it('get backtest history for a strategy', async () => { 67 | const strategyId = '123456' 68 | const data = await btDao.getBtHistory(strategyId) 69 | expect(data).to.be.an('array') 70 | expect(data.length).to.be.greaterThan(0) 71 | expect(queryDataStub.calledOnce).to.be.true 72 | }) 73 | }) 74 | 75 | describe('#setBtFavorite', () => { 76 | it('sets backtest as favorite', async () => { 77 | const executionId = testBt.executionId 78 | const isFavorite = 1 79 | await btDao.setBtFavorite(executionId, isFavorite) 80 | expect(executeQueryStub.calledOnce).to.be.true 81 | }) 82 | }) 83 | }) 84 | -------------------------------------------------------------------------------- /test/unit/db/sqlite_db.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | /* eslint-disable no-unused-expressions */ 3 | 'use strict' 4 | 5 | const { expect } = require('chai') 6 | const os = require('os') 7 | const fs = require('fs') 8 | const sqliteDb = require('../../../lib/db/sqlite_db') 9 | 10 | const testSqlitePath = `${os.homedir()}/.bitfinexhoney/testsqlite` 11 | const testBtValues = ['bitfinex', '123456', 1691491641045, 1691577141045, 'TESTBTC:TESTUSDT', '5m', 12 | true, false, 10, true, false, 'd80adf68-f672-4f78-a111-5d72585a2c43', 13, 56, 76, 13 | false, false, 0, false, 0, 0, Date.now()] 14 | 15 | describe('Sqlite DB', () => { 16 | before(async () => { 17 | if (!fs.existsSync(testSqlitePath)) { 18 | fs.mkdirSync(testSqlitePath) 19 | } 20 | await sqliteDb.connectDb(testSqlitePath) 21 | }) 22 | 23 | after(() => { 24 | if (fs.existsSync(testSqlitePath)) { 25 | fs.rmSync(testSqlitePath, { recursive: true, force: true }) 26 | } 27 | }) 28 | 29 | describe('#createData', () => { 30 | it('adds new entry in bt_history table', async () => { 31 | const createQuery = 'insert into bt_history (exchange, strategyId, start, end, symbol, timeframe, includeCandles, includeTrades, candleSeed, sync, margin, executionId, capitalAllocation, stopLossPerc, maxDrawdownPerc, useMaxLeverage, increaseLeverage, leverage, addStopOrder, stopOrderPercent, isFavorite, timestamp, btResult) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)' 32 | await sqliteDb.createData(createQuery, testBtValues) 33 | const getQuery = 'SELECT * FROM bt_history' 34 | const data = await sqliteDb.queryData(getQuery) 35 | expect(data).to.be.an('array') 36 | expect(data.length).to.be.greaterThan(0) 37 | }) 38 | }) 39 | 40 | describe('#getData', () => { 41 | it('gets data by id from bt_history table', async () => { 42 | const strategyId = '123456' 43 | const getQuery = 'SELECT * FROM bt_history where strategyId=?' 44 | const data = await sqliteDb.getData(getQuery, [strategyId]) 45 | expect(data).to.be.an('object') 46 | expect(data.strategyId).to.be.equal(strategyId) 47 | }) 48 | }) 49 | 50 | describe('#queryData', () => { 51 | it('gets data rows by query from bt_history table', async () => { 52 | const getQuery = 'SELECT * FROM bt_history' 53 | const data = await sqliteDb.queryData(getQuery) 54 | expect(data).to.be.an('array') 55 | expect(data.length).to.be.greaterThan(0) 56 | }) 57 | }) 58 | 59 | describe('#executeQuery', () => { 60 | it('executes query to update data in bt_history table', async () => { 61 | const id = 1 62 | const isFavorite = 1 63 | const query = 'update bt_history set isFavorite=? where id=?' 64 | const values = [isFavorite, id] 65 | await sqliteDb.executeQuery(query, values) 66 | 67 | const getQuery = 'SELECT * FROM bt_history where id=?' 68 | const data = await sqliteDb.getData(getQuery, [id]) 69 | expect(data.id).to.be.equal(id) 70 | expect(data.isFavorite).to.be.equal(isFavorite) 71 | }) 72 | }) 73 | }) 74 | --------------------------------------------------------------------------------