├── .ci-test-results └── jest │ └── PLACEHOLDER ├── .circleci └── config.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── WhyLeaistic.md ├── bin └── leaistic.js ├── commitlint.config.js ├── docker-compose.local.yml ├── docker-compose.yml ├── example.js ├── index.js ├── index.test.js ├── lib ├── errors.js ├── errors.test.js ├── es │ ├── index.js │ ├── index.test.js │ ├── logger.js │ └── logger.test.js ├── hlrollbacks.js ├── hlrollbacks.test.js ├── indices.js ├── indices.test.js ├── logger.js ├── logger.test.js ├── memoryStore.js ├── memoryStore.test.js ├── ops.js ├── ops.test.js ├── rollbacks.js ├── rollbacks.test.js ├── state.js ├── state.test.js ├── validation.js └── validation.test.js ├── package-lock.json ├── package.json ├── release.config.js ├── server ├── failures.js ├── failures.test.js ├── handlers │ ├── index.js │ └── index.test.js ├── index.js ├── index.test.js ├── routes.js ├── routes.test.js ├── validation.js └── validation.test.js └── yarn.lock /.ci-test-results/jest/PLACEHOLDER: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nearform/leaistic/9133c7b7076fa10a0c23bdd822f4ac222aa11911/.ci-test-results/jest/PLACEHOLDER -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Javascript Node CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-javascript/ for more details 4 | # 5 | 6 | defaults: &defaults 7 | docker: 8 | - image: circleci/node:10.1 9 | 10 | version: 2 11 | jobs: 12 | checkout: 13 | <<: *defaults 14 | steps: 15 | - checkout 16 | - persist_to_workspace: 17 | root: . 18 | paths: . 19 | 20 | docker-compose: 21 | <<: *defaults 22 | steps: 23 | - run: 24 | name: Install Docker Compose 25 | command: | 26 | curl -L https://github.com/docker/compose/releases/download/1.21.2/docker-compose-`uname -s`-`uname -m` > ~/docker-compose 27 | chmod +x ~/docker-compose 28 | sudo mv ~/docker-compose /usr/local/bin/docker-compose 29 | - persist_to_workspace: 30 | root: /usr/local/bin 31 | paths: docker-compose 32 | 33 | install: 34 | <<: *defaults 35 | steps: 36 | - attach_workspace: 37 | at: . 38 | - checkout 39 | # Download and cache dependencies 40 | - restore_cache: 41 | keys: 42 | - leaistic-dependencies-{{ checksum "package.json" }} 43 | # fallback to using the latest cache if no exact match is found 44 | - leaistic-dependencies- 45 | 46 | - run: yarn 47 | # Persist the specified paths into the workspace for usage in later jobs 48 | - persist_to_workspace: 49 | root: . 50 | paths: . 51 | 52 | test: 53 | <<: *defaults 54 | steps: 55 | - attach_workspace: 56 | at: . 57 | - setup_remote_docker 58 | - run: 59 | name: Start container and verify it's working 60 | command: | 61 | set -x 62 | docker-compose up -d 63 | # docker-compose will start 2 containers, the one with service will be named `elasticsearch` 64 | # we start another container with curl in the same network as `elasticsearch`, this way we have 65 | # all exposed ports from `elasticsearch` available on `localhost` in this new container 66 | docker run --network container:elasticsearch \ 67 | appropriate/curl --retry 60 --retry-delay 1 --retry-connrefused http://127.0.0.1:9200/_cat/health 68 | - run: 69 | name: Change ES log level 70 | command: | 71 | docker run --network container:elasticsearch \ 72 | appropriate/curl -X PUT --retry 60 --retry-delay 1 --retry-connrefused \ 73 | -H 'Content-Type: application/json; charset=UTF-8' \ 74 | -d '{"transient": {"logger._root": "DEBUG"}}' \ 75 | http://127.0.0.1:9200/_cluster/settings 76 | - run: 77 | name: Tests 78 | command: | 79 | # create a dummy container which will hold a volume with config 80 | docker create -v "$PWD":/usr/src/app --name project alpine:3.4 /bin/true 81 | # copy a config file into this volume 82 | docker cp . project:/usr/src/app 83 | # start an application container using this volume 84 | docker run -t -e CI=true --network container:elasticsearch --volumes-from project -w /usr/src/app "node:10" npm run test:ci 85 | - run: 86 | name: Copy tests results back to project 87 | command: docker cp project:/usr/src/app/.ci-test-results "$PWD/.ci-test-results" 88 | - run: 89 | name: Check ES is still alive on error 90 | command: docker run --network container:elasticsearch appropriate/curl --retry 5 --retry-delay 1 --retry-connrefused http://127.0.0.1:9200/_cat/health 91 | when: on_fail 92 | # Allow to show ES version of events on error 93 | - run: 94 | name: Get ES logs for this failure 95 | command: docker logs elasticsearch 96 | when: on_fail 97 | - store_test_results: 98 | path: .ci-test-results 99 | 100 | lint: 101 | <<: *defaults 102 | steps: 103 | - attach_workspace: 104 | at: . 105 | # Run lint 106 | - run: yarn lint 107 | 108 | release-dry-run: 109 | <<: *defaults 110 | steps: 111 | - attach_workspace: 112 | at: . 113 | - run: 114 | name: release 115 | command: npm run semantic-release --dry-run || true 116 | release: 117 | <<: *defaults 118 | steps: 119 | - attach_workspace: 120 | at: . 121 | - run: 122 | name: release 123 | command: npm run semantic-release 124 | 125 | workflows: 126 | version: 2 127 | all: 128 | jobs: 129 | - docker-compose 130 | - checkout 131 | - install: 132 | requires: 133 | - checkout 134 | - test: 135 | requires: 136 | - docker-compose 137 | - install 138 | - lint: 139 | requires: 140 | - install 141 | - release-dry-run: 142 | requires: 143 | - test 144 | - lint 145 | filters: 146 | branches: 147 | ignore: master 148 | - release: 149 | requires: 150 | - test 151 | - lint 152 | filters: 153 | branches: 154 | only: master 155 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # next.js build output 61 | .next 62 | 63 | # orig files 64 | *.orig 65 | 66 | # test results 67 | .ci-test-results/jest/results.xml 68 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:8-alpine 2 | 3 | # Tell node we are running in prod 4 | ARG NODE_ENV=production 5 | ENV NODE_ENV $NODE_ENV 6 | ENV HOST 0.0.0.0 7 | 8 | # Create app directory 9 | WORKDIR /usr/src/app 10 | 11 | # Install app dependencies 12 | COPY package*.json ./ 13 | RUN yarn install --production=false --silent 14 | 15 | # Bundle app source 16 | COPY . . 17 | 18 | # Expose the port the app listens on 19 | EXPOSE 3000 20 | 21 | # Start the application 22 | CMD [ "npm", "start" ] 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![CircleCI](https://circleci.com/gh/nearform/leaistic.svg?style=svg&circle-token=4b2a232c7e549a0ef8df33ad69929077cb15acb4)](https://circleci.com/gh/nearform/leaistic) 2 | [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) 3 | 4 | [Changelog](https://github.com/nearform/leaistic/releases/) 5 | 6 | # Leaistic 7 | Leaistic is an opinionated ElasticSearch manager micro-service and embeddable library. It allows to manage index creation, its mapping, its settings and update in ElasticSearch with no downtime and high availability. 8 | 9 | ## TLDR; Usage 10 | Install it and use it as a library in your project: 11 | ```console 12 | $> npm i --save leaistic 13 | ``` 14 | 15 | Use it instantly using npx, e.g.: 16 | ``` 17 | $> env ES_URL=http://myhost:9200 npx leaistic | npx pino-colada 18 | ``` 19 | then open [http://localhost:3000/documentation]() in your browser. 20 | 21 | 22 | ... or just clone the repository, then read the rest of the README to have more control over it 😉 23 | 24 | ## Why Leaistic ? 25 | 26 | Brief, [ElasticSearch is smart, but not enough, with your data](./WhyLeaistic.md) 27 | 28 | ## The Leaistic way 29 | 30 | In order to provide high availability and simplicity of usage, it uses a set of simple rules: 31 | 32 | 1. Every `index` in usage should have an `alias` 33 | 2. Every programs should use the previously mentionned `alias` to read and write data 34 | 3. It is highly advised to deploy an `index template` to setup the `settings` and the `mappings` of the indices 35 | 36 | ## How Leaistic creates an index 37 | 38 | In order to follow te above rules, to simplify the developer work, Leaistic has a simple convention for "creating an index": 39 | 40 | 1. When you want to use `myIndex` for an index, Leaistic will first, if you provide it, create an index template `myIndex` matching `myIndex-*`, so any index using this convention will use the latest mapping 41 | 2. Then, Leaistic actually creates an index with the convention `{alias}-{date}`, e.g. `myIndex-1970-01-01t00:00:00.000z` (with the end part being the current ISO-like UTC date), matching the index template 42 | 3. Once the index created and refreshed, it creates an alias `myIndex` pointing to the index previously created, `myIndex-1970-01-01t00:00:00.000z` 43 | 4. Then you can use `myIndex` like if it was an index, and start to work ! 44 | 45 | ## How Leaistic updates an index 46 | 47 | Updating an index is a bit more complicated: ElasticSearch does not manage breaking changes on mappings and settings, but thanks to the aliases and the reindexing API, it provides a good way to make the change. 48 | 49 | 1. For updating `myIndex`, Leaistic will first check the alias `myIndex` is existing, and what is the index it points to 50 | 2. Then, it will update the index template if one is provided (it is likely) 51 | 3. After that, it will create a new index using the same convention, `{alias}-{date}` , e.g. `myIndex-1970-01-02t00:00:00.000z` 52 | 4. Then it will ask ElasticSearch to reindex the source index that was pointed by `myIndex` alias and await for the reindexation to complete 53 | 5. After some checks, it will then switch the alias `myIndex` to the new index `myIndex-1970-01-02t00:00:00.000z` when it will be ready 54 | 6. It will finally both check everything is alright and delete the old index that is no more useful 55 | 56 | ## How Leaistic deletes an index 57 | 58 | Deleting an index is pretty simple: 59 | 60 | 1. Leaistic first finds out what is the first index pointed by the alias 61 | 2. Then, it will delete both the alias and the index in parallel 62 | 63 | ## And if something goes wrong ? 64 | 65 | Given ElasticSearch does not have a transaction system, the best we can do is trying to manage rollbacks as well as possible when things goes wrong 66 | 67 | For now, during rollbacks, the last `index template` will be deleted if we deployed a new one in the query we would need to back it up somewhere to be able to rollback to the original one. 68 | 69 | Every created resource will be deleted, and alias switched back to their original indices 70 | 71 | # Run as a microservice 72 | 73 | First you need an ElasticSearch server accessible at [http://127.0.0.1:9200](http://127.0.0.1:9200) if you don't want any configuration. We provide a `docker-compose` file to allow spawning a cluster and a Cerebro interface easily. 74 | 75 | To spawn an ElasticSearch cluster, and Cerebro, run: 76 | ``` console 77 | $> npm run es:local & 78 | ``` 79 | 80 | Start the server (using nodemon to monitor the process, and with human oriented logs) 81 | ```console 82 | $> npm start 83 | ``` 84 | 85 | Then go to: 86 | - [http://localhost:3000/documentation](http://localhost:3000/documentation) to use the Swagger interface 87 | - [http://localhost:9000](http://localhost:9000) with `http://elasticsearch:9200` as a connection Url to use the Cerebro interface 88 | 89 | **Note**: *you'll find more commands you can use to run the service, using:* 90 | ```console 91 | $> npm run 92 | ``` 93 | 94 | # Usage as a library 95 | 96 | You can use `Leaistic` as a library, allowing you, notably, to create scripts, for migrations, etc. 97 | 98 | You can have a look at the [full fledge example](./examples.js) in order to see how you can use it in a script. 99 | 100 | See more explanations and context about the API below 101 | 102 | ## Index and its Alias creation, update and deletion 103 | 104 | ### Creation 105 | ```javascript 106 | const {create} = require('leaistic') 107 | 108 | const indexTemplate = { 109 | index_patterns: ['myindex-*'], 110 | settings: { 111 | number_of_shards: 1 112 | } 113 | } 114 | 115 | // create an index, optionally with an indexTemplate that should match it: 116 | await create('an-index', indexTemplate) 117 | ``` 118 | 119 | ### Update 120 | 121 | #### With automatic reindexation ( e.g. for index template change, if compatible ) 122 | 123 | ```javascript 124 | const {update} = require('leaistic') 125 | 126 | const indexTemplate = { 127 | index_patterns: ['myindex-*'], 128 | settings: { 129 | number_of_shards: 2 130 | } 131 | } 132 | 133 | // create an index, optionally with an indexTemplate that should match it: 134 | await update('an-index', indexTemplate) 135 | ``` 136 | 137 | #### With manual reindexation ( e.g. for creating a whole new version of the data ) 138 | 139 | ```javascript 140 | const {update} = require('leaistic') 141 | 142 | const indexTemplate = { 143 | index_patterns: ['myindex-*'], 144 | settings: { 145 | number_of_shards: 2 146 | } 147 | } 148 | 149 | const data = [ 150 | { hello: 'world'}, 151 | { hello: 'foo'}, 152 | { hello: 'bar'}, 153 | { hello: 'baz'} 154 | ] 155 | 156 | const reindexer = async (indexName, client) => client.bulk({ 157 | refresh: true, 158 | body: data.reduce((acc, current) => { 159 | return acc.concat([ 160 | { 161 | index: { 162 | _index: indexName, 163 | _id: current.hello, 164 | }, 165 | }, 166 | current 167 | ]) 168 | }, []) 169 | }) 170 | } 171 | 172 | // create an index, optionally with an indexTemplate that should match it: 173 | await update('an-index', indexTemplate, reindexer) 174 | ``` 175 | 176 | 177 | ### Deletion 178 | 179 | ```javascript 180 | const {delete: del} = require('leaistic') 181 | 182 | await del('an-index') 183 | ``` 184 | 185 | ### Build a new Index name for a given alias (useful to manage some updates on your own) 186 | ```javascript 187 | const { newIndexName } = require('leaistic') 188 | 189 | newIndexName('an-index') 190 | // 'an-index-1234-12-12t12:34:56.789z' 191 | ``` 192 | 193 | 194 | ## ElasticSearch Connection 195 | 196 | By default, Leaistic will connect to `http://127.0.0.1:9200` or the value provided by `ES_URL` environment variable. 197 | 198 | You can provide your own `url` to connect to: 199 | 200 | ```javascript 201 | const {connect} = require('leaistic') 202 | 203 | connect({url: 'http://myhost:9200'}) 204 | // ... use Leaistic with this url 205 | ``` 206 | 207 | or you can define your own client, providing your own options, including the logger, instead of the default `pino` based one reserved for elasticsearch logging 208 | 209 | For example, using ES default logger is simple to setup: 210 | ```javascript 211 | const {connect} = require('leaistic') 212 | const elasticsearch = require('elasticsearch') 213 | 214 | const client = new elasticsearch.Client({ 215 | host: 'http://myhost:9200', 216 | log: 'trace' // note that using the default ES logger is not advised for production 217 | }) 218 | 219 | connect({client}) 220 | ``` 221 | 222 | `connect` will also always return the ElasticSearch client currently used 223 | 224 | ```javascript 225 | const {connect} = require('leaistic') 226 | 227 | const es = connect() 228 | await es.bulk({ 229 | body: [ 230 | { index: { _index: 'myindex', _type: 'mytype', _id: 1 } }, { title: 'foo' }, 231 | { update: { _index: 'myindex', _type: 'mytype', _id: 2 } }, { doc: { title: 'foo' } }, 232 | { delete: { _index: 'myindex', _type: 'mytype', _id: 3 } }, 233 | ] 234 | }) 235 | ``` 236 | 237 | ## Logger 238 | 239 | Leaistic is using [pino](https://getpino.io) loggers for providing fast and useful logs, however you may want to overrided them to use your own. There is one for the http service when used using `start`, one for the main code, and one dedicated to exchanges with ElasticSearch. 240 | 241 | ### Change main Logger 242 | 243 | You can override it using a [`pino`-compatible]() logger syntax (just the log levels functions are needed). 244 | 245 | For example, this is enough: 246 | 247 | ```javascript 248 | const {logger} = require('leaistic') 249 | 250 | // change the main logger, with a pino-like interface ( https://getpino.io/#/docs/API) 251 | const log = { 252 | trace: (...args) => console.log(...args), 253 | debug: (...args) => console.debug(...args), 254 | info: (...args) => console.info(...args), 255 | warn: (...args) => console.warn(...args), 256 | error: (...args) => console.error(...args), 257 | fatal: (...args) => console.error('💀', ...args) 258 | } 259 | 260 | logger(log) 261 | ``` 262 | 263 | ### ElasticSearch logger 264 | 265 | ElasticSearch uses its own logger. 266 | 267 | You can override it by setting the `ElasticSearch` client by yourself, as described [here](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/logging.html): 268 | 269 | ```javascript 270 | const {connect} = require('leaistic') 271 | const elasticsearch = require('elasticsearch') 272 | 273 | // change the ElasticSearch logger ( https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/logging.html ) 274 | const esLog = function () { 275 | this.trace = (...args) => console.log(...args) 276 | this.debug = (...args) => console.debug(...args) 277 | this.info = (...args) => console.info(...args) 278 | this.warn = (...args) => console.warn(...args) 279 | this.error = (...args) => console.error(...args) 280 | this.fatal = (...args) => console.error('💀', ...args) 281 | } 282 | 283 | const client = new elasticsearch.Client({ 284 | host: 'http://127.0.0.1:9200', 285 | log: esLog 286 | }) 287 | 288 | const es = connect({client}) 289 | ``` 290 | 291 | ## Example usage 292 | 293 | See also a more [in depth example](./example.js), but you should get the idea with this : 294 | 295 | ```javascript 296 | const {connect, create, update, delete: del} = require('leaistic') 297 | 298 | // only needed if you want to obtain the ES client ot set up its 299 | const es = connect() 300 | 301 | // create an index and an alias 302 | const {name, index} = await create('myindex') 303 | 304 | // load some data 305 | await es.bulk({ 306 | body: [ 307 | { index: { _index: 'myindex', _type: 'mytype', _id: 1 } }, { title: 'foo', createdAt: Date.now() }, 308 | { index: { _index: 'myindex', _type: 'mytype', _id: 2 } }, { title: 'bar', createdAt: Date.now() }, 309 | { index: { _index: 'myindex', _type: 'mytype', _id: 3 } }, { title: 'baz', createdAt: Date.now() } 310 | ] 311 | }) 312 | 313 | // oh, wait, createdAt was considered as a number by ES, not a date, 314 | // and we actually need an exact match on 'title'? Let's fix that: 315 | const indexTemplate = { 316 | index_patterns: ['myindex-*'], 317 | settings: { 318 | number_of_shards: 1 319 | }, 320 | mappings: { 321 | mytype: { 322 | properties: { 323 | title: { 324 | type: 'keyword' 325 | }, 326 | createdAt: { 327 | type: 'date', 328 | format: 'epoch_millis' 329 | } 330 | } 331 | } 332 | } 333 | } 334 | 335 | // update the settings to add a index template (you could have done it during creation as well) 336 | await update(name, { indexTemplate }) 337 | 338 | // now 'createdAt' will be actually considered like a date 339 | const res = await es.search({ 340 | index: 'myindex', 341 | body: { 342 | "query": { 343 | "bool": { 344 | "must": { 345 | "match": { 346 | "title": "foo" 347 | } 348 | }, 349 | "filter": { 350 | "range": { 351 | "createdAt": { 352 | "gte": "01/01/2012", 353 | "lte": "2019", 354 | "format": "dd/MM/yyyy||yyyy" 355 | } 356 | } 357 | } 358 | } 359 | } 360 | }) 361 | 362 | // let's say this index is now deprecated, delete it 363 | await del(name) 364 | ``` 365 | 366 | Note: if anything goes wrong during one of these steps, the promise will be rejected. When using `async`/`await` like above, it means an exception will be thrown 367 | 368 | # Development 369 | 370 | ## Running the tests 371 | 372 | To run the local ElasticSearch cluster using docker for running the tests : 373 | 374 | ```console 375 | $> npm run es:local & 376 | $> npm test 377 | ``` 378 | *Note*: alternatively, feel free to run `npm run es:local` in an alternative tab, or use `npm run es:local:start -- -d` 379 | 380 | You can also run the tests in watch mode: 381 | ``` 382 | $> npm run test:watch 383 | ``` 384 | -------------------------------------------------------------------------------- /WhyLeaistic.md: -------------------------------------------------------------------------------- 1 | # The risk of ElasticSearch trying to be smart with your data, but failing 2 | 3 | Leaistic was made because ElasticSearch is really cool, yet it has some drawbacks due to its mapping system: 4 | 1. Bad datatype guesses 5 | 2. Cannot keep up with the data structure 6 | 3. Forgotten, deprecated mappings still applied 7 | 4. Exact String mapping not active by default 8 | 9 | ElasticSearch will figure out the `datatype` of a given field on its own, thanks to its defaults, the static `mapping` you declare for a given field, or the [`dynamic mapping`](https://www.elastic.co/guide/en/elasticsearch/reference/6.3/dynamic-field-mapping.html) rules it has, using only the first document you index containing a given field as its only context for deciding what datatype is this field. 10 | 11 | Also ElasticSearch [can't](https://www.elastic.co/guide/en/elasticsearch/reference/6.3/mapping.html#_updating_existing_field_mappings) mutate an existing `datatype` for a given field, and the only solution to do so is to create a new index with the proper mapping, then reindex the original index data into it, and finally use the new one. 12 | 13 | During the life of a project, if you don't have a strategy to avoid `mapping` issues, it will surely break. Let's have a look at a few reasons. 14 | 15 | ## Bad datatype guesses 16 | 17 | Some common bad guesses from ElasticSearch: 18 | 19 | 1. You add a `createdAt` field containing a timestamp as a number, like 1234567890. For you, it represents the date of creation for a document. For Elasticsearch, as you did not define a mapping, it decided that this is a `long`: querying it like a `date` will not work. You must add a specific mapping for ElasticSearch to consider it a proper `date` 20 | 21 | 2. You add an `updatedAt` field with a `string` repesenting a date for a human, but which is not recognized by the [date detection](https://www.elastic.co/guide/en/elasticsearch/reference/6.3/dynamic-field-mapping.html#date-detection) defaults (or your [custom version](https://www.elastic.co/guide/en/elasticsearch/reference/6.3/dynamic-field-mapping.html#_customising_detected_date_formats)). 22 | 23 | 3. You add a field `score`, containing a number, which should be a `float`. No luck, the first document you index has a numeric value `3` and not `3.0` in your indexation JSON. ElasticSearch will then consider it to be a `long`... So every number you store in there will be considered an integer. 24 | 25 | 4. You add a field `rating`, coming back from an API, containing a `string`, which is actually a `float` value (.e.g. `{"rating": "3.0"}`) ElasticSearch will [coerce](https://www.elastic.co/guide/en/elasticsearch/reference/6.3/dynamic-field-mapping.html#numeric-detection) it by default, to the first type it discovers in it, in this case it would be a `float`, but if the `.0` is not there, it could also be considered as a `long` 26 | 27 | ## Cannot keep up with the data structure 28 | 29 | You used to have a field `checkout`, which had a `date` in it, recognized as a `date`. But later in the project, you decide that your `checkout` field should actually contain an object with metadata about checkout, like the `location`, the `date`, etc. 30 | 31 | ## Forgotten, deprecated mappings still applied 32 | 33 | In your index, you used to have a `creation` field containing a `date`, working properly. At some point in the project, you renamed it `createdAt` because it made more sense regarding your convention. After a while, at the root of your main document, you tend to have a lot of creation-related data. You decide, then, that you should have an object containing data related to creation in a sub-object, stored in the key `creation`. Albeit there is no document containing `creation` anymore in your indice that is a date, the mapping has stayed, and you will have a mapping conflict. 34 | 35 | This issue tends to happen in production-like platforms as you are less likely to reset the indices as often locally or in development platforms. 36 | 37 | ## Exact String mapping not active by default 38 | 39 | In your document, you add a field containing an identifier which is some `base64` stuff, like `TGVhaXN0aWM=`: it contains `uppercase`, and `equal signs`. You will want to filter it as an exact match, and it will fail: ElasticSearch will have indexed it by default as `['tgvhaxn0awm']`! 40 | 41 | When storing a `string` in a field, by default, ElasticSearch will (if it does not 'look like' a `float`, a `long`, a `boolean`, a `date`, ...) consider it as a [`text`](https://www.elastic.co/guide/en/elasticsearch/reference/6.3/text.html) to parse with [the standard analyzer](https://www.elastic.co/guide/en/elasticsearch/reference/6.3/analysis-standard-analyzer.html), which allows full text search on this field. If this string is actually something like an identifier you want to filter on, you may or may not be able to do an exact match on it by default, because the analyzer includes the [`lower case token filter`](https://www.elastic.co/guide/en/elasticsearch/reference/6.3/analysis-lowercase-tokenfilter.html) and the [`standard tokenizer`](https://www.elastic.co/guide/en/elasticsearch/reference/6.3/analysis-standard-tokenizer.html) which considers notably [`=`](https://unicode.org/reports/tr29/#ALetter) as a separator. 42 | 43 | There is a good way to handle that in ElasticSearch, which is to add the `keyword` `datatype` instead of the default `text` to your field mapping. Of course, you are likely to discover that afterward. It may even have partially worked for a while, and you may not have noticed. 44 | 45 | # How does Leaistic solve those issues ? 46 | 47 | 1. `Bad datatype guesses`: Probably 80% of all the issues you're likely to encounter. Leaistic helps you to deploy `index templates` for updating your mappings incrementally, whether it is before or after you encounter such an issue. Of course, before is better as it will not break! 48 | 49 | 2. `Cannot keep up with the data structure`: Leaistic right now is not 100% helpful with that case, because it uses ElasticSearch-side reindexation when making an update, which means it keeps the same structure. Yet, fixing [#10](https://github.com/nearform/leaistic/issues/10) and [#7](https://github.com/nearform/leaistic/issues/7) should allow to manage that properly, PRs are welcome 😉 50 | 51 | 3. `Forgotten, deprecated mappings still applied`: If you use Leaistic for all your mapping updates, you should never encounter this issue! 52 | 53 | 4. `Exact string mapping not active by default`: You still have to remember to use the `keyword` datatype explicitly for your identifiers in ElasticSearch, but should you forget to do it, Leaistic allows you to simply update the index template and reindex the data without any downtime. 54 | -------------------------------------------------------------------------------- /bin/leaistic.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const { start } = require('..') 4 | 5 | start() 6 | -------------------------------------------------------------------------------- /commitlint.config.js: -------------------------------------------------------------------------------- 1 | module.exports = {extends: ['@commitlint/config-conventional']} 2 | -------------------------------------------------------------------------------- /docker-compose.local.yml: -------------------------------------------------------------------------------- 1 | version: '2.2' 2 | services: 3 | # Cerebro Elasticsearch client 4 | cerebro: 5 | image: yannart/cerebro:latest 6 | ports: 7 | - "9000:9000" 8 | networks: 9 | - esnet 10 | 11 | elasticsearch: 12 | image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4 13 | container_name: elasticsearch 14 | environment: 15 | - logger.level=DEBUG 16 | - cluster.name=docker-cluster 17 | - bootstrap.memory_lock=true 18 | - "transport.host=elasticsearch" 19 | - "bootstrap.system_call_filter=false" 20 | - "cluster.routing.allocation.disk.threshold_enabled=false" 21 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 22 | - "discovery.zen.ping.unicast.hosts=elasticsearch2" 23 | ulimits: 24 | memlock: 25 | soft: -1 26 | hard: -1 27 | volumes: 28 | - esdata1:/usr/share/elasticsearch/data 29 | ports: 30 | - 9200:9200 31 | networks: 32 | - esnet 33 | elasticsearch2: 34 | image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4 35 | container_name: elasticsearch2 36 | environment: 37 | - logger.level=DEBUG 38 | - cluster.name=docker-cluster 39 | - bootstrap.memory_lock=true 40 | - "transport.host=elasticsearch2" 41 | - "bootstrap.system_call_filter=false" 42 | - "cluster.routing.allocation.disk.threshold_enabled=false" 43 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 44 | - "discovery.zen.ping.unicast.hosts=elasticsearch" 45 | ulimits: 46 | memlock: 47 | soft: -1 48 | hard: -1 49 | volumes: 50 | - esdata2:/usr/share/elasticsearch/data 51 | networks: 52 | - esnet 53 | elasticsearch3: 54 | image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4 55 | container_name: elasticsearch3 56 | environment: 57 | - logger.level=DEBUG 58 | - cluster.name=docker-cluster 59 | - bootstrap.memory_lock=true 60 | - "transport.host=elasticsearch3" 61 | - "bootstrap.system_call_filter=false" 62 | - "cluster.routing.allocation.disk.threshold_enabled=false" 63 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 64 | - "discovery.zen.ping.unicast.hosts=elasticsearch" 65 | ulimits: 66 | memlock: 67 | soft: -1 68 | hard: -1 69 | volumes: 70 | - esdata3:/usr/share/elasticsearch/data 71 | networks: 72 | - esnet 73 | volumes: 74 | esdata1: 75 | driver: local 76 | esdata2: 77 | driver: local 78 | esdata3: 79 | driver: local 80 | 81 | networks: 82 | esnet: 83 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.2' 2 | services: 3 | elasticsearch: 4 | image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4 5 | container_name: elasticsearch 6 | environment: 7 | - logger.level=DEBUG 8 | - cluster.name=docker-cluster 9 | - bootstrap.memory_lock=true 10 | - "transport.host=localhost" 11 | - "bootstrap.system_call_filter=false" 12 | - "cluster.routing.allocation.disk.threshold_enabled=false" 13 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 14 | ulimits: 15 | memlock: 16 | soft: -1 17 | hard: -1 18 | volumes: 19 | - esdata1:/usr/share/elasticsearch/data 20 | ports: 21 | - 9200:9200 22 | networks: 23 | - esnet 24 | volumes: 25 | esdata1: 26 | driver: local 27 | 28 | networks: 29 | esnet: 30 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | const elasticsearch = require('elasticsearch') 2 | const {logger, connect, create, update, delete: del} = require('.') 3 | 4 | // change the main logger, with a pino-like interface ( https://getpino.io/#/docs/API?id=fatal ) 5 | const log = { 6 | trace: (...args) => console.log(...args), 7 | debug: (...args) => console.debug(...args), 8 | info: (...args) => console.info(...args), 9 | warn: (...args) => console.warn(...args), 10 | error: (...args) => console.error(...args), 11 | fatal: (...args) => console.error('💀', ...args) 12 | } 13 | 14 | logger(log) 15 | 16 | // change the ElasticSearch logger ( https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/logging.html ) 17 | const esLog = function () { 18 | this.trace = (...args) => console.log(...args) 19 | this.debug = (...args) => console.debug(...args) 20 | this.info = (...args) => console.info(...args) 21 | this.warning = (...args) => console.warn(...args) 22 | this.error = (...args) => console.error(...args) 23 | this.fatal = (...args) => console.error('💀', ...args) 24 | } 25 | 26 | const client = new elasticsearch.Client({ 27 | host: 'http://127.0.0.1:9200', 28 | log: esLog 29 | }) 30 | 31 | const es = connect({client}) 32 | 33 | const run = async () => { 34 | const {name, index} = await create('myindex') 35 | 36 | console.log({name, index}) 37 | 38 | // load some data 39 | await es.bulk({ 40 | body: [ 41 | { index: { _index: 'myindex', _type: 'mytype', _id: 1 } }, { title: 'foo', createdAt: Date.now() }, 42 | { index: { _index: 'myindex', _type: 'mytype', _id: 2 } }, { title: 'bar', createdAt: Date.now() }, 43 | { index: { _index: 'myindex', _type: 'mytype', _id: 3 } }, { title: 'baz', createdAt: Date.now() } 44 | ] 45 | }) 46 | 47 | // oh, wait, createdAt was considered as a number by ES, not a date, 48 | // and we actually need an exact match on 'title'? Let's fix that: 49 | const indexTemplate = { 50 | index_patterns: ['myindex-*'], 51 | settings: { 52 | number_of_shards: 1 53 | }, 54 | mappings: { 55 | mytype: { 56 | properties: { 57 | title: { 58 | type: 'keyword' 59 | }, 60 | createdAt: { 61 | type: 'date', 62 | format: 'epoch_millis' 63 | } 64 | } 65 | } 66 | } 67 | } 68 | 69 | // update the settings to add a index template (you could have done it during creation as well) 70 | await update(name, { indexTemplate }) 71 | 72 | // now 'createdAt' will be actually considered like a date 73 | const res = await es.search({ 74 | index: 'myindex', 75 | body: { 76 | 'query': { 77 | 'bool': { 78 | 'must': { 79 | 'match': { 80 | 'title': 'foo' 81 | } 82 | }, 83 | 'filter': { 84 | 'range': { 85 | 'createdAt': { 86 | 'gte': '01/01/2012', 87 | 'lte': '2019', 88 | 'format': 'dd/MM/yyyy||yyyy' 89 | } 90 | } 91 | } 92 | } 93 | } 94 | } 95 | }) 96 | 97 | console.log({res}) 98 | 99 | await del(name) 100 | } 101 | 102 | run() 103 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const start = require('./server') 2 | const lib = require('./lib/indices') 3 | const {connect} = require('./lib/es') 4 | const {log} = require('./lib/logger') 5 | const {store} = require('./lib/state') 6 | 7 | // start the micro service 8 | exports.start = start 9 | 10 | // use a high level library 11 | exports.create = (name, {indexTemplate} = {}) => lib.create(name, {body: indexTemplate}) 12 | exports.update = (name, {indexTemplate} = {}, reindexer) => lib.update(name, {body: indexTemplate}, reindexer) 13 | exports.delete = name => lib.delete(name) 14 | 15 | exports.newIndexName = (aliasName) => lib.suffix(aliasName) 16 | 17 | // override ES client or get a reference to it 18 | exports.connect = connect 19 | 20 | // override the logger or use it 21 | exports.logger = log 22 | 23 | // override the store or use it 24 | exports.store = store 25 | -------------------------------------------------------------------------------- /index.test.js: -------------------------------------------------------------------------------- 1 | const { start } = require('.') 2 | const { name } = require('./package.json') 3 | 4 | describe('start', () => { 5 | let server 6 | beforeAll(async () => { server = await start({ host: 'localhost', port: 0 }) }) 7 | 8 | it(`should start ${name} server`, async () => { 9 | expect(server).toBeDefined() 10 | expect(server.info).toBeDefined() 11 | expect(server.info.started).toBeGreaterThan(0) 12 | }) 13 | 14 | afterAll(async () => server.stop()) 15 | }) 16 | -------------------------------------------------------------------------------- /lib/errors.js: -------------------------------------------------------------------------------- 1 | const Boom = require('boom') 2 | 3 | const {esError} = require('./es') 4 | 5 | const esIsDisconnectedMessage = /No Living connections$/ 6 | 7 | const checkIfElasticSearchIsNotAvailable = (error) => { 8 | if (error && esIsDisconnectedMessage.test(error.message)) { 9 | throw Boom.boomify(error, {statusCode: 502}) 10 | } 11 | // do not manage anything else 12 | } 13 | 14 | const checkIfElasticSearchHasAConflict = (error) => { 15 | if (error && error.body && error.body.error && error.body.error.type === 'resource_already_exists_exception') { 16 | // already exists => conflict ! HTTP 409 17 | throw Boom.boomify(error, {statusCode: 409}) 18 | } 19 | // do not manage anything else 20 | } 21 | 22 | exports.rollbackStep = async (error, rollbackFn) => { 23 | try { 24 | return await rollbackFn() 25 | } catch (e) { 26 | if (!error.rollbackErrors) { 27 | error.rollbackErrors = [] 28 | } 29 | error.rollbackErrors.push(e) 30 | } 31 | } 32 | 33 | exports.manageErrors = async (fn, description, rollbackFn) => { 34 | try { 35 | return await fn() 36 | } catch (error) { 37 | if (error instanceof esError) { 38 | error.isElasticSearch = true 39 | } 40 | if (description) { 41 | error.message = `${description}: ${error.message}` 42 | } 43 | if (rollbackFn) { 44 | await exports.rollbackStep(error, () => rollbackFn(error)) 45 | } 46 | checkIfElasticSearchIsNotAvailable(error) 47 | checkIfElasticSearchHasAConflict(error) 48 | throw error 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /lib/errors.test.js: -------------------------------------------------------------------------------- 1 | const delay = require('delay') 2 | const { manageErrors, rollbackStep } = require('./errors') 3 | const { esError: EsError } = require('../lib/es') 4 | 5 | describe('manageErrors', () => { 6 | it('should add isElasticSearch if coming from ES', async () => { 7 | expect.assertions(2) 8 | try { 9 | await manageErrors(() => { throw new EsError('test manageErrors') }) 10 | } catch (err) { 11 | expect(err).toBeDefined() 12 | expect(err).toHaveProperty('isElasticSearch', true) 13 | } 14 | }) 15 | 16 | it('should set statusCode 409 if it is a conflict', async () => { 17 | expect.assertions(2) 18 | try { 19 | await manageErrors(() => { 20 | const e = new EsError('test manageErrors') 21 | e.body = {error: {type: 'resource_already_exists_exception'}} 22 | throw e 23 | }) 24 | } catch (err) { 25 | expect(err).toBeDefined() 26 | expect(err).toHaveProperty('output.statusCode', 409) 27 | } 28 | }) 29 | 30 | it('should set a contextual description as a prefix to the error message if given', async () => { 31 | expect.assertions(2) 32 | try { 33 | await manageErrors(() => { throw new Error('test manageErrors') }, 'My error contextual prefix') 34 | } catch (err) { 35 | expect(err).toBeDefined() 36 | expect(err).toHaveProperty('message', 'My error contextual prefix: test manageErrors') 37 | } 38 | }) 39 | 40 | it('should call the optional rollback function', async () => { 41 | expect.assertions(4) 42 | const rollback = jest.fn() 43 | try { 44 | await manageErrors(() => { throw new Error('test manageErrors') }, 'My error contextual prefix', rollback) 45 | } catch (err) { 46 | expect(err).toBeDefined() 47 | expect(err).toHaveProperty('message', 'My error contextual prefix: test manageErrors') 48 | expect(rollback).toHaveBeenCalled() 49 | expect(err).not.toHaveProperty('rollbackErrors') 50 | } 51 | }) 52 | 53 | it('should call the optional rollback function and store its error', async () => { 54 | expect.assertions(4) 55 | const rollback = jest.fn().mockRejectedValue(new Error('test manageErrors callback error')) 56 | try { 57 | await manageErrors(() => { throw new Error('test manageErrors') }, 'My error contextual prefix', rollback) 58 | } catch (err) { 59 | expect(err).toBeDefined() 60 | expect(err).toHaveProperty('message', 'My error contextual prefix: test manageErrors') 61 | expect(rollback).toHaveBeenCalled() 62 | expect(err).toHaveProperty('rollbackErrors', expect.any(Array)) 63 | } 64 | }) 65 | 66 | it('should call the optional rollback function and store its error', async () => { 67 | expect.assertions(4) 68 | const rollback = jest.fn().mockRejectedValue(new Error('test manageErrors callback error')) 69 | const error = new Error('test manageErrors') 70 | error.rollbackErrors = [ new Error('previous error') ] 71 | try { 72 | await manageErrors(() => { throw error }, 'My error contextual prefix', rollback) 73 | } catch (err) { 74 | expect(err).toBeDefined() 75 | expect(err).toHaveProperty('message', 'My error contextual prefix: test manageErrors') 76 | expect(rollback).toHaveBeenCalled() 77 | expect(err).toHaveProperty('rollbackErrors', expect.any(Array)) 78 | } 79 | }) 80 | // nominal - raw Error connection, no details 81 | it(`should throw on a raw sync connection Error`, async () => { 82 | expect.assertions(3) 83 | try { 84 | const err = new EsError('No Living connections') 85 | await manageErrors(() => { throw err }) 86 | } catch (err) { 87 | expect(err).toBeDefined() 88 | expect(err).toHaveProperty('output.payload.statusCode', 502) 89 | expect(err.message).toMatch(/No Living connections$/) 90 | } 91 | }) 92 | 93 | it(`should throw on a raw async connection Error`, async () => { 94 | expect.assertions(3) 95 | try { 96 | const err = new EsError('No Living connections') 97 | await manageErrors(() => delay(10).then(() => { throw err })) 98 | } catch (err) { 99 | expect(err).toBeDefined() 100 | expect(err).toHaveProperty('output.payload.statusCode', 502) 101 | expect(err.message).toMatch(/No Living connections$/) 102 | } 103 | }) 104 | 105 | // nominal - prefixed Error connection, no details 106 | it(`should throw on a prefixed connection Error`, async () => { 107 | expect.assertions(3) 108 | try { 109 | const err = new EsError('No Living connections') 110 | err.message = `This is a test: ${err.message}` 111 | await manageErrors(() => { throw err }) 112 | } catch (err) { 113 | expect(err).toBeDefined() 114 | expect(err).toHaveProperty('output.payload.statusCode', 502) 115 | expect(err).toHaveProperty('message', 'This is a test: No Living connections') 116 | } 117 | }) 118 | 119 | // nominal - prefixed Error connection, some details 120 | it(`should throw on a prefixed connection Error with ops`, async () => { 121 | expect.assertions(3) 122 | try { 123 | const err = new EsError('No Living connections') 124 | err.message = `This is a test: ${err.message}` 125 | err.ops = {more: 'details'} 126 | await manageErrors(() => { throw err }) 127 | } catch (err) { 128 | expect(err).toBeDefined() 129 | expect(err).toHaveProperty('output.payload.statusCode', 502) 130 | expect(err).toHaveProperty('message', 'This is a test: No Living connections') 131 | } 132 | }) 133 | }) 134 | 135 | describe('rollbackStep', () => { 136 | var error 137 | beforeEach(() => { 138 | error = new Error('"rollbackStep" test suite origin error') 139 | }) 140 | 141 | it('should run a sync rollback not failing', async () => { 142 | expect.assertions(0) 143 | try { 144 | await rollbackStep(error, () => true) 145 | } catch (e) { 146 | expect(e).not.toBeDefined() 147 | } 148 | }) 149 | 150 | it('should run an async rollback not failing', async () => { 151 | expect.assertions(0) 152 | try { 153 | await rollbackStep(error, () => delay(10)) 154 | } catch (e) { 155 | expect(e).not.toBeDefined() 156 | } 157 | }) 158 | 159 | it('should run a sync rollback throwing', async () => { 160 | expect.assertions(2) 161 | const rollbackError = new Error('"should run a sync rollback throwing" test') 162 | try { 163 | await rollbackStep(error, () => { throw rollbackError }) 164 | } catch (e) { 165 | expect(e).not.toBeDefined() 166 | } 167 | expect(error).toHaveProperty('rollbackErrors', expect.any(Array)) 168 | expect(error).toHaveProperty('rollbackErrors.0', rollbackError) 169 | }) 170 | 171 | it('should run an async rollback throwing', async () => { 172 | expect.assertions(2) 173 | const rollbackError = new Error('"should run an async rollback throwing" test') 174 | try { 175 | await rollbackStep(error, () => delay(10).then(() => { throw rollbackError })) 176 | } catch (e) { 177 | expect(e).not.toBeDefined() 178 | } 179 | expect(error).toHaveProperty('rollbackErrors', expect.any(Array)) 180 | expect(error).toHaveProperty('rollbackErrors.0', rollbackError) 181 | }) 182 | }) 183 | -------------------------------------------------------------------------------- /lib/es/index.js: -------------------------------------------------------------------------------- 1 | const Joi = require('joi') 2 | const elasticsearch = require('elasticsearch') 3 | const delay = require('delay') 4 | 5 | const FORCE_LONG_STACK_TRACE = Joi.attempt(process.env.LEAISTIC_FORCE_LONG_STACK_TRACE, 6 | Joi.boolean().default(false).empty('').label('Environment Variable LEAISTIC_FORCE_LONG_STACK_TRACE')) 7 | const ES_URL = Joi.attempt(process.env.ES_URL, 8 | Joi.string().uri().default('http://127.0.0.1:9200').empty('').label('Environment Variable ES_URL')) 9 | const ES_REFRESH = Joi.attempt(process.env.ES_REFRESH, 10 | Joi.number().default(1000).empty('').label('Environment Variable ES_REFRESH')) 11 | 12 | if (FORCE_LONG_STACK_TRACE || process.env.NODE_ENV !== 'production') { 13 | // long stack traces could be harmful in production ( more memory, slower, etc. ) 14 | require('trace') 15 | } 16 | const { log } = require('./logger') 17 | 18 | exports.es = ({url, client} = {}) => { 19 | if (url) { 20 | const host = Joi.attempt(url, Joi.string().uri().required().empty('').label('url')) 21 | exports._es = new elasticsearch.Client({host, log}) 22 | } 23 | 24 | if (client) { 25 | exports._es = client 26 | } 27 | 28 | if (!exports._es) { 29 | exports._es = new elasticsearch.Client({host: ES_URL, log}) 30 | } 31 | 32 | return exports._es 33 | } 34 | 35 | exports.connect = exports.es 36 | 37 | exports.esError = elasticsearch.errors._Abstract 38 | 39 | exports.awaitRefresh = async () => delay(ES_REFRESH) 40 | -------------------------------------------------------------------------------- /lib/es/index.test.js: -------------------------------------------------------------------------------- 1 | const elasticsearch = require('elasticsearch') 2 | 3 | const { es, awaitRefresh } = require('.') 4 | 5 | jest.setTimeout(60000) 6 | 7 | describe('es', () => { 8 | // nominal 9 | it(`should return a default client that is connected to an ES cluster`, async () => { 10 | const client = es() 11 | expect(client).toBeDefined() 12 | expect(typeof client.close).toBe('function') 13 | }) 14 | 15 | it(`should allow to specify an Url for the ES client`, async () => { 16 | const client = es({url: 'http://[::]:9200'}) 17 | expect(client).toBeDefined() 18 | expect(typeof client.close).toBe('function') 19 | }) 20 | 21 | it(`should allow to specify my own ES client and keep it as default`, async () => { 22 | const myClient = new elasticsearch.Client({host: 'http://[::]:9200'}) 23 | const client = es({client: myClient}) 24 | expect(client).toBeDefined() 25 | expect(typeof client.close).toBe('function') 26 | expect(client).toBe(myClient) 27 | 28 | const defaultClient = es() 29 | expect(client).toBeDefined() 30 | expect(typeof client.close).toBe('function') 31 | expect(defaultClient).toBe(client) 32 | }) 33 | }) 34 | 35 | describe('awaitRefresh', () => { 36 | // nominal 37 | it(`should wait for about 1s by default (default value for ElasticSearch refresh time)`, async () => { 38 | const start = new Date() 39 | await awaitRefresh() 40 | const end = new Date() 41 | const durationInMs = end - start 42 | const durationInS = durationInMs / 1000 43 | expect(durationInS).toBeCloseTo(1) 44 | }) 45 | }) 46 | -------------------------------------------------------------------------------- /lib/es/logger.js: -------------------------------------------------------------------------------- 1 | const pino = require('pino') 2 | const JSONtruncate = require('json-truncate') 3 | const Joi = require('joi') 4 | 5 | const DEBUG_ES = Joi.attempt(process.env.LEAISTIC_DEBUG_ES_IO, 6 | Joi.boolean().default(true).empty('').label('Environment Variable LEAISTIC_DEBUG_ES_IO')) 7 | const MAX_BODY_LENGTH = Joi.attempt(process.env.LEAISTIC_MAX_ES_BODY_LENGTH, 8 | Joi.number().default(240).empty('').label('Environment Variable LEAISTIC_MAX_ES_BODY_LENGTH')) 9 | const MAX_JSON_DEPTH = Joi.attempt(process.env.LEAISTIC_MAX_JSON_DEPTH, 10 | Joi.number().default(4).empty('').label('Environment Variable LEAISTIC_MAX_ES_JSON_DEPTH')) 11 | 12 | const ES_LOG_LEVEL_OK = Joi.attempt(process.env.LEAISTIC_ES_LOG_LEVEL_OK, 13 | Joi.string().trim().lowercase().default('info').empty('').valid('trace', 'debug', 'info', 'warn', 'error', 'fatal').label('Environment Variable LEAISTIC_ES_LOG_LEVEL_OK')) 14 | const ES_LOG_LEVEL_ERROR = Joi.attempt(process.env.LEAISTIC_ES_LOG_LEVEL_ERROR, 15 | Joi.string().trim().lowercase().default('error').empty('').valid('trace', 'debug', 'info', 'warn', 'error', 'fatal').label('Environment Variable LEAISTIC_ES_LOG_LEVEL_ERROR')) 16 | const ES_LOG_THRESHOLD = Joi.attempt(process.env.LEAISTIC_ES_LOG_THRESHOLD, 17 | Joi.string().trim().lowercase().default('info').empty('').valid('trace', 'debug', 'info', 'warn', 'error', 'fatal').label('Environment Variable LEAISTIC_ES_LOG_THRESHOLD')) 18 | 19 | const defaultParseConfig = {maxDepth: MAX_JSON_DEPTH, maxLength: MAX_BODY_LENGTH} 20 | 21 | const parse = (strOrJson = '', config = defaultParseConfig) => { 22 | const {maxDepth, maxLength} = config 23 | try { 24 | const json = JSON.parse(strOrJson) 25 | if (maxDepth <= 0) { 26 | return json 27 | } 28 | return JSONtruncate(json, { 29 | maxDepth, 30 | replace: '[⤵]' 31 | }) 32 | } catch (e) { 33 | return strOrJson.length > maxLength ? `${strOrJson.substr(0, maxLength)}[⤵]` : strOrJson 34 | } 35 | } 36 | exports.parse = parse 37 | 38 | // see https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/logging.html 39 | exports.log = function esLogger (config = {}) { 40 | // config is the object passed to the client constructor. 41 | const logger = pino({ 42 | name: 'Leaistic ↔ ElasticSearch', 43 | serializers: { 44 | err: pino.stdSerializers.err 45 | }, 46 | level: ES_LOG_THRESHOLD, 47 | ...config 48 | }) 49 | 50 | this.error = (message, object, ...rest) => logger.error(object, message, ...rest) 51 | this.warning = (message, object, ...rest) => logger.warn(object, message, ...rest) 52 | this.info = (message, object, ...rest) => logger.info(object, message, ...rest) 53 | this.debug = (message, object, ...rest) => logger.debug(object, message, ...rest) 54 | 55 | // ES trace mode is used to track HTTP requests, which tends to be actually more important than `debug` level content 56 | // pino has some standard format ( from default serializers) for `req` and `res` that we can leverage to have nice looking logs 57 | this.trace = (method, req, body, responseBody, statusCode) => { 58 | const level = statusCode < 500 ? ES_LOG_LEVEL_OK : ES_LOG_LEVEL_ERROR 59 | const {protocol, hostname, port, path, headers} = req 60 | const message = 'request completed' 61 | 62 | logger[level]({ 63 | req: { 64 | method: (method || '').toLowerCase(), 65 | url: `${protocol}//${hostname}${((protocol === 'http:' && port === 80) || (protocol === 'https:' && port === 443)) ? '' : `:${port}`}${path}`, 66 | headers: headers === null ? undefined : headers, 67 | remoteAddress: hostname, 68 | remotePort: port, 69 | body: DEBUG_ES ? parse(body) : undefined 70 | }, 71 | res: { 72 | statusCode, 73 | body: DEBUG_ES ? parse(responseBody) : undefined 74 | } 75 | }, message) 76 | } 77 | this.close = () => { /* pino's loggers do not need to be closed */ } 78 | } 79 | -------------------------------------------------------------------------------- /lib/es/logger.test.js: -------------------------------------------------------------------------------- 1 | const repeat = require('lodash.repeat') 2 | 3 | const { parse, log: Logger } = require('./logger') 4 | 5 | describe('parse', () => { 6 | // nominal 7 | it(`should be able to parse a short string`, async () => { 8 | const res = parse('aaa') 9 | expect(res).toBe('aaa') 10 | }) 11 | 12 | it(`should be able to parse a long string and shorten it`, async () => { 13 | const res = parse(repeat('a', 10), {maxLength: 5, maxDepth: -1}) 14 | expect(res).toMatch(/^a{5}\[⤵\]$/) 15 | }) 16 | 17 | it(`should be able to parse a JSON with no depth `, async () => { 18 | const res = parse('{"hello": "world!"}', {maxLength: -1, maxDepth: -1}) 19 | expect(res).toBeDefined() 20 | expect(res).toHaveProperty('hello', 'world!') 21 | }) 22 | 23 | it(`should be able to parse a JSON with a limited depth `, async () => { 24 | const res = parse('{"foo": {"bar": {"baz": true}}}', {maxLength: -1, maxDepth: 2}) 25 | expect(res).toHaveProperty('foo') 26 | expect(res).toHaveProperty('foo.bar') 27 | expect(res).not.toHaveProperty('foo.bar.baz') 28 | }) 29 | }) 30 | 31 | describe('log', () => { 32 | it(`should provide a default logger compatible with ES`, async () => { 33 | const logger = new Logger() 34 | expect(logger).toBeDefined() 35 | expect(logger).toHaveProperty('error') 36 | expect(logger).toHaveProperty('warning') 37 | expect(logger).toHaveProperty('info') 38 | expect(logger).toHaveProperty('debug') 39 | expect(logger).toHaveProperty('trace') 40 | }) 41 | }) 42 | -------------------------------------------------------------------------------- /lib/hlrollbacks.js: -------------------------------------------------------------------------------- 1 | const { 2 | rollbackAliasCreation, 3 | rollbackIndexTemplateCreation, 4 | rollbackIndexCreation, 5 | rollbackAliasSwitch 6 | } = require('./rollbacks') 7 | 8 | const { 9 | rollbackStep 10 | } = require('./errors') 11 | 12 | const rollbackFromIndexCreation = ops => name => async (index, error, origin) => { 13 | await rollbackStep(error, () => rollbackIndexCreation({index}, error, origin)) 14 | if (ops.template) { 15 | await rollbackStep(error, () => rollbackIndexTemplateCreation({name}, error, origin)) 16 | } 17 | } 18 | exports.rollbackFromIndexCreation = rollbackFromIndexCreation 19 | 20 | const rollbackFromAliasCreation = ops => async (name, index, error, origin) => { 21 | await rollbackStep(error, () => rollbackAliasCreation({index, name}, error, origin)) 22 | await rollbackStep(error, () => rollbackFromIndexCreation(ops)(name)(index, error, origin)) 23 | } 24 | exports.rollbackFromAliasCreation = rollbackFromAliasCreation 25 | 26 | const rollbackFromReindex = ops => async (name, index, error, origin) => { 27 | await rollbackStep(error, () => rollbackIndexCreation({index}, error, origin)) 28 | if (ops.template) { 29 | await rollbackStep(error, () => rollbackIndexTemplateCreation({name}, error, origin)) 30 | } 31 | } 32 | exports.rollbackFromReindex = rollbackFromReindex 33 | 34 | const rollbackFromAliasSwitch = ops => async (name, sourceIndex, index, error, origin) => { 35 | await rollbackStep(error, () => rollbackFromReindex(ops)(name, index, error, origin)) // is this really a good idea ? 36 | await rollbackStep(error, () => rollbackAliasSwitch({name, sourceIndex, index}, error, origin)) 37 | } 38 | exports.rollbackFromAliasSwitch = rollbackFromAliasSwitch 39 | -------------------------------------------------------------------------------- /lib/hlrollbacks.test.js: -------------------------------------------------------------------------------- 1 | const uuid = require('uuid/v1') 2 | const { 3 | rollbackFromIndexCreation, 4 | rollbackFromAliasCreation, 5 | rollbackFromReindex, 6 | rollbackFromAliasSwitch 7 | } = require('./hlrollbacks') 8 | const { es, esError } = require('./es') 9 | const { suffix } = require('./indices') 10 | 11 | jest.setTimeout(60000) 12 | 13 | describe('rollbackFromIndexCreation', () => { 14 | const name = uuid() 15 | const index = suffix(name) 16 | 17 | const ensureIndexIsDeleted = async ({index}) => { 18 | try { 19 | await es().indices.delete({index}) 20 | } catch (e) { 21 | // ignore 22 | expect(e instanceof esError).toBeTruthy() 23 | expect(e.statusCode).toBe(404) 24 | } 25 | } 26 | 27 | beforeEach(async () => { 28 | await es().indices.create({index}) 29 | await es().indices.refresh({index}) 30 | }) 31 | 32 | afterEach(async () => { 33 | await ensureIndexIsDeleted({index}) 34 | }) 35 | 36 | it(`should delete an existing index`, async () => { 37 | expect(await es().indices.exists({index})).toBeTruthy() 38 | const ops = {} 39 | const rollback = rollbackFromIndexCreation(ops)(name) 40 | await rollback(index, new Error('"should delete an existing index" test cause'), `"should delete an existing index" test`) 41 | expect(await es().indices.exists({index})).toBeFalsy() 42 | }) 43 | 44 | it(`should error and do nothing when trying to create an existing index`, async () => { 45 | const cause = Error('[resource_already_exists_exception] "should error and do nothing when trying to create an existing index" test cause') 46 | cause.statusCode = 400 47 | cause.body = {error: {type: 'resource_already_exists_exception'}} 48 | try { 49 | const ops = {} 50 | const rollback = rollbackFromIndexCreation(ops)(name) 51 | await rollback(index, cause, `"should error and do nothing when trying to create an existing index" test`) 52 | } catch (e) { 53 | expect(e).not.toBeDefined() 54 | } 55 | expect(cause).toHaveProperty('rollbackErrors', expect.any(Array)) 56 | expect(cause).toHaveProperty('rollbackErrors.0', expect.any(Error)) 57 | expect(cause.rollbackErrors[0].message).toMatch(/^Could not rollback/) 58 | expect(cause.rollbackErrors[0]).toHaveProperty('statusCode', 400) 59 | // it should still exist 60 | expect(await es().indices.exists({index})).toBeTruthy() 61 | }) 62 | 63 | it(`should error when index to delete during rollback has already been deleted`, async () => { 64 | await ensureIndexIsDeleted({index}) 65 | const cause = new Error('"should error when index to delete during rollback has already been deleted" test cause') 66 | try { 67 | const ops = {} 68 | const rollback = rollbackFromIndexCreation(ops)(name) 69 | await rollback(index, cause, `"should error when index to delete during rollback has already been deleted" test`) 70 | } catch (e) { 71 | expect(e).not.toBeDefined() 72 | } 73 | expect(cause).toHaveProperty('rollbackErrors', expect.any(Array)) 74 | expect(cause).toHaveProperty('rollbackErrors.0', expect.any(Error)) 75 | expect(cause.rollbackErrors[0].message).toMatch(/^Could not rollback/) 76 | expect(cause.rollbackErrors[0]).toHaveProperty('statusCode', 404) 77 | // it was already delted, it should still be 78 | expect(await es().indices.exists({index})).toBeFalsy() 79 | }) 80 | }) 81 | 82 | describe('rollbackFromAliasCreation', () => { 83 | const name = uuid() 84 | const index = suffix(uuid()) 85 | 86 | const ensureAliasIsDeleted = async ({name}) => { 87 | try { 88 | await es().indices.deleteAlias({name, index: '_all'}) 89 | } catch (e) { 90 | // ignore 91 | expect(e instanceof esError).toBeTruthy() 92 | expect(e.statusCode).toBe(404) 93 | } 94 | } 95 | const ensureIndexIsDeleted = async ({index}) => { 96 | try { 97 | await es().indices.delete({index}) 98 | } catch (e) { 99 | // ignore 100 | expect(e instanceof esError).toBeTruthy() 101 | expect(e.statusCode).toBe(404) 102 | } 103 | } 104 | 105 | beforeEach(async () => { 106 | await es().indices.create({index}) 107 | await es().indices.putAlias({name, index}) 108 | await es().indices.refresh({index}) 109 | }) 110 | 111 | afterEach(async () => { 112 | await ensureAliasIsDeleted({name}) 113 | await ensureIndexIsDeleted({index}) 114 | }) 115 | 116 | it(`should delete an existing alias and index`, async () => { 117 | expect(await es().indices.exists({index})).toBeTruthy() 118 | expect(await es().indices.existsAlias({name})).toBeTruthy() 119 | const ops = {} 120 | const rollback = rollbackFromAliasCreation(ops) 121 | await rollback(name, index, new Error('"should delete an existing index" test cause'), `"should delete an existing index" test`) 122 | expect(await es().indices.existsAlias({name})).toBeFalsy() 123 | expect(await es().indices.exists({index})).toBeFalsy() 124 | }) 125 | 126 | it(`should not error when trying to delete an unknown alias, but still delete the index`, async () => { 127 | await ensureAliasIsDeleted({name}) 128 | const cause = new Error('"should not error when trying to delete an unknown alias, but still delete the index" test cause') 129 | try { 130 | const ops = {} 131 | const rollback = rollbackFromAliasCreation(ops) 132 | await rollback(name, index, cause, `"should not error when trying to delete an unknown alias, but still delete the index" test`) 133 | } catch (e) { 134 | expect(e).not.toBeDefined() 135 | } 136 | expect(cause).toHaveProperty('rollbackErrors', expect.any(Array)) 137 | expect(cause).toHaveProperty('rollbackErrors.0', expect.any(Error)) 138 | expect(cause.rollbackErrors[0].message).toMatch(/^Could not rollback/) 139 | expect(cause.rollbackErrors[0]).toHaveProperty('statusCode', 404) 140 | // it was already deleted, it should still be 141 | expect(await es().indices.existsAlias({name})).toBeFalsy() 142 | // index should also be deleted in the process 143 | expect(await es().indices.exists({index})).toBeFalsy() 144 | }) 145 | 146 | it(`should not error when index to delete during rollback has already been deleted`, async () => { 147 | await ensureIndexIsDeleted({index}) 148 | const cause = new Error('"should error when index to delete during rollback has already been deleted" test cause') 149 | try { 150 | const ops = {} 151 | const rollback = rollbackFromAliasCreation(ops) 152 | await rollback(name, index, cause, `"should error when index to delete during rollback has already been deleted" test`) 153 | } catch (e) { 154 | expect(e).not.toBeDefined() 155 | } 156 | expect(cause).toHaveProperty('rollbackErrors', expect.any(Array)) 157 | expect(cause).toHaveProperty('rollbackErrors.0', expect.any(Error)) 158 | expect(cause.rollbackErrors[0].message).toMatch(/^Could not rollback/) 159 | expect(cause.rollbackErrors[0]).toHaveProperty('statusCode', 404) 160 | expect(await es().indices.existsAlias({name})).toBeFalsy() 161 | // it was already deleted, it should still be 162 | expect(await es().indices.exists({index})).toBeFalsy() 163 | }) 164 | }) 165 | 166 | describe('rollbackFromReindex', () => { 167 | const name = uuid() 168 | const sourceIndex = `${suffix(name)}-source` 169 | const index = suffix(name) 170 | 171 | const ensureAliasIsDeleted = async ({name}) => { 172 | try { 173 | await es().indices.deleteAlias({name, index: '_all'}) 174 | } catch (e) { 175 | // ignore 176 | expect(e instanceof esError).toBeTruthy() 177 | expect(e.statusCode).toBe(404) 178 | } 179 | } 180 | const ensureIndexIsDeleted = async ({index}) => { 181 | try { 182 | await es().indices.delete({index}) 183 | } catch (e) { 184 | // ignore 185 | expect(e instanceof esError).toBeTruthy() 186 | expect(e.statusCode).toBe(404) 187 | } 188 | } 189 | const ensureAliasIndexIs = async (name, index) => Object.keys((await es().indices.getAlias({name})))[0] === index 190 | 191 | beforeEach(async () => { 192 | await es().indices.create({index: sourceIndex}) 193 | await es().indices.create({index}) 194 | await es().indices.putAlias({name, index: sourceIndex}) 195 | await es().indices.refresh({index: sourceIndex}) 196 | }) 197 | 198 | afterEach(async () => { 199 | await ensureAliasIsDeleted({name}) 200 | await ensureIndexIsDeleted({index: sourceIndex}) 201 | await ensureIndexIsDeleted({index}) 202 | }) 203 | 204 | it(`should delete the new index and keep the source one`, async () => { 205 | expect(await es().indices.exists({index: sourceIndex})).toBeTruthy() 206 | expect(await es().indices.exists({index})).toBeTruthy() 207 | expect(await ensureAliasIndexIs(name, sourceIndex)).toBeTruthy() 208 | const cause = new Error('"should delete the new index and keep the source one" test cause') 209 | try { 210 | const ops = {} 211 | const rollback = rollbackFromReindex(ops) 212 | await rollback(name, index, cause, `"should delete the new index and keep the source one" test`) 213 | } catch (e) { 214 | expect(e).not.toBeDefined() 215 | } 216 | expect(await es().indices.exists({index: sourceIndex})).toBeTruthy() 217 | expect(await es().indices.exists({index})).toBeFalsy() 218 | expect(await ensureAliasIndexIs(name, sourceIndex)).toBeTruthy() 219 | }) 220 | 221 | it(`should not error when trying to delete an unknown index`, async () => { 222 | expect(await es().indices.exists({index: sourceIndex})).toBeTruthy() 223 | await ensureIndexIsDeleted({index}) 224 | expect(await es().indices.exists({index})).toBeFalsy() 225 | expect(await ensureAliasIndexIs(name, sourceIndex)).toBeTruthy() 226 | const cause = new Error('should not error when trying to delete an unknown index" test cause') 227 | try { 228 | const ops = {} 229 | const rollback = rollbackFromReindex(ops) 230 | await rollback(name, index, cause, `"should not error when trying to delete an unknown index" test`) 231 | } catch (e) { 232 | expect(e).not.toBeDefined() 233 | } 234 | expect(cause).toHaveProperty('rollbackErrors', expect.any(Array)) 235 | expect(cause).toHaveProperty('rollbackErrors.0', expect.any(Error)) 236 | expect(cause.rollbackErrors[0].message).toMatch(/^Could not rollback/) 237 | expect(cause.rollbackErrors[0]).toHaveProperty('statusCode', 404) 238 | expect(await es().indices.exists({index: sourceIndex})).toBeTruthy() 239 | expect(await es().indices.exists({index})).toBeFalsy() 240 | expect(await ensureAliasIndexIs(name, sourceIndex)).toBeTruthy() 241 | }) 242 | }) 243 | 244 | describe('rollbackFromAliasSwitch', () => { 245 | const name = uuid() 246 | const sourceIndex = `${suffix(name)}-source` 247 | const index = suffix(name) 248 | 249 | const ensureAliasIsDeleted = async ({name}) => { 250 | try { 251 | await es().indices.deleteAlias({name, index: '_all'}) 252 | } catch (e) { 253 | // ignore 254 | expect(e instanceof esError).toBeTruthy() 255 | expect(e.statusCode).toBe(404) 256 | } 257 | } 258 | const ensureIndexIsDeleted = async ({index}) => { 259 | try { 260 | await es().indices.delete({index}) 261 | } catch (e) { 262 | // ignore 263 | expect(e instanceof esError).toBeTruthy() 264 | expect(e.statusCode).toBe(404) 265 | } 266 | } 267 | const ensureAliasIndexIs = async (name, index) => { 268 | try { 269 | const alias = await es().indices.getAlias({name}) 270 | return Object.keys(alias)[0] === index 271 | } catch (e) { 272 | // ignore 273 | expect(e instanceof esError).toBeTruthy() 274 | expect(e.statusCode).toBe(404) 275 | } 276 | } 277 | 278 | beforeEach(async () => { 279 | await es().indices.create({index: sourceIndex}) 280 | await es().indices.create({index}) 281 | await es().indices.putAlias({name, index}) 282 | await es().indices.refresh({index}) 283 | }) 284 | 285 | afterEach(async () => { 286 | await ensureAliasIsDeleted({name}) 287 | await ensureIndexIsDeleted({index: sourceIndex}) 288 | await ensureIndexIsDeleted({index}) 289 | }) 290 | 291 | it(`should delete the new index and keep the source one, and switch back the alias to the source one`, async () => { 292 | expect(await es().indices.exists({index: sourceIndex})).toBeTruthy() 293 | expect(await es().indices.exists({index})).toBeTruthy() 294 | expect(await ensureAliasIndexIs(name, index)).toBeTruthy() 295 | const cause = new Error('"should delete the new index and keep the source one, and switch back the alias to the source one" test cause') 296 | try { 297 | const ops = {} 298 | const rollback = rollbackFromAliasSwitch(ops) 299 | await rollback(name, sourceIndex, index, cause, `"should delete the new index and keep the source one, and switch back the alias to the source one" test`) 300 | } catch (e) { 301 | expect(e).not.toBeDefined() 302 | } 303 | expect(await es().indices.exists({index: sourceIndex})).toBeTruthy() 304 | expect(await es().indices.exists({index})).toBeFalsy() 305 | expect(await ensureAliasIndexIs(name, sourceIndex)).toBeTruthy() 306 | }) 307 | 308 | it(`should not error when trying to delete an unknown index, and still switch back the alias to the source one`, async () => { 309 | expect(await ensureAliasIndexIs(name, index)).toBeTruthy() 310 | await ensureIndexIsDeleted({index}) 311 | expect(await es().indices.exists({index})).toBeFalsy() 312 | const cause = new Error('"should not error when trying to delete an unknown index, and still switch back the alias to the source one" test cause') 313 | try { 314 | const ops = {} 315 | const rollback = rollbackFromAliasSwitch(ops) 316 | await rollback(name, sourceIndex, index, cause, `"should not error when trying to delete an unknown index, and still switch back the alias to the source one" test`) 317 | } catch (e) { 318 | expect(e).not.toBeDefined() 319 | } 320 | expect(cause).toHaveProperty('rollbackErrors', expect.any(Array)) 321 | expect(cause).toHaveProperty('rollbackErrors.0', expect.any(Error)) 322 | expect(cause.rollbackErrors[0].message).toMatch(/^Could not rollback/) 323 | expect(cause.rollbackErrors[0]).toHaveProperty('statusCode', 404) 324 | expect(await es().indices.exists({index: sourceIndex})).toBeTruthy() 325 | expect(await es().indices.exists({index})).toBeFalsy() 326 | expect(await ensureAliasIndexIs(name, sourceIndex)).toBeTruthy() 327 | }) 328 | }) 329 | -------------------------------------------------------------------------------- /lib/indices.js: -------------------------------------------------------------------------------- 1 | const Boom = require('boom') 2 | 3 | const { 4 | checkAliasDoesNotExists, 5 | checkAliasAlreadyExists, 6 | shouldUpdateTemplate, 7 | updateTemplate, 8 | createIndex, 9 | deleteIndex, 10 | checkIndexAlreadyExists, 11 | checkIndexDoesNotExist, 12 | findAliasIndex, 13 | createAlias, 14 | switchAlias, 15 | reindex, 16 | client 17 | } = require('./ops') 18 | 19 | const {log} = require('./logger') 20 | 21 | const { 22 | rollbackFromAliasCreation, 23 | rollbackFromIndexCreation, 24 | rollbackFromAliasSwitch, 25 | rollbackFromReindex 26 | } = require('./hlrollbacks') 27 | 28 | const { 29 | indexCreation, 30 | indexUpdate, 31 | indexDeletion 32 | } = require('./state') 33 | 34 | const { 35 | manageErrors 36 | } = require('./errors') 37 | 38 | const suffix = (name, date) => `${name}-${(date || new Date()).toISOString().toLowerCase()}` 39 | exports.suffix = suffix 40 | 41 | const getOps = async (opsFn) => { 42 | const {ops} = await opsFn() 43 | return ops 44 | } 45 | 46 | exports.create = async (name, { body } = { body: {} }) => { 47 | const ops = {} 48 | const index = suffix(name) 49 | return indexCreation(name, async () => { 50 | await manageErrors(async () => { 51 | ops.preChecks = await Promise.all([ 52 | getOps((ops) => checkAliasDoesNotExists(name, ops)), 53 | getOps((ops) => checkIndexDoesNotExist(index, ops)) 54 | ]) 55 | }, 56 | `The index (${index}) or alias (${name}) were already existing` 57 | ) 58 | 59 | if (shouldUpdateTemplate(body)) { 60 | await updateTemplate(name, body, ops) 61 | } 62 | 63 | const noRollback = () => {} 64 | await checkAliasDoesNotExists(name, noRollback, ops) 65 | await createIndex(index, rollbackFromIndexCreation(ops)(name), ops) 66 | 67 | await checkAliasDoesNotExists(name, rollbackFromAliasCreation(ops), ops) 68 | await createAlias(name, index, rollbackFromAliasCreation(ops), ops) 69 | const {sourceIndex} = await findAliasIndex(name, ops) 70 | if (index !== sourceIndex) { 71 | const origin = `"${name}" After Index "${index}" creation, the Alias was already bound to "${sourceIndex}", if you want to enforce a new index, either 'delete' the existing one, or update it` 72 | const error = Boom.conflict() 73 | await rollbackFromAliasCreation(name, index, error, origin) 74 | } 75 | return {name, index, ops} 76 | }) 77 | } 78 | 79 | exports.update = async (name, { body } = { body: {} }, reindexer) => { 80 | let ops = {} 81 | 82 | return indexUpdate(name, async () => { 83 | await checkAliasAlreadyExists(name, ops) 84 | const {sourceIndex} = await findAliasIndex(name, ops) 85 | if (shouldUpdateTemplate(body)) { 86 | await updateTemplate(name, body, ops) 87 | } 88 | 89 | const index = suffix(name) 90 | 91 | await createIndex(index, rollbackFromIndexCreation(ops), ops) 92 | 93 | // custom reindexer should take care of the refresh 94 | reindexer = reindexer || (index => reindex(name, sourceIndex, index, rollbackFromReindex(ops), ops)) 95 | 96 | try { 97 | const reindexResult = await reindexer(index, client()) 98 | log().info(reindexResult, `✅"${index}" reindexation done`) 99 | if (reindexResult.ops) { 100 | ops = { ...ops, ...reindexResult.ops } 101 | } else { 102 | ops.reindex = reindexResult || { failures: [] } 103 | } 104 | } catch (err) { 105 | log().error({ err }, `🚨"${index}" reindexation failed`) 106 | rollbackFromReindex(ops) 107 | } 108 | 109 | await manageErrors(async () => { 110 | ops.postReindex = await Promise.all([ 111 | getOps((ops) => checkIndexAlreadyExists(sourceIndex, ops)), 112 | getOps((ops) => checkIndexAlreadyExists(index, ops)), 113 | getOps((ops) => checkAliasAlreadyExists(name, ops)) 114 | ]) 115 | }, 116 | `The original index (${sourceIndex})/destination index (${index})/alias (${name}) status after reindexation was not consistent and has probably been altered by a third party`, 117 | error => rollbackFromReindex(name, index, error, `the checks after ${sourceIndex} reindexation to ${index}, and before the switch of ${name} alias`) 118 | ) 119 | 120 | await switchAlias(name, sourceIndex, index, rollbackFromAliasSwitch, ops) 121 | 122 | ops.postAliasSwitch = await Promise.all([ 123 | getOps((ops) => deleteIndex(sourceIndex, () => { throw Boom.failedDependency(`Source Index "${sourceIndex}" could not be deleted`) }, ops)), 124 | getOps((ops) => checkAliasAlreadyExists(name, ops)) 125 | ]) 126 | return {name, sourceIndex, index, ops} 127 | }) 128 | } 129 | 130 | exports.delete = async (name) => { 131 | const ops = {} 132 | return indexDeletion(name, async () => { 133 | return manageErrors(async () => { 134 | const {sourceIndex: index} = await findAliasIndex(name, ops) 135 | ops.preChecks = await Promise.all([ 136 | getOps((ops) => checkAliasAlreadyExists(name, ops)), 137 | getOps((ops) => checkIndexAlreadyExists(index, ops)) 138 | ]) 139 | 140 | const cantRollbackIndex = (index, e, origin) => { throw e } 141 | ops.deletions = await Promise.all([ 142 | getOps((ops) => deleteIndex(index, cantRollbackIndex, ops)) 143 | ]) 144 | 145 | return {name, index, ops} 146 | }, 147 | `The alias (${name}) or the index it is pointing to, was probably missing` 148 | ) 149 | }) 150 | } 151 | -------------------------------------------------------------------------------- /lib/indices.test.js: -------------------------------------------------------------------------------- 1 | const uuid = require('uuid/v1') 2 | 3 | const { create, update, delete: del, suffix } = require('./indices') 4 | const { es } = require('./es') 5 | 6 | jest.setTimeout(60000) 7 | 8 | describe('create', () => { 9 | const name = uuid() 10 | 11 | // nominal 12 | it(`should allow creating a new index with no template`, async () => { 13 | const res = await create(name) 14 | expect(res.name).toBe(name) 15 | expect(res.index).toMatch(new RegExp(`^${name}-\\d+-\\d+-\\d+t\\d+:\\d+:\\d+.\\d+z`)) 16 | expect(res.ops).toHaveProperty('preChecks') 17 | expect(res.ops.preChecks).toEqual(expect.any(Array)) 18 | expect(res.ops.preChecks.length).toBe(2) 19 | expect(res.ops.preChecks[0]).toHaveProperty('aliasExists', false) 20 | expect(res.ops.preChecks[1]).toHaveProperty('indexExists', false) 21 | expect(res.ops).toHaveProperty('index') 22 | expect(res.ops.index.acknowledged).toBe(true) 23 | expect(res.ops.index.shards_acknowledged).toBe(true) 24 | expect(res.ops.index.index).toEqual(res.index) 25 | }) 26 | 27 | // existing 28 | it(`should forbid creating the same index again`, async () => { 29 | expect.assertions(3) 30 | try { 31 | await create(name) 32 | } catch (error) { 33 | expect(error).toBeDefined() 34 | expect(error.output.statusCode).toBe(409) 35 | expect(error.message).toMatch(name) 36 | } 37 | }) 38 | 39 | it(`should forbid creating the same index again in parallel`, async () => { 40 | expect.assertions(3) 41 | const name = uuid() + '-parallel' 42 | try { 43 | await Promise.all([ 44 | create(name), 45 | create(name), 46 | create(name), 47 | create(name), 48 | create(name), 49 | create(name), 50 | create(name), 51 | create(name) 52 | ]) 53 | } catch (error) { 54 | expect(error).toBeDefined() 55 | expect(error.output.statusCode).toBe(423) 56 | expect(error.message).toMatch(name) 57 | } 58 | }) 59 | 60 | it(`should allow creating a new index with a template`, async () => { 61 | const name = uuid() 62 | const body = { 63 | 'index_patterns': [`${name}-*`], 64 | 'settings': { 65 | 'number_of_shards': 1 66 | }, 67 | 'aliases': { 68 | 'alias1': {}, 69 | '{index}-alias': {} 70 | } 71 | } 72 | const res = await create(name, {body}) 73 | expect(res.name).toBe(name) 74 | expect(res.index).toMatch(new RegExp(`^${name}-\\d+-\\d+-\\d+t\\d+:\\d+:\\d+.\\d+z`)) 75 | expect(res.ops).toHaveProperty('preChecks') 76 | expect(res.ops.preChecks).toEqual(expect.any(Array)) 77 | expect(res.ops.preChecks.length).toBe(2) 78 | expect(res.ops.preChecks[0]).toHaveProperty('aliasExists', false) 79 | expect(res.ops.preChecks[1]).toHaveProperty('indexExists', false) 80 | expect(res.ops).toHaveProperty('index') 81 | expect(res.ops.index.acknowledged).toBe(true) 82 | expect(res.ops.index.shards_acknowledged).toBe(true) 83 | expect(res.ops.index.index).toEqual(res.index) 84 | expect(res.ops).toHaveProperty('template', {acknowledged: true}) 85 | }) 86 | }) 87 | 88 | describe('update', () => { 89 | const name = uuid() 90 | const index = suffix(name) 91 | 92 | beforeEach(async () => { 93 | await es().indices.create({index}) 94 | await es().indices.refresh({index}) 95 | await es().indices.putAlias({name, index}) 96 | }) 97 | 98 | afterEach(async () => { 99 | try { 100 | await es().indices.delete({index}) 101 | } catch (e) { 102 | if (!e || e.statusCode !== 404) { 103 | throw e 104 | } 105 | } 106 | try { 107 | await es().indices.deleteAlias({name, index}) 108 | } catch (e) { 109 | if (!e || e.statusCode !== 404) { 110 | throw e 111 | } 112 | } 113 | }) 114 | 115 | // nominal - you are likely to only reindex if setting something new in its mapping or settings 116 | it(`should allow updating an index with a template`, async () => { 117 | const body = { 118 | 'index_patterns': [`${name}-*`], 119 | 'settings': { 120 | 'number_of_shards': 1 121 | }, 122 | 'aliases': { 123 | 'alias1': {}, 124 | '{index}-alias': {} 125 | } 126 | } 127 | const res = await update(name, {body}) 128 | const indexShouldMatch = new RegExp(`^${name}-\\d+-\\d+-\\d+t\\d+:\\d+:\\d+.\\d+z`) 129 | expect(res.name).toBe(name) 130 | expect(res.index).toMatch(indexShouldMatch) 131 | expect(res.sourceIndex).toMatch(indexShouldMatch) 132 | expect(res.ops).toHaveProperty('aliasExists', true) 133 | 134 | expect(res.ops).toHaveProperty('findAlias') 135 | expect(res.ops.findAlias[index]).toBeDefined() 136 | expect(res.ops.findAlias[index]).toHaveProperty('aliases') 137 | expect(res.ops.findAlias[index].aliases).toHaveProperty(name, {}) 138 | 139 | expect(res.ops).toHaveProperty('index') 140 | expect(res.ops.index).toHaveProperty('acknowledged', true) 141 | expect(res.ops.index).toHaveProperty('shards_acknowledged', true) 142 | expect(res.ops.index).toHaveProperty('index') 143 | expect(res.ops.index.index).toMatch(indexShouldMatch) 144 | 145 | expect(res.ops).toHaveProperty('postReindex') 146 | expect(res.ops.postReindex).toEqual(expect.any(Array)) 147 | expect(res.ops.postReindex.length).toBe(3) 148 | expect(res.ops.postReindex[0]).toHaveProperty('indexExists', true) 149 | expect(res.ops.postReindex[1]).toHaveProperty('indexExists', true) 150 | expect(res.ops.postReindex[2]).toHaveProperty('aliasExists', true) 151 | 152 | expect(res.ops).toHaveProperty('postAliasSwitch') 153 | expect(res.ops.postAliasSwitch).toEqual(expect.any(Array)) 154 | expect(res.ops.postAliasSwitch.length).toBe(2) 155 | 156 | expect(res.ops.postAliasSwitch[0]).toHaveProperty('indexDeletion', {acknowledged: true}) 157 | expect(res.ops.postAliasSwitch[1]).toHaveProperty('aliasExists', true) 158 | 159 | expect(res.ops).toHaveProperty('reindex') 160 | expect(res.ops.reindex).toHaveProperty('failures') 161 | expect(res.ops.reindex.failures).toHaveProperty('length', 0) 162 | expect(res.ops.reindex).toHaveProperty('total', 0) 163 | 164 | expect(res.ops).toHaveProperty('switchAlias') 165 | expect(res.ops.switchAlias).toHaveProperty('acknowledged', true) 166 | 167 | expect(res.ops.index.acknowledged).toBe(true) 168 | expect(res.ops.index.shards_acknowledged).toBe(true) 169 | expect(res.ops.index.index).toEqual(res.index) 170 | 171 | expect(res.ops).toHaveProperty('template') 172 | expect(res.ops.template).toHaveProperty('acknowledged', true) 173 | }) 174 | 175 | it(`should allow updating an index with a template and a custom reindexer`, async () => { 176 | // use a new name to avoid concurrent conflict with previous test 177 | const name = uuid() 178 | const index = suffix(name) 179 | 180 | await es().indices.create({index}) 181 | await es().indices.refresh({index}) 182 | await es().indices.putAlias({name, index}) 183 | 184 | const body = { 185 | 'index_patterns': [`${name}-*`], 186 | 'mappings': { 187 | '_doc': { 188 | '_source': { 189 | 'enabled': false 190 | }, 191 | 'properties': { 192 | 'hello': { type: 'keyword' } 193 | } 194 | } 195 | } 196 | } 197 | 198 | const data = [ 199 | { hello: 'world' }, 200 | { hello: 'foo' }, 201 | { hello: 'bar' }, 202 | { hello: 'baz' } 203 | ] 204 | 205 | const reindexer = async (indexName, client) => client.bulk({ 206 | refresh: true, 207 | body: data.reduce((acc, current) => { 208 | return acc.concat([ 209 | { 210 | index: { 211 | _index: indexName, 212 | _type: '_doc' 213 | } 214 | }, 215 | current 216 | ]) 217 | }, []) 218 | }) 219 | 220 | const res = await update(name, {body}, reindexer) 221 | 222 | const docs = await es().search({ 223 | index: name, 224 | q: '*' 225 | }) 226 | 227 | // cleanup 228 | try { 229 | await es().indices.delete({index}) 230 | } catch (e) {} 231 | try { 232 | await es().indices.deleteAlias({name, index}) 233 | } catch (e) {} 234 | 235 | expect(docs.hits.total).toBe(data.length) 236 | 237 | const indexShouldMatch = new RegExp(`^${name}-\\d+-\\d+-\\d+t\\d+:\\d+:\\d+.\\d+z`) 238 | expect(res.name).toBe(name) 239 | expect(res.index).toMatch(indexShouldMatch) 240 | expect(res.sourceIndex).toMatch(indexShouldMatch) 241 | expect(res.ops).toHaveProperty('aliasExists', true) 242 | 243 | expect(res.ops).toHaveProperty('findAlias') 244 | expect(res.ops.findAlias[index]).toBeDefined() 245 | expect(res.ops.findAlias[index]).toHaveProperty('aliases') 246 | expect(res.ops.findAlias[index].aliases).toHaveProperty(name, {}) 247 | 248 | expect(res.ops).toHaveProperty('index') 249 | expect(res.ops.index).toHaveProperty('acknowledged', true) 250 | expect(res.ops.index).toHaveProperty('shards_acknowledged', true) 251 | expect(res.ops.index).toHaveProperty('index') 252 | expect(res.ops.index.index).toMatch(indexShouldMatch) 253 | 254 | expect(res.ops).toHaveProperty('postReindex') 255 | expect(res.ops.postReindex).toEqual(expect.any(Array)) 256 | expect(res.ops.postReindex.length).toBe(3) 257 | expect(res.ops.postReindex[0]).toHaveProperty('indexExists', true) 258 | expect(res.ops.postReindex[1]).toHaveProperty('indexExists', true) 259 | expect(res.ops.postReindex[2]).toHaveProperty('aliasExists', true) 260 | 261 | expect(res.ops).toHaveProperty('postAliasSwitch') 262 | expect(res.ops.postAliasSwitch).toEqual(expect.any(Array)) 263 | expect(res.ops.postAliasSwitch.length).toBe(2) 264 | 265 | expect(res.ops.postAliasSwitch[0]).toHaveProperty('indexDeletion', {acknowledged: true}) 266 | expect(res.ops.postAliasSwitch[1]).toHaveProperty('aliasExists', true) 267 | 268 | expect(res.ops).toHaveProperty('reindex') 269 | 270 | expect(res.ops).toHaveProperty('switchAlias') 271 | expect(res.ops.switchAlias).toHaveProperty('acknowledged', true) 272 | 273 | expect(res.ops.index.acknowledged).toBe(true) 274 | expect(res.ops.index.shards_acknowledged).toBe(true) 275 | expect(res.ops.index.index).toEqual(res.index) 276 | 277 | expect(res.ops).toHaveProperty('template') 278 | expect(res.ops.template).toHaveProperty('acknowledged', true) 279 | }) 280 | 281 | // nominal - can be useful for migrating the index version after an ES update 282 | it(`should allow updating an index with no template`, async () => { 283 | const res = await update(name) 284 | const indexShouldMatch = new RegExp(`^${name}-\\d+-\\d+-\\d+t\\d+:\\d+:\\d+.\\d+z`) 285 | 286 | expect(res.name).toBe(name) 287 | expect(res.index).toMatch(indexShouldMatch) 288 | expect(res.sourceIndex).toMatch(indexShouldMatch) 289 | expect(res.ops).toHaveProperty('aliasExists', true) 290 | 291 | expect(res.ops).toHaveProperty('findAlias') 292 | expect(res.ops.findAlias[res.sourceIndex]).toBeDefined() 293 | expect(res.ops.findAlias[res.sourceIndex]).toHaveProperty('aliases') 294 | expect(res.ops.findAlias[res.sourceIndex].aliases).toHaveProperty(name, {}) 295 | 296 | expect(res.ops).toHaveProperty('index') 297 | expect(res.ops.index).toHaveProperty('acknowledged', true) 298 | expect(res.ops.index).toHaveProperty('shards_acknowledged', true) 299 | expect(res.ops.index).toHaveProperty('index') 300 | expect(res.ops.index.index).toMatch(indexShouldMatch) 301 | 302 | expect(res.ops).toHaveProperty('postReindex') 303 | expect(res.ops.postReindex).toEqual(expect.any(Array)) 304 | expect(res.ops.postReindex.length).toBe(3) 305 | expect(res.ops.postReindex[0]).toHaveProperty('indexExists', true) 306 | expect(res.ops.postReindex[1]).toHaveProperty('indexExists', true) 307 | expect(res.ops.postReindex[2]).toHaveProperty('aliasExists', true) 308 | 309 | expect(res.ops).toHaveProperty('postAliasSwitch') 310 | expect(res.ops.postAliasSwitch).toEqual(expect.any(Array)) 311 | expect(res.ops.postAliasSwitch.length).toBe(2) 312 | expect(res.ops.postAliasSwitch[0]).toHaveProperty('indexDeletion', {acknowledged: true}) 313 | expect(res.ops.postAliasSwitch[1]).toHaveProperty('aliasExists', true) 314 | 315 | expect(res.ops).toHaveProperty('reindex') 316 | expect(res.ops.reindex).toHaveProperty('failures') 317 | expect(res.ops.reindex.failures).toHaveProperty('length', 0) 318 | expect(res.ops.reindex).toHaveProperty('total', 0) 319 | 320 | expect(res.ops).toHaveProperty('switchAlias') 321 | expect(res.ops.switchAlias).toHaveProperty('acknowledged', true) 322 | 323 | expect(res.ops.index.acknowledged).toBe(true) 324 | expect(res.ops.index.shards_acknowledged).toBe(true) 325 | expect(res.ops.index.index).toEqual(res.index) 326 | 327 | expect(res.ops).not.toHaveProperty('template') 328 | }) 329 | 330 | it(`should forbid updating the same index again in parallel`, async () => { 331 | expect.assertions(3) 332 | try { 333 | await Promise.all([ 334 | update(name), 335 | update(name), 336 | update(name), 337 | update(name), 338 | update(name), 339 | update(name), 340 | update(name), 341 | update(name) 342 | ]) 343 | } catch (error) { 344 | expect(error).toBeDefined() 345 | expect(error.output.statusCode).toBe(423) 346 | expect(error.message).toMatch(name) 347 | } 348 | }) 349 | }) 350 | 351 | describe('delete', () => { 352 | // nominal 353 | it(`should allow deleting an index and its alias`, async () => { 354 | const name = uuid() 355 | const index = suffix(name) 356 | await es().indices.create({index}) 357 | await es().indices.refresh({index}) 358 | await es().indices.putAlias({name, index}) 359 | 360 | const res = await del(name) 361 | expect(res.name).toBe(name) 362 | expect(res.index).toMatch(new RegExp(`^${name}-\\d+-\\d+-\\d+t\\d+:\\d+:\\d+.\\d+z`)) 363 | expect(res.ops).toHaveProperty('preChecks') 364 | expect(res.ops.preChecks).toEqual(expect.any(Array)) 365 | expect(res.ops.preChecks.length).toBe(2) 366 | expect(res.ops.preChecks[0]).toHaveProperty('aliasExists', true) 367 | expect(res.ops.preChecks[1]).toHaveProperty('indexExists', true) 368 | expect(res.ops).toHaveProperty('deletions') 369 | expect(res.ops.preChecks).toEqual(expect.any(Array)) 370 | expect(res.ops.deletions).toHaveProperty('length', 1) 371 | expect(res.ops.deletions[0]).toBeDefined() 372 | expect(res.ops.deletions[0]).toHaveProperty('indexDeletion') 373 | expect(res.ops.deletions[0].indexDeletion).toHaveProperty('acknowledged', true) 374 | }) 375 | 376 | // existing 377 | it(`should forbid deleting an unknown index`, async () => { 378 | const name = uuid() 379 | const index = suffix(name) 380 | await es().indices.create({index}) 381 | await es().indices.refresh({index}) 382 | await es().indices.putAlias({name, index}) 383 | 384 | expect.assertions(3) 385 | try { 386 | await del(`${name}-invalid`) 387 | } catch (error) { 388 | expect(error).toBeDefined() 389 | expect(error).toHaveProperty('statusCode', 404) 390 | expect(error.message).toMatch(name) 391 | } 392 | }) 393 | 394 | it(`should allow deleting an index but not the alias under a multi-indices alias`, async () => { 395 | const name = uuid() 396 | const index = suffix(name) 397 | const otherIndex = suffix(name, new Date(Math.random() * 1e14)) 398 | await es().indices.create({index}) 399 | await es().indices.refresh({index}) 400 | await es().indices.putAlias({name, index}) 401 | await es().indices.create({index: otherIndex}) 402 | await es().indices.refresh({index: otherIndex}) 403 | await es().indices.putAlias({name, index: otherIndex}) 404 | 405 | const res = await del(name) 406 | expect(res.name).toBe(name) 407 | expect(res.index).toMatch(new RegExp(`^${name}-\\d+-\\d+-\\d+t\\d+:\\d+:\\d+.\\d+z`)) 408 | expect(res.ops).toHaveProperty('preChecks') 409 | expect(res.ops.preChecks).toEqual(expect.any(Array)) 410 | expect(res.ops.preChecks.length).toBe(2) 411 | expect(res.ops.preChecks[0]).toHaveProperty('aliasExists', true) 412 | expect(res.ops.preChecks[1]).toHaveProperty('indexExists', true) 413 | expect(res.ops.deletions).toEqual(expect.any(Array)) 414 | expect(res.ops.deletions).toHaveProperty('length', 1) 415 | expect(res.ops.deletions[0]).toBeDefined() 416 | expect(res.ops.deletions[0]).toHaveProperty('indexDeletion') 417 | expect(res.ops.deletions[0].indexDeletion).toHaveProperty('acknowledged', true) 418 | }) 419 | 420 | it(`should forbid deleting the same index again in parallel`, async () => { 421 | const name = uuid() 422 | const index = suffix(name) 423 | await es().indices.create({index}) 424 | await es().indices.refresh({index}) 425 | await es().indices.putAlias({name, index}) 426 | 427 | expect.assertions(3) 428 | try { 429 | await Promise.all([ 430 | del(name), 431 | del(name), 432 | del(name), 433 | del(name), 434 | del(name), 435 | del(name), 436 | del(name), 437 | del(name) 438 | ]) 439 | } catch (error) { 440 | expect(error).toBeDefined() 441 | expect(error.output.statusCode).toBe(423) 442 | expect(error.message).toMatch(name) 443 | } 444 | }) 445 | }) 446 | -------------------------------------------------------------------------------- /lib/logger.js: -------------------------------------------------------------------------------- 1 | const pino = require('pino') 2 | 3 | const {req, res, err} = pino.stdSerializers 4 | const serializers = {req, res, err} 5 | 6 | exports._log = undefined 7 | 8 | exports.log = (logger) => { 9 | if (logger) { 10 | exports._log = logger 11 | } 12 | if (!exports._log) { 13 | exports._log = pino({ name: 'Leaistic', safe: true, level: 'debug', serializers }) 14 | } 15 | return exports._log 16 | } 17 | -------------------------------------------------------------------------------- /lib/logger.test.js: -------------------------------------------------------------------------------- 1 | const pino = require('pino') 2 | 3 | const { log } = require('./logger') 4 | 5 | describe('log', () => { 6 | // nominal - raw Error connection, no details 7 | it(`should return a default logger without params`, async () => { 8 | const logger = log() 9 | expect(logger).toBeDefined() 10 | expect(logger).toHaveProperty('trace') 11 | expect(logger).toHaveProperty('debug') 12 | expect(logger).toHaveProperty('info') 13 | expect(logger).toHaveProperty('warn') 14 | expect(logger).toHaveProperty('error') 15 | expect(logger).toHaveProperty('fatal') 16 | }) 17 | 18 | it(`should return my logger if I give it in param`, async () => { 19 | const myLogger = pino({ name: 'Leaistic Tests', safe: true, level: 'debug' }) 20 | const logger = log(myLogger) 21 | expect(logger).toBeDefined() 22 | expect(logger).toBe(myLogger) 23 | expect(logger).toHaveProperty('trace') 24 | expect(logger).toHaveProperty('debug') 25 | expect(logger).toHaveProperty('info') 26 | expect(logger).toHaveProperty('warn') 27 | expect(logger).toHaveProperty('error') 28 | expect(logger).toHaveProperty('fatal') 29 | }) 30 | 31 | it(`should define my logger as the default one`, async () => { 32 | const myLogger = pino({ name: 'Leaistic Tests', safe: true, level: 'debug' }) 33 | const logger = log(myLogger) 34 | expect(logger).toBeDefined() 35 | expect(logger).toBe(myLogger) 36 | const defaultLogger = log() 37 | expect(defaultLogger).toBeDefined() 38 | expect(defaultLogger).toBe(myLogger) 39 | }) 40 | }) 41 | -------------------------------------------------------------------------------- /lib/memoryStore.js: -------------------------------------------------------------------------------- 1 | const moment = require('moment') 2 | const Boom = require('boom') 3 | 4 | const {log} = require('./logger') 5 | 6 | const store = {} 7 | 8 | exports.memoryStore = { 9 | name: 'In Memory Store', 10 | 11 | save: async (operation, key, timeout) => { 12 | log().info({operation, key, timeout}, `🔒 Start '${operation}' on '${key}'`) 13 | if (!store[key]) { 14 | const date = new Date() 15 | 16 | store[key] = { 17 | operation, 18 | start: date, 19 | end: moment(date).add(timeout, 'milliseconds') 20 | } 21 | return store[key] 22 | } 23 | 24 | throw Boom.locked(`An Operation '${store[key].operation}' started ${moment(store[key].start).fromNow()} is already running for '${key}', you have to await for it to finish (it should be finished well before ${moment(store[key].end).fromNow()}) to do a '${operation}'`) 25 | }, 26 | 27 | delete: async (operation, key) => { 28 | log().info({operation, key}, `🔓 Finish '${operation}' on '${key}'`) 29 | 30 | if (!store[key] || store[key].operation !== operation) { 31 | log().debug({store, operation}, 'current store state') 32 | throw Boom.resourceGone(`No Operation '${operation}' exist (anymore) '${key}' ! Maybe you did set up a too short timeout ?`) 33 | } 34 | delete store[key] 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /lib/memoryStore.test.js: -------------------------------------------------------------------------------- 1 | const uuid = require('uuid/v1') 2 | const delay = require('delay') 3 | 4 | const { memoryStore } = require('./memoryStore') 5 | 6 | describe('memoryStore', () => { 7 | it('should have a name, and save/delete methods', () => { 8 | expect(memoryStore).toHaveProperty('name', expect.any(String)) 9 | expect(memoryStore).toHaveProperty('save', expect.any(Function)) 10 | expect(memoryStore).toHaveProperty('delete', expect.any(Function)) 11 | }) 12 | 13 | // nominal 14 | it('should be able to save/delete operation "test"', async () => { 15 | const key = uuid() 16 | expect.assertions(0) 17 | try { 18 | const {timeout} = await memoryStore.save('test', key, 10000) 19 | await Promise.race([ 20 | timeout, 21 | delay(100).then(() => memoryStore.delete('test', key)) 22 | ]) 23 | } catch (e) { 24 | expect(e).not.toBeDefined() 25 | } 26 | }) 27 | 28 | it('should not be able to delete an non existing operation "deletionOnlyTest"', async () => { 29 | const key = uuid() 30 | expect.assertions(3) 31 | try { 32 | await memoryStore.delete('deletionOnlyTest', key) 33 | } catch (e) { 34 | expect(e).toBeDefined() 35 | expect(e).toHaveProperty('isBoom', true) 36 | expect(e).toHaveProperty('output.statusCode', 410) // gone 37 | } 38 | }) 39 | }) 40 | -------------------------------------------------------------------------------- /lib/ops.js: -------------------------------------------------------------------------------- 1 | const Boom = require('boom') 2 | 3 | const {es, awaitRefresh} = require('./es') 4 | const {manageErrors} = require('./errors') 5 | const {indexName, indexTemplate, validate} = require('./validation') 6 | const {log} = require('./logger') 7 | 8 | exports.client = es 9 | 10 | exports.shouldUpdateTemplate = (template) => template ? (!!Object.keys(template).length) : false 11 | 12 | exports.checkAliasAlreadyExists = async (_name, ops = {}) => { 13 | await manageErrors(async () => { 14 | const name = validate(_name, indexName.label('alias name')) 15 | 16 | ops.aliasExists = await es().indices.existsAlias({name}) 17 | }, 18 | `Could not check if Alias "${_name}" already exists` 19 | ) 20 | if (!ops.aliasExists) { 21 | throw Boom.conflict(`An Alias "${_name}" does not exist yet, and it should!`) 22 | } 23 | log().debug({ops}, `✅ Already existing Alias '${_name}'`) 24 | 25 | return {ops} 26 | } 27 | 28 | exports.checkAliasDoesNotExists = async (_name, ops = {}) => { 29 | let name = _name 30 | await manageErrors(async () => { 31 | name = validate(_name, indexName.label('alias name')) 32 | ops.aliasExists = await es().indices.existsAlias({name}) 33 | }, `Could not check if Alias "${_name}" already exists`) 34 | 35 | if (ops.aliasExists) { 36 | throw Boom.conflict(`An Alias "${name}" already exists.`) 37 | } 38 | log().debug({ops}, `✅ No already existing Alias '${name}'`) 39 | 40 | return {ops} 41 | } 42 | 43 | exports.updateTemplate = async (_name, _template, ops = {}) => { 44 | await manageErrors(async () => { 45 | const name = validate(_name, indexName.label('index template name')) 46 | const template = validate(_template, indexTemplate(name).required().label('index template')) 47 | 48 | ops.template = await es().indices.putTemplate({name, body: template}) 49 | log().debug({ops}, `✅"${name}" Index Template creation done`) 50 | }, `Could not create "${_name}" Index Template`) 51 | 52 | return {ops} 53 | } 54 | 55 | exports.createIndex = async (_index, rollback, ops = {}) => { 56 | await manageErrors(async () => { 57 | const index = validate(_index, indexName.label('index name')) 58 | ops.index = await es().indices.create({index}) 59 | await awaitRefresh() 60 | log().debug({ops}, `✅"${index}" Index creation done`) 61 | }, 62 | `Could not create "${_index}" Index`, 63 | error => rollback(_index, error, `"${_index}" Index creation`) 64 | ) 65 | 66 | return {ops} 67 | } 68 | 69 | exports.deleteIndex = async (_index, rollback, ops = {}) => { 70 | await manageErrors(async () => { 71 | const index = validate(_index, indexName.label('index name')) 72 | ops.indexDeletion = await es().indices.delete({index}) 73 | await awaitRefresh() 74 | log().debug({ops}, `✅"${index}" Index deletion done`) 75 | }, 76 | `Could not delete "${_index}" Index`, 77 | error => rollback(_index, error, `"${_index}" Index deletion`) 78 | ) 79 | 80 | return {ops} 81 | } 82 | 83 | exports.checkIndexAlreadyExists = async (_index, ops = {}) => { 84 | await manageErrors(async () => { 85 | const index = validate(_index, indexName.label('index name')) 86 | ops.indexExists = await es().indices.exists({index}) 87 | if (!ops.indexExists) { 88 | throw Boom.conflict(`An Index "${index}" does not exist yet, and it should!`) 89 | } 90 | log().debug({ops}, `✅ Already existing Index '${index}'`) 91 | }, 92 | `Could not check if Index "${_index}" already exists`) 93 | 94 | return {ops} 95 | } 96 | 97 | exports.checkIndexDoesNotExist = async (_index, ops = {}) => { 98 | await manageErrors(async () => { 99 | const index = validate(_index, indexName.label('index name')) 100 | ops.indexExists = await es().indices.exists({index}) 101 | if (ops.indexExists) { 102 | throw Boom.conflict(`An Index "${index}" already exists, and it shouldn't!`) 103 | } 104 | log().debug({ops}, `✅ No existing Index '${index}'`) 105 | }, 106 | `Could not check if Index "${_index}" does not exist`) 107 | 108 | return {ops} 109 | } 110 | 111 | exports.reindex = async (_name, _sourceIndex, _index, rollback, ops = {}) => { 112 | await manageErrors(async () => { 113 | const sourceIndex = validate(_sourceIndex, indexName.label('source index name')) 114 | const index = validate(_index, indexName.label('alias destination index name')) 115 | 116 | log().debug({ops}, `✅"${sourceIndex}" Index will be used as source for reindexing "${index}"`) 117 | const body = {source: {index: sourceIndex}, dest: {index}} 118 | 119 | ops.reindex = await es().reindex({body}) 120 | log().debug({ops}, `✅"${sourceIndex}"👉"${index}" reindexation awaiting for refresh`) 121 | await awaitRefresh() 122 | log().debug({ops}, `✅"${sourceIndex}"👉"${index}" reindexation done`) 123 | }, 124 | `Could not create "${_name}" Index`, 125 | error => rollback(_name, _index, error, `"${_name}" Index reindexation`) 126 | ) 127 | 128 | return {ops} 129 | } 130 | 131 | exports.createAlias = async (_name, _index, rollback, ops = {}) => { 132 | await manageErrors(async () => { 133 | const name = validate(_name, indexName.label('alias name')) 134 | const index = validate(_index, indexName.label('index name')) 135 | 136 | // Used to be ops.alias = await es().indices.putAlias({index, name}), but it allows ES to have 2 137 | ops.alias = await es().indices.updateAliases({ 138 | body: { 139 | actions: [ 140 | { remove: { index: '_all', alias: name } }, 141 | { add: { index, alias: name } } 142 | ] 143 | } 144 | }) 145 | log().debug({ops}, `✅"${name}" Alias creation done`) 146 | }, 147 | `Could not create "${_name}" Alias`, 148 | error => rollback(_name, _index, error, `"${_name}" Alias creation`) 149 | ) 150 | 151 | return {ops} 152 | } 153 | 154 | exports.deleteAlias = async (_name, _index, rollback, ops = {}) => { 155 | await manageErrors(async () => { 156 | const name = validate(_name, indexName.label('alias name')) 157 | const index = validate(_index, indexName.allow('_all').label('index name')) 158 | 159 | ops.aliasDeletion = await es().indices.updateAliases({ 160 | body: { 161 | actions: [ 162 | { remove: { index, alias: name } } 163 | ] 164 | } 165 | }) 166 | log().debug({ops}, `✅"${name}" Alias deletion of index "${index}" done`) 167 | }, 168 | `Could not delete "${_index}" Index of "${_name}" Alias`, 169 | error => rollback(_name, _index, error, `"${_name}" Alias deletion of index "${_index}"`) 170 | ) 171 | 172 | return {ops} 173 | } 174 | 175 | exports.findAliasIndex = async (_name, ops = {}) => { 176 | return manageErrors(async () => { 177 | const name = validate(_name, indexName.label('alias name')) 178 | 179 | ops.findAlias = await es().indices.getAlias({name}) 180 | const sourceIndex = Object.keys(ops.findAlias)[0] 181 | log().debug({ops}, `✅"${name}" Alias source Index found`) 182 | return {sourceIndex, ops} 183 | }, 184 | `Could not find Alias "${_name}" nominal Index` 185 | ) 186 | } 187 | 188 | exports.switchAlias = async (_name, _sourceIndex, _destinationIndex, rollback, ops = {}) => { 189 | await manageErrors(async () => { 190 | const name = validate(_name, indexName.label('alias name')) 191 | const sourceIndex = validate(_sourceIndex, indexName.label('source index name')) 192 | const destinationIndex = validate(_destinationIndex, indexName.label('destination index name')) 193 | 194 | ops.switchAlias = await es().indices.updateAliases({ 195 | body: { 196 | actions: [ 197 | { remove: { index: sourceIndex, alias: name } }, 198 | { add: { index: destinationIndex, alias: name } } 199 | ] 200 | } 201 | }) 202 | log().debug({ops}, `✅"${name}" Alias switched from "${sourceIndex}" to "${destinationIndex}"`) 203 | }, 204 | `Could not switch "${_name}" Alias`, 205 | error => rollback(_name, _sourceIndex, _destinationIndex, error, `"${_name}" Alias switch from "${_sourceIndex}" to "${_destinationIndex}"`) 206 | ) 207 | 208 | return {ops} 209 | } 210 | -------------------------------------------------------------------------------- /lib/ops.test.js: -------------------------------------------------------------------------------- 1 | const uuid = require('uuid/v1') 2 | const { 3 | es, 4 | esError, 5 | awaitRefresh 6 | } = require('./es') 7 | const {log} = require('./logger') 8 | const { 9 | shouldUpdateTemplate, 10 | checkAliasDoesNotExists, 11 | updateTemplate, 12 | createIndex, 13 | deleteIndex, 14 | checkIndexAlreadyExists, 15 | checkIndexDoesNotExist, 16 | reindex, 17 | createAlias, 18 | deleteAlias, 19 | findAliasIndex, 20 | switchAlias, 21 | checkAliasAlreadyExists 22 | } = require('./ops') 23 | 24 | const {suffix} = require('./indices') 25 | 26 | jest.setTimeout(60000) 27 | 28 | describe('shouldUpdateTemplate', () => { 29 | const name = uuid() 30 | 31 | // nominal 32 | it(`should be truthy for a non empty object`, () => { 33 | expect(shouldUpdateTemplate({ 34 | 'index_patterns': [`${name}-*`], 35 | 'settings': { 36 | 'number_of_shards': 1 37 | }, 38 | 'aliases': { 39 | 'alias1': {}, 40 | '{index}-alias': {} 41 | } 42 | })).toBeTruthy() 43 | }) 44 | 45 | // usual alternative 46 | it(`should be falsy for an object`, () => { 47 | expect(shouldUpdateTemplate({})).toBeFalsy() 48 | }) 49 | 50 | // common pitfall 51 | it(`should be falsy for undefined`, () => { 52 | expect(shouldUpdateTemplate()).toBeFalsy() 53 | }) 54 | }) 55 | 56 | describe('checkAliasDoesNotExists', () => { 57 | const name = uuid() 58 | const index = suffix(name) 59 | 60 | // nominal 61 | it(`should check true for an alias that does not exist`, async () => { 62 | const {ops} = await checkAliasDoesNotExists(name) 63 | expect(ops).toHaveProperty('aliasExists', false) 64 | }) 65 | 66 | // typical issue 67 | it(`should error for an existing alias`, async () => { 68 | expect.assertions(3) 69 | await es().indices.create({index}) 70 | await es().indices.refresh({index}) 71 | await es().indices.putAlias({name, index}) 72 | try { 73 | const {ops} = await checkAliasDoesNotExists(name) 74 | expect(ops).not.toBeDefined() 75 | } catch (error) { 76 | expect(error).toBeDefined() 77 | expect(error).toHaveProperty('isBoom', true) 78 | expect(error).toHaveProperty('output.statusCode', 409) // Conflict 79 | } 80 | }) 81 | 82 | it(`should error for an anonymous alias`, async () => { 83 | expect.assertions(4) 84 | try { 85 | const {ops} = await checkAliasDoesNotExists(undefined) 86 | expect(ops).not.toBeDefined() 87 | } catch (error) { 88 | expect(error).toBeDefined() 89 | expect(error).toHaveProperty('isJoi', true) 90 | expect(error).toHaveProperty('name', 'ValidationError') 91 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 92 | } 93 | }) 94 | }) 95 | 96 | describe('updateTemplate', () => { 97 | const name = uuid() 98 | const template = { 99 | 'index_patterns': [`${name}-*`], 100 | 'settings': { 101 | 'number_of_shards': 1 102 | }, 103 | 'aliases': { 104 | 'alias1': {}, 105 | '{index}-alias': {} 106 | } 107 | } 108 | 109 | // nominal, no check is done by this method 110 | it(`should successfully deploy a template`, async () => { 111 | const {ops} = await updateTemplate(name, template) 112 | expect(ops).toBeDefined() 113 | expect(ops).toHaveProperty('template.acknowledged', true) 114 | }) 115 | 116 | it(`should unsuccessfully deploy an undefined template`, async () => { 117 | try { 118 | const {ops} = await updateTemplate(name, undefined) 119 | expect(ops).not.toBeDefined() 120 | } catch (error) { 121 | expect(error).toBeDefined() 122 | expect(error).toHaveProperty('isJoi', true) 123 | expect(error).toHaveProperty('name', 'ValidationError') 124 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 125 | } 126 | }) 127 | 128 | it(`should unsuccessfully deploy an anonymous template`, async () => { 129 | expect.assertions(4) 130 | try { 131 | const {ops} = await updateTemplate(undefined, template) 132 | expect(ops).not.toBeDefined() 133 | } catch (error) { 134 | expect(error).toBeDefined() 135 | expect(error).toHaveProperty('isJoi', true) 136 | expect(error).toHaveProperty('name', 'ValidationError') 137 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 138 | } 139 | }) 140 | }) 141 | 142 | describe('createIndex', () => { 143 | const name = uuid() 144 | const index = suffix(name) 145 | 146 | // nominal 147 | it(`should allow creating a new index`, async () => { 148 | const rollback = jest.fn() 149 | const {ops} = await createIndex(index, rollback, {}) 150 | expect(rollback).not.toHaveBeenCalled() 151 | expect(ops).toHaveProperty('index.acknowledged', true) 152 | expect(ops).toHaveProperty('index.shards_acknowledged', true) 153 | expect(ops).toHaveProperty('index.index', index) 154 | }) 155 | 156 | // typical issue 157 | it(`should rollback and error when trying to create an existing index`, async () => { 158 | const index = suffix(uuid()) 159 | const rollback = jest.fn() 160 | await es().indices.create({index}) 161 | await es().indices.refresh({index}) 162 | expect.assertions(4) 163 | try { 164 | await createIndex(index, rollback, {}) 165 | } catch (error) { 166 | expect(rollback).toHaveBeenCalled() 167 | expect(error).toBeDefined() 168 | expect(error).toHaveProperty('isBoom', true) 169 | expect(error).toHaveProperty('output.statusCode', 409) // Conflict 170 | } 171 | }) 172 | 173 | it(`should not allow creating an anonymous index`, async () => { 174 | const rollback = jest.fn() 175 | expect.assertions(5) 176 | try { 177 | const {ops} = await createIndex(undefined, rollback, {}) 178 | expect(ops).not.toBeDefined() 179 | } catch (error) { 180 | expect(rollback).toHaveBeenCalled() 181 | expect(error).toBeDefined() 182 | expect(error).toHaveProperty('isJoi', true) 183 | expect(error).toHaveProperty('name', 'ValidationError') 184 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 185 | } 186 | }) 187 | }) 188 | 189 | describe('deleteIndex', () => { 190 | const name = uuid() 191 | const index = suffix(name) 192 | 193 | // nominal 194 | it(`should allow deleting an existing index`, async () => { 195 | await es().indices.create({index}) 196 | await es().indices.refresh({index}) 197 | const rollback = jest.fn() 198 | const {ops} = await deleteIndex(index, rollback, {}) 199 | expect(rollback).not.toHaveBeenCalled() 200 | expect(ops).toHaveProperty('indexDeletion.acknowledged', true) 201 | }) 202 | 203 | // typical issue 204 | it(`should rollback and error when trying to delete an index that does not exist`, async () => { 205 | const rollback = jest.fn() 206 | expect.assertions(4) 207 | try { 208 | const {ops} = await deleteIndex(index, rollback, {}) 209 | expect(ops).not.toBeDefined() 210 | } catch (error) { 211 | expect(rollback).toHaveBeenCalled() 212 | expect(error).toBeDefined() 213 | expect(error).toHaveProperty('statusCode', 404) // Unknown 214 | expect(error.message).toMatch(name) 215 | } 216 | }) 217 | 218 | it(`should not allow deleting an anonymous index`, async () => { 219 | const rollback = jest.fn() 220 | expect.assertions(5) 221 | try { 222 | const {ops} = await deleteIndex(undefined, rollback, {}) 223 | expect(ops).not.toBeDefined() 224 | } catch (error) { 225 | expect(rollback).toHaveBeenCalled() 226 | expect(error).toBeDefined() 227 | expect(error).toHaveProperty('isJoi', true) 228 | expect(error).toHaveProperty('name', 'ValidationError') 229 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 230 | } 231 | }) 232 | }) 233 | 234 | describe('checkIndexAlreadyExists', () => { 235 | const name = uuid() 236 | const index = suffix(name) 237 | 238 | // nominal 239 | it(`should be true for an existing index`, async () => { 240 | await es().indices.create({index}) 241 | await es().indices.refresh({index}) 242 | const {ops} = await checkIndexAlreadyExists(index) 243 | expect(ops).toHaveProperty('indexExists', true) 244 | }) 245 | 246 | // typical issue 247 | it(`should error for an index that does not exist`, async () => { 248 | expect.assertions(3) 249 | const index = suffix(uuid()) 250 | try { 251 | const {ops} = await checkIndexAlreadyExists(index) 252 | expect(ops).not.toBeDefined() 253 | } catch (error) { 254 | expect(error).toBeDefined() 255 | expect(error).toHaveProperty('isBoom', true) 256 | expect(error).toHaveProperty('output.statusCode', 409) // Conflict 257 | } 258 | }) 259 | 260 | it(`should error for an anonymous index`, async () => { 261 | expect.assertions(4) 262 | try { 263 | const {ops} = await checkIndexAlreadyExists(undefined) 264 | expect(ops).not.toBeDefined() 265 | } catch (error) { 266 | expect(error).toBeDefined() 267 | expect(error).toHaveProperty('isJoi', true) 268 | expect(error).toHaveProperty('name', 'ValidationError') 269 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 270 | } 271 | }) 272 | }) 273 | 274 | describe('checkIndexDoesNotExist', () => { 275 | const name = uuid() 276 | const index = suffix(name) 277 | 278 | // nominal 279 | it(`should be true for an index that does not exist`, async () => { 280 | const index = suffix(uuid()) 281 | const {ops} = await checkIndexDoesNotExist(index) 282 | expect(ops).toHaveProperty('indexExists', false) 283 | }) 284 | 285 | // typical issue 286 | it(`should error for an index that already exists`, async () => { 287 | await es().indices.create({index}) 288 | await es().indices.refresh({index}) 289 | expect.assertions(3) 290 | 291 | try { 292 | const {ops} = await checkIndexDoesNotExist(index) 293 | expect(ops).not.toBeDefined() 294 | } catch (error) { 295 | expect(error).toBeDefined() 296 | expect(error).toHaveProperty('isBoom', true) 297 | expect(error).toHaveProperty('output.statusCode', 409) // Conflict 298 | } 299 | }) 300 | 301 | it(`should error for an anonymous index`, async () => { 302 | expect.assertions(4) 303 | try { 304 | const {ops} = await checkIndexDoesNotExist(undefined) 305 | expect(ops).not.toBeDefined() 306 | } catch (error) { 307 | expect(error).toBeDefined() 308 | expect(error).toHaveProperty('isJoi', true) 309 | expect(error).toHaveProperty('name', 'ValidationError') 310 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 311 | } 312 | }) 313 | }) 314 | 315 | describe('reindex', () => { 316 | const name = uuid() 317 | const sourceIndex = suffix(name) 318 | const index = `${name}-2` 319 | 320 | // nominal 321 | it(`should allow reindexing an existing index`, async () => { 322 | await es().indices.create({index: sourceIndex}) 323 | await es().indices.refresh({index: sourceIndex}) 324 | await es().indices.putAlias({name, index: sourceIndex}) 325 | const rollback = jest.fn() 326 | const {ops} = await reindex(name, sourceIndex, index, rollback, {}) 327 | expect(rollback).not.toHaveBeenCalled() 328 | expect(ops).toHaveProperty('reindex') 329 | expect(ops.reindex.failures).toEqual([]) 330 | expect(ops.reindex.timed_out).toBeFalsy() 331 | }) 332 | 333 | // typical issue 334 | it(`should rollback and error when trying to reindex an unknown source index`, async () => { 335 | const name = uuid() 336 | const sourceIndex = `${name}-1` 337 | const index = `${name}-2` 338 | const rollback = jest.fn() 339 | expect.assertions(4) 340 | try { 341 | const {ops} = await reindex(name, sourceIndex, index, rollback, {}) 342 | expect(ops).not.toBeDefined() 343 | } catch (error) { 344 | expect(rollback).toHaveBeenCalled() 345 | expect(error).toBeDefined() 346 | expect(error.statusCode).toBe(404) 347 | expect(error.message).toMatch(name) 348 | } 349 | }) 350 | 351 | it(`should not allow reindex from an anonymous index`, async () => { 352 | const rollback = jest.fn() 353 | expect.assertions(5) 354 | try { 355 | const {ops} = await reindex(name, undefined, index, rollback, {}) 356 | expect(ops).not.toBeDefined() 357 | } catch (error) { 358 | expect(rollback).toHaveBeenCalled() 359 | expect(error).toBeDefined() 360 | expect(error).toHaveProperty('isJoi', true) 361 | expect(error).toHaveProperty('name', 'ValidationError') 362 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 363 | } 364 | }) 365 | 366 | it(`should not allow reindex to an anonymous index`, async () => { 367 | const rollback = jest.fn() 368 | expect.assertions(5) 369 | try { 370 | const {ops} = await reindex(name, sourceIndex, undefined, rollback, {}) 371 | expect(ops).not.toBeDefined() 372 | } catch (error) { 373 | expect(rollback).toHaveBeenCalled() 374 | expect(error).toBeDefined() 375 | expect(error).toHaveProperty('isJoi', true) 376 | expect(error).toHaveProperty('name', 'ValidationError') 377 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 378 | } 379 | }) 380 | }) 381 | 382 | describe('createAlias', () => { 383 | const name = uuid() 384 | const index = suffix(name) 385 | 386 | beforeAll(async () => { 387 | await es().indices.create({index}) 388 | await es().indices.refresh({index}) 389 | }) 390 | 391 | // nominal 392 | it(`should allow creating a new alias`, async () => { 393 | const rollback = jest.fn() 394 | const {ops} = await createAlias(name, index, rollback, {}) 395 | expect(rollback).not.toHaveBeenCalled() 396 | expect(ops).toHaveProperty('alias.acknowledged', true) 397 | }) 398 | 399 | // still nominal : ES allows it 400 | it(`should allow recreating the same alias`, async () => { 401 | await es().indices.putAlias({name, index}) 402 | const rollback = jest.fn() 403 | const {ops} = await createAlias(name, index, rollback, {}) 404 | 405 | expect(rollback).not.toHaveBeenCalled() 406 | expect(ops).toHaveProperty('alias') 407 | expect(ops.alias.acknowledged).toBe(true) 408 | const res = await es().indices.getAlias({name}) 409 | expect(res).toBeDefined() 410 | expect(res[index]).toBeDefined() 411 | expect(res[index]).toHaveProperty(`aliases.${name}`) 412 | }) 413 | 414 | // still nominal: ES allows it, this implementation however remove anything else 415 | it(`should allow pointing the same alias to a new index`, async () => { 416 | const index = suffix(name) 417 | await es().indices.create({index}) 418 | await es().indices.refresh({index}) 419 | 420 | const rollback = jest.fn() 421 | const {ops} = await createAlias(name, index, rollback, {}) 422 | expect(rollback).not.toHaveBeenCalled() 423 | expect(ops).toHaveProperty('alias') 424 | expect(ops.alias.acknowledged).toBe(true) 425 | const res = await es().indices.getAlias({name}) 426 | expect(res).toBeDefined() 427 | expect(res[index]).toBeDefined() 428 | expect(res[index]).toHaveProperty(`aliases.${name}`) 429 | }) 430 | 431 | it(`should not allow creating an anonymous alias`, async () => { 432 | const rollback = jest.fn() 433 | expect.assertions(5) 434 | try { 435 | const {ops} = await createAlias(undefined, index, rollback, {}) 436 | expect(ops).not.toBeDefined() 437 | } catch (error) { 438 | expect(rollback).toHaveBeenCalled() 439 | expect(error).toBeDefined() 440 | expect(error).toHaveProperty('isJoi', true) 441 | expect(error).toHaveProperty('name', 'ValidationError') 442 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 443 | } 444 | }) 445 | 446 | it(`should not allow creating an alias for an anonymous index`, async () => { 447 | const rollback = jest.fn() 448 | expect.assertions(5) 449 | try { 450 | const {ops} = await createAlias(name, undefined, rollback, {}) 451 | expect(ops).not.toBeDefined() 452 | } catch (error) { 453 | expect(rollback).toHaveBeenCalled() 454 | expect(error).toBeDefined() 455 | expect(error).toHaveProperty('isJoi', true) 456 | expect(error).toHaveProperty('name', 'ValidationError') 457 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 458 | } 459 | }) 460 | }) 461 | 462 | describe('deleteAlias', () => { 463 | const name = uuid() 464 | const index = suffix(name) 465 | const otherIndex = suffix(name, new Date(Math.random() * 1e14)) 466 | 467 | const ensureIndexCreation = (index) => { 468 | try { 469 | return es().indices.create({index}) 470 | } catch (e) { 471 | if (e && e.statusCode === 404) { 472 | return Promise.resolve() 473 | } 474 | throw e 475 | } 476 | } 477 | 478 | beforeEach(async () => { 479 | try { 480 | await Promise.all([ 481 | ensureIndexCreation(index).then(() => es().indices.refresh({index})), 482 | ensureIndexCreation(otherIndex).then(() => es().indices.refresh({index: otherIndex})) 483 | ]) 484 | } catch (e) { 485 | log().debug({err: e}, '🐞 beforeEach deleteAlias') 486 | } finally { 487 | try { 488 | await es().indices.putAlias({name, index}) 489 | await awaitRefresh() 490 | } catch (e) { 491 | log().debug({err: e}, '🐞 beforeEach deleteAlias putAlias') 492 | } 493 | } 494 | }) 495 | 496 | const ensureIndexDeletion = (name) => { 497 | try { 498 | return es().indices.deleteAlias({name, index: '_all'}) 499 | } catch (e) { 500 | if (e && e.statusCode === 404) { 501 | return Promise.resolve() 502 | } 503 | throw e 504 | } 505 | } 506 | 507 | const ensureAliasDeletion = (index) => { 508 | try { 509 | return es().indices.delete({index}) 510 | } catch (e) { 511 | if (e && e.statusCode === 404) { 512 | return Promise.resolve() 513 | } 514 | throw e 515 | } 516 | } 517 | 518 | afterEach(async () => { 519 | try { 520 | await Promise.all([ 521 | await ensureIndexDeletion(index), 522 | await ensureIndexDeletion(otherIndex) 523 | ]) 524 | } catch (e) { 525 | log().debug({err: e}, '🐞 afterEach deleteAlias') 526 | } finally { 527 | try { 528 | await ensureAliasDeletion(name) 529 | await awaitRefresh() 530 | } catch (e) { 531 | log().debug({err: e}, '🐞 afterEach deleteAlias ensureAliasDeletion') 532 | } 533 | } 534 | }) 535 | 536 | // nominal 537 | it(`should allow deleting an index in an existing alias`, async () => { 538 | expect.assertions(6) 539 | const rollback = jest.fn() 540 | const {ops} = await deleteAlias(name, index, rollback, {}) 541 | expect(rollback).not.toHaveBeenCalled() 542 | expect(ops).toHaveProperty('aliasDeletion') 543 | expect(ops.aliasDeletion.acknowledged).toBe(true) 544 | try { 545 | await es().indices.getAlias({name}) 546 | } catch (e) { 547 | expect(e).toBeDefined() 548 | expect(e instanceof esError).toBeTruthy() 549 | expect(e.statusCode).toBe(404) 550 | } 551 | }) 552 | 553 | it(`should allow deleting an index in an existing alias refering multiple aliases`, async () => { 554 | await es().indices.putAlias({name, index: otherIndex}) 555 | await awaitRefresh() 556 | const rollback = jest.fn() 557 | const {ops} = await deleteAlias(name, index, rollback, {}) 558 | expect(rollback).not.toHaveBeenCalled() 559 | expect(ops).toHaveProperty('aliasDeletion') 560 | expect(ops.aliasDeletion.acknowledged).toBe(true) 561 | const res = await es().indices.getAlias({name}) 562 | expect(res).toBeDefined() 563 | expect(res[otherIndex]).toBeDefined() 564 | expect(res[otherIndex]).toHaveProperty('aliases') 565 | expect(res[otherIndex].aliases).toHaveProperty(name) 566 | }) 567 | 568 | it(`should allow deleting all indices in an existing alias`, async () => { 569 | expect.assertions(6) 570 | await es().indices.putAlias({name, index: otherIndex}) 571 | const rollback = jest.fn() 572 | const {ops} = await deleteAlias(name, '_all', rollback, {}) 573 | expect(rollback).not.toHaveBeenCalled() 574 | expect(ops).toHaveProperty('aliasDeletion') 575 | expect(ops.aliasDeletion.acknowledged).toBe(true) 576 | try { 577 | await es().indices.getAlias({name}) 578 | } catch (e) { 579 | expect(e).toBeDefined() 580 | expect(e instanceof esError).toBeTruthy() 581 | expect(e.statusCode).toBe(404) 582 | } 583 | }) 584 | 585 | it(`should not allow deleting an anonymous alias`, async () => { 586 | const rollback = jest.fn() 587 | expect.assertions(5) 588 | try { 589 | const {ops} = await deleteAlias(undefined, index, rollback, {}) 590 | expect(ops).not.toBeDefined() 591 | } catch (error) { 592 | expect(rollback).toHaveBeenCalled() 593 | expect(error).toBeDefined() 594 | expect(error).toHaveProperty('isJoi') 595 | expect(error.isJoi).toBeTruthy() 596 | expect(error.name).toBe('ValidationError') 597 | } 598 | }) 599 | 600 | it(`should not allow deleting an alias from an anonymous index`, async () => { 601 | const rollback = jest.fn() 602 | expect.assertions(5) 603 | try { 604 | const {ops} = await deleteAlias(name, undefined, rollback, {}) 605 | expect(ops).not.toBeDefined() 606 | } catch (error) { 607 | expect(rollback).toHaveBeenCalled() 608 | expect(error).toBeDefined() 609 | expect(error).toHaveProperty('isJoi', true) 610 | expect(error).toHaveProperty('name', 'ValidationError') 611 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 612 | } 613 | }) 614 | }) 615 | 616 | describe('findAliasIndex', () => { 617 | const name = uuid() 618 | const index = suffix(name) 619 | 620 | // nominal 621 | it(`should be able to find the correct index`, async () => { 622 | await es().indices.create({index}) 623 | await es().indices.refresh({index}) 624 | await es().indices.putAlias({name, index}) 625 | const ops = {} 626 | const res = await findAliasIndex(name, ops) 627 | expect(ops).toHaveProperty('findAlias') 628 | expect(ops.findAlias).toBeTruthy() 629 | expect(ops.findAlias[index]).toBeTruthy() 630 | expect(ops.findAlias[index]).toHaveProperty('aliases') 631 | expect(ops.findAlias[index].aliases).toHaveProperty(name) 632 | expect(res).toBeTruthy() 633 | }) 634 | 635 | // typical issue 636 | it(`should error for an alias that does not exists`, async () => { 637 | expect.assertions(3) 638 | const name = uuid() 639 | try { 640 | const {ops} = await findAliasIndex(name) 641 | expect(ops).not.toBeDefined() 642 | } catch (error) { 643 | expect(error).toBeDefined() 644 | expect(error).toHaveProperty('statusCode') 645 | expect(error.statusCode).toBe(404) 646 | } 647 | }) 648 | 649 | it(`should error for an anonymous alias`, async () => { 650 | expect.assertions(4) 651 | try { 652 | const {ops} = await findAliasIndex(undefined) 653 | expect(ops).not.toBeDefined() 654 | } catch (error) { 655 | expect(error).toBeDefined() 656 | expect(error).toHaveProperty('isJoi', true) 657 | expect(error).toHaveProperty('name', 'ValidationError') 658 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 659 | } 660 | }) 661 | }) 662 | 663 | describe('switchAlias', () => { 664 | const name = uuid() 665 | const index = suffix(name) 666 | const destinationIndex = suffix(name) 667 | 668 | beforeAll(async () => { 669 | await es().indices.create({index}) 670 | await es().indices.refresh({index}) 671 | await es().indices.putAlias({name, index}) 672 | }) 673 | 674 | // nominal 675 | it(`should allow switching an existing alias`, async () => { 676 | const rollback = jest.fn() 677 | const {ops} = await switchAlias(name, index, destinationIndex, rollback, {}) 678 | expect(rollback).not.toHaveBeenCalled() 679 | expect(ops).toHaveProperty('switchAlias') 680 | expect(ops.switchAlias.acknowledged).toBe(true) 681 | const res = await es().indices.getAlias({name}) 682 | expect(res).toBeDefined() 683 | expect(res[index]).toBeDefined() 684 | expect(res[index]).toHaveProperty('aliases') 685 | expect(res[index].aliases).toHaveProperty(name) 686 | }) 687 | 688 | // still nominal : ES allows it 689 | it(`should allow switching to the same index`, async () => { 690 | const rollback = jest.fn() 691 | const {ops} = await switchAlias(name, index, index, rollback, {}) 692 | 693 | expect(rollback).not.toHaveBeenCalled() 694 | expect(ops).toHaveProperty('switchAlias') 695 | expect(ops.switchAlias.acknowledged).toBe(true) 696 | const res = await es().indices.getAlias({name}) 697 | expect(res).toBeDefined() 698 | expect(res[index]).toBeDefined() 699 | expect(res[index]).toHaveProperty('aliases') 700 | expect(res[index].aliases).toHaveProperty(name) 701 | }) 702 | 703 | it(`should not allow switching an anonymous alias`, async () => { 704 | const rollback = jest.fn() 705 | expect.assertions(5) 706 | try { 707 | const {ops} = await switchAlias(undefined, index, destinationIndex, rollback, {}) 708 | expect(ops).not.toBeDefined() 709 | } catch (error) { 710 | expect(rollback).toHaveBeenCalled() 711 | expect(error).toBeDefined() 712 | expect(error).toHaveProperty('isJoi', true) 713 | expect(error).toHaveProperty('name', 'ValidationError') 714 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 715 | } 716 | }) 717 | 718 | it(`should not allow switching an alias from an anonymous index`, async () => { 719 | const rollback = jest.fn() 720 | expect.assertions(5) 721 | try { 722 | const {ops} = await switchAlias(undefined, index, destinationIndex, rollback, {}) 723 | expect(ops).not.toBeDefined() 724 | } catch (error) { 725 | expect(rollback).toHaveBeenCalled() 726 | expect(error).toBeDefined() 727 | expect(error).toHaveProperty('isJoi', true) 728 | expect(error).toHaveProperty('name', 'ValidationError') 729 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 730 | } 731 | }) 732 | 733 | it(`should not allow creating an alias to an anonymous index`, async () => { 734 | const rollback = jest.fn() 735 | expect.assertions(5) 736 | try { 737 | const {ops} = await switchAlias(undefined, index, destinationIndex, rollback, {}) 738 | expect(ops).not.toBeDefined() 739 | } catch (error) { 740 | expect(rollback).toHaveBeenCalled() 741 | expect(error).toBeDefined() 742 | expect(error).toHaveProperty('isJoi', true) 743 | expect(error).toHaveProperty('name', 'ValidationError') 744 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 745 | } 746 | }) 747 | }) 748 | 749 | describe('checkAliasAlreadyExists', () => { 750 | const name = uuid() 751 | const index = suffix(name) 752 | 753 | // nominal 754 | it(`should be true for an existing alias`, async () => { 755 | await es().indices.create({index}) 756 | await es().indices.refresh({index}) 757 | await es().indices.putAlias({name, index}) 758 | const {ops} = await checkAliasAlreadyExists(name) 759 | expect(ops).toHaveProperty('aliasExists') 760 | expect(ops.aliasExists).toBeTruthy() 761 | }) 762 | 763 | // typical issue 764 | it(`should error for an alias that does not exists`, async () => { 765 | expect.assertions(5) 766 | const name = uuid() 767 | try { 768 | const {ops} = await checkAliasAlreadyExists(name) 769 | expect(ops).not.toBeDefined() 770 | } catch (error) { 771 | expect(error).toBeDefined() 772 | expect(error.isBoom).toBeTruthy() 773 | expect(error).toHaveProperty('output') 774 | expect(error.output).toHaveProperty('statusCode') 775 | expect(error.output.statusCode).toBe(409) 776 | } 777 | }) 778 | 779 | it(`should error for an anonymous alias`, async () => { 780 | expect.assertions(4) 781 | try { 782 | const {ops} = await checkAliasAlreadyExists(undefined) 783 | expect(ops).not.toBeDefined() 784 | } catch (error) { 785 | expect(error).toBeDefined() 786 | expect(error).toHaveProperty('isJoi', true) 787 | expect(error).toHaveProperty('name', 'ValidationError') 788 | expect(error).toHaveProperty('output.statusCode', 400) // Bad Request 789 | } 790 | }) 791 | }) 792 | -------------------------------------------------------------------------------- /lib/rollbacks.js: -------------------------------------------------------------------------------- 1 | const {es} = require('./es') 2 | const {log} = require('./logger') 3 | 4 | exports.rollbackAliasCreation = async ({name, index}, causeError, originDescription) => { 5 | log().warn({err: causeError}, `🚨 Rollback "${name}" Alias creation after error`) 6 | try { 7 | const op = await es().indices.deleteAlias({name, index}) 8 | log().debug(`⏪ Alias Creation Rollback: "${name}" Alias has been deleted ✅`) 9 | return op 10 | } catch (deletionError) { 11 | deletionError.message = `Could not rollback "${name}" Alias creation after an error during ${originDescription}: ` + deletionError.message 12 | deletionError.cause = causeError 13 | log().info({err: deletionError}, `Could not delete "${name}" Alias during rollback:`) 14 | throw deletionError 15 | } 16 | } 17 | 18 | exports.rollbackAliasSwitch = async ({name, sourceIndex, index}, causeError, originDescription) => { 19 | log().warn({err: causeError}, `🚨 Rollback "${name}" Alias switch after error`) 20 | try { 21 | try { 22 | const op = await es().indices.updateAliases({ 23 | body: { 24 | actions: [ 25 | { remove: { index, alias: name } }, 26 | { add: { index: sourceIndex, alias: name } } 27 | ] 28 | } 29 | }) 30 | log().debug(`⏪ Alias Switch Rollback: "${name}" Alias has been switched back to "${sourceIndex}" from "${index}" ✅`) 31 | return op 32 | } catch (error) { 33 | if (error.statusCode === 404 && error.body && error.body.error && error.body.error.type === 'index_not_found_exception' && error.body.error['resource.id'] === index) { 34 | log().warn(`Could not rollback delete "${name}" Alias reference to "${index}" Index because "${index} it did not exist, will just change back the Alias to ${sourceIndex} index" ✅`) 35 | const op = await es().indices.updateAliases({ 36 | body: { 37 | actions: [ 38 | { add: { index: sourceIndex, alias: name } } 39 | ] 40 | } 41 | }) 42 | log().debug(`⏪ Alias Switch Rollback: "${name}" Alias has been switched back to "${sourceIndex}" from "${index}" (but "${index}"" did not exist anymore) ✅`) 43 | return op 44 | } 45 | throw error 46 | } 47 | } catch (switchError) { 48 | switchError.message = `Could not rollback "${name}" Alias switch after an error during ${originDescription}: ` + switchError.message 49 | switchError.cause = causeError 50 | log().info({err: switchError}, `Could not switch back "${name}" Alias during rollback:`) 51 | throw switchError 52 | } 53 | } 54 | 55 | exports.rollbackIndexTemplateCreation = async ({name}, causeError, originDescription) => { 56 | log().warn({err: causeError}, `🚨 Rollback "${name}" Index Template creation after error`) 57 | try { 58 | const op = await es().indices.deleteTemplate({name}) 59 | log().debug(`⏪ Index Template Creation Rollback: "${name}" has been deleted ✅`) 60 | return op 61 | } catch (deletionError) { 62 | deletionError.message = `Could not rollback "${name}" Index Template creation after an error during ${originDescription}: ` + deletionError.message 63 | deletionError.cause = causeError 64 | log().info({err: deletionError}, `Could not delete "${name}" Index Template during rollback:`) 65 | throw deletionError 66 | } 67 | } 68 | 69 | exports.rollbackIndexCreation = async ({index}, causeError, originDescription) => { 70 | try { 71 | if (causeError.statusCode === 400 && causeError.message.startsWith('[resource_already_exists_exception]')) { 72 | log().info({err: causeError}, `Won't rollback '${index}' Index creation, given it seems that it was already existing.`) 73 | throw causeError 74 | } else { 75 | log().warn({err: causeError}, `🚨 Rollback "${index}" Index creation after error`) 76 | } 77 | const op = await es().indices.delete({index}) 78 | log().debug(`⏪ Index Creation Rollback: "${index}" has been deleted ✅`) 79 | return op 80 | } catch (deletionError) { 81 | deletionError.message = `Could not rollback "${index}" Index creation after an error during ${originDescription}: ` + deletionError.message 82 | deletionError.cause = causeError 83 | log().info({err: deletionError}, `Could not delete "${index}" Index during rollback:`) 84 | throw deletionError 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /lib/rollbacks.test.js: -------------------------------------------------------------------------------- 1 | const uuid = require('uuid/v1') 2 | 3 | const {suffix} = require('./indices') 4 | const { 5 | rollbackAliasCreation, 6 | rollbackIndexTemplateCreation, 7 | rollbackIndexCreation, 8 | rollbackAliasSwitch 9 | } = require('./rollbacks') 10 | const {es, esError, awaitRefresh} = require('./es') 11 | 12 | jest.setTimeout(60000) 13 | 14 | describe('rollbackAliasCreation', () => { 15 | const name = uuid() 16 | const index = suffix(uuid()) 17 | 18 | const ensureAliasIsDeleted = async ({name}) => { 19 | try { 20 | await es().indices.deleteAlias({name, index: '_all'}) 21 | } catch (e) { 22 | // ignore 23 | expect(e instanceof esError).toBeTruthy() 24 | expect(e.statusCode).toBe(404) 25 | } 26 | } 27 | const ensureIndexIsDeleted = async ({index}) => { 28 | try { 29 | await es().indices.delete({index}) 30 | } catch (e) { 31 | // ignore 32 | expect(e instanceof esError).toBeTruthy() 33 | expect(e.statusCode).toBe(404) 34 | } 35 | } 36 | 37 | beforeEach(async () => { 38 | await es().indices.create({index}) 39 | await es().indices.putAlias({name, index}) 40 | await es().indices.refresh({index}) 41 | }) 42 | 43 | afterEach(async () => { 44 | await ensureAliasIsDeleted({name}) 45 | await ensureIndexIsDeleted({index}) 46 | }) 47 | 48 | it(`should delete an existing alias`, async () => { 49 | expect(await es().indices.existsAlias({name})).toBeTruthy() 50 | await rollbackAliasCreation({name, index}, Error('"should delete an existing alias" test cause'), `"should delete an existing alias" test`) 51 | expect(await es().indices.existsAlias({name})).toBeFalsy() 52 | }) 53 | 54 | it(`should error when trying to delete an unknown alias`, async () => { 55 | await ensureAliasIsDeleted({name, index}) 56 | try { 57 | await rollbackAliasCreation({name, index}, Error('"should error when trying to delete an unknown alias" test cause'), `"should error when trying to delete an unknown alias" test`) 58 | } catch (e) { 59 | expect(e).toBeDefined() 60 | } 61 | expect(await es().indices.existsAlias({name})).toBeFalsy() 62 | }) 63 | }) 64 | 65 | describe('rollbackAliasSwitch', () => { 66 | const name = uuid() 67 | const sourceIndex = suffix(uuid(), new Date(Math.random() * 1e10)) 68 | const index = suffix(uuid()) 69 | 70 | const aliasExists = async (name) => es().indices.existsAlias({name}) 71 | const aliasIndex = async (name) => Object.keys((await es().indices.getAlias({name})))[0] 72 | const ensureAliasIsDeleted = async ({name}) => { 73 | try { 74 | await es().indices.deleteAlias({name, index: '_all'}) 75 | } catch (e) { 76 | // ignore 77 | expect(e instanceof esError).toBeTruthy() 78 | expect(e.statusCode).toBe(404) 79 | } 80 | } 81 | const ensureIndexIsDeleted = async ({index}) => { 82 | try { 83 | await es().indices.delete({index}) 84 | } catch (e) { 85 | // ignore 86 | expect(e instanceof esError).toBeTruthy() 87 | expect(e.statusCode).toBe(404) 88 | } 89 | } 90 | 91 | beforeEach(async () => { 92 | await es().indices.create({index: sourceIndex}).then(() => es().indices.refresh({index: sourceIndex})) 93 | await es().indices.create({index}).then(() => es().indices.refresh({index})).then(() => es().indices.putAlias({name, index})) 94 | }) 95 | 96 | afterEach(async () => { 97 | await ensureAliasIsDeleted({name}) 98 | await Promise.all([ 99 | ensureIndexIsDeleted({index}), 100 | ensureIndexIsDeleted({index: sourceIndex}) 101 | ]) 102 | await awaitRefresh() 103 | }) 104 | 105 | it(`should revert an existing alias`, async () => { 106 | expect(await es().indices.existsAlias({name})).toBeTruthy() 107 | expect(await aliasIndex(name)).toBe(index) 108 | 109 | await rollbackAliasSwitch({name, sourceIndex, index}, Error('"should revert an existing alias" test cause'), `"should revert an existing alias" test`) 110 | 111 | expect(await es().indices.existsAlias({name})).toBeTruthy() 112 | expect(await aliasIndex(name)).toBe(sourceIndex) 113 | }) 114 | 115 | it(`should create a valid rollbacked alias even if the alias is not existing`, async () => { 116 | const fakeAlias = uuid() 117 | expect(await aliasExists(fakeAlias)).toBeFalsy() 118 | 119 | await rollbackAliasSwitch({name, sourceIndex, index}, Error('"should revert an existing alias" test cause'), `"should revert an existing alias" test`) 120 | 121 | expect(await es().indices.existsAlias({name})).toBeTruthy() 122 | expect(await aliasIndex(name)).toBe(sourceIndex) 123 | }) 124 | 125 | it(`should fallback to just add the new alias when trying to remove an unknown index from alias`, async () => { 126 | const fakeIndex = suffix(uuid()) 127 | 128 | await rollbackAliasSwitch({name, sourceIndex, index: fakeIndex}, Error('"should fallback to just add the new alias when trying to remove an unknown index from alias" test cause'), `"should fallback to just add the new alias when trying to remove an unknown index from alias" test`) 129 | 130 | await await es().indices.refresh({index: sourceIndex}) 131 | 132 | expect(await es().indices.existsAlias({name})).toBeTruthy() 133 | expect(await aliasIndex(name)).toBe(sourceIndex) 134 | }) 135 | 136 | it(`should error when trying to add an unknown sourceIndex`, async () => { 137 | const fakeSourceIndex = suffix(uuid()) 138 | expect.assertions(3) 139 | try { 140 | await rollbackAliasSwitch({name, sourceIndex: fakeSourceIndex, index}, Error('"should error when trying to revert an unknown alias" test cause'), `"should error when trying to revert an unknown alias" test`) 141 | } catch (e) { 142 | expect(e).toBeDefined() 143 | expect(await es().indices.existsAlias({name})).toBeTruthy() 144 | // did not actually rollback 145 | expect(await aliasIndex(name)).toBe(index) 146 | } 147 | }) 148 | }) 149 | 150 | describe('rollbackIndexTemplateCreation', () => { 151 | const name = uuid() 152 | 153 | const templateExists = async (name) => es().indices.existsTemplate({name}) 154 | const ensureIndexTemplateIsDeleted = async ({name}) => { 155 | try { 156 | await es().indices.deleteTemplate({name}) 157 | } catch (e) { 158 | // ignore 159 | expect(e instanceof esError).toBeTruthy() 160 | expect(e.statusCode).toBe(404) 161 | } 162 | } 163 | 164 | beforeEach(async () => { 165 | await es().indices.putTemplate({ 166 | name, 167 | body: { 168 | 'index_patterns': [`${name}-*`], 169 | 'settings': { 170 | 'number_of_shards': 1 171 | }, 172 | 'aliases': { 173 | 'alias1': {}, 174 | '{index}-alias': {} 175 | } 176 | }}) 177 | }) 178 | 179 | afterEach(async () => { 180 | await ensureIndexTemplateIsDeleted({name}) 181 | }) 182 | 183 | it(`should delete an existing template`, async () => { 184 | expect(await templateExists(name)).toBeTruthy() 185 | await rollbackIndexTemplateCreation({name}, Error('"should delete an existing template" test cause'), `"should delete an existing template" test`) 186 | expect(await templateExists(name)).toBeFalsy() 187 | }) 188 | 189 | it(`should error when trying to delete an unknown template`, async () => { 190 | await ensureIndexTemplateIsDeleted({name}) 191 | try { 192 | await rollbackIndexTemplateCreation({name}, Error('"should error when trying to delete an unknown template" test cause'), `"should error when trying to delete an unknown template" test`) 193 | } catch (e) { 194 | expect(e).toBeDefined() 195 | } 196 | expect(await templateExists(name)).toBeFalsy() 197 | }) 198 | }) 199 | 200 | describe('rollbackIndexCreation', () => { 201 | const index = uuid() 202 | 203 | const ensureIndexIsDeleted = async ({index}) => { 204 | try { 205 | await es().indices.delete({index}) 206 | } catch (e) { 207 | // ignore 208 | expect(e instanceof esError).toBeTruthy() 209 | expect(e.statusCode).toBe(404) 210 | } 211 | } 212 | 213 | beforeEach(async () => { 214 | await es().indices.create({index}) 215 | await es().indices.refresh({index}) 216 | }) 217 | 218 | afterEach(async () => { 219 | await ensureIndexIsDeleted({index}) 220 | }) 221 | 222 | it(`should delete an existing index`, async () => { 223 | expect(await es().indices.exists({index})).toBeTruthy() 224 | await rollbackIndexCreation({index}, Error('"should delete an existing index" test cause'), `"should delete an existing index" test`) 225 | expect(await es().indices.exists({index})).toBeFalsy() 226 | }) 227 | 228 | it(`should error and do nothing when trying to create an existing index`, async () => { 229 | try { 230 | const causeError = Error('[resource_already_exists_exception] "should error and do nothing when trying to create an existing index" test cause') 231 | causeError.statusCode = 400 232 | causeError.body = {error: {type: 'resource_already_exists_exception'}} 233 | await rollbackIndexCreation({index}, causeError, `"should error and do nothing when trying to create an existing index" test`) 234 | } catch (e) { 235 | expect(e).toBeDefined() 236 | } 237 | // it should still exist 238 | expect(await es().indices.exists({index})).toBeTruthy() 239 | }) 240 | 241 | it(`should error when index to delete during rollback has already been deleted`, async () => { 242 | await ensureIndexIsDeleted({index}) 243 | try { 244 | await rollbackIndexCreation({index}, Error('"should error when index to delete during rollback has already been deleted" test cause'), `"should error when index to delete during rollback has already been deleted" test`) 245 | } catch (e) { 246 | expect(e).toBeDefined() 247 | } 248 | // it was already delted, it should still be 249 | expect(await es().indices.exists({index})).toBeFalsy() 250 | }) 251 | }) 252 | -------------------------------------------------------------------------------- /lib/state.js: -------------------------------------------------------------------------------- 1 | const pTimeout = require('p-timeout') 2 | const moment = require('moment') 3 | const Boom = require('boom') 4 | 5 | const {memoryStore} = require('./memoryStore') 6 | const {log} = require('./logger') 7 | 8 | // timeouts should like 10 times the maximum expected value 9 | const CREATION_TIMEOUT_IN_MS = 10 * 60 * 1000 // 10 minutes 10 | const UPDATE_TIMEOUT_IN_MS = 48 * 60 * 60 * 1000 // 2 days, reindexations can be very long 11 | const DELETION_TIMEOUT_IN_MS = 10 * 60 * 1000 // 10 minutes 12 | 13 | const store = exports.store = (store) => { 14 | if (store) { 15 | exports._store = store 16 | log().info({store: store.name}, `Will use provided store to keep the state`) 17 | } 18 | 19 | if (!exports._store) { 20 | exports._store = memoryStore 21 | log().warn({store: memoryStore.name}, `Will use default in-memory store to keep the state, you should probably override it for a production micro-service`) 22 | } 23 | return exports._store 24 | } 25 | 26 | const run = async (operation, name, timeoutDurationInMs, task) => { 27 | log().info({operation, name, timeoutDurationInMs}, `Will try to run operation "${operation}" on "${name}" (timeout: ${timeoutDurationInMs} ms)`) 28 | const {start, end} = await store().save(operation, name, timeoutDurationInMs) 29 | const remainingTimeInMs = moment(start).add(timeoutDurationInMs, 'milliseconds').diff(moment(new Date())) 30 | 31 | try { 32 | return await pTimeout(task(), remainingTimeInMs, () => { 33 | throw Boom.gatewayTimeout(`Operation '${operation}' Timeout: the operation started ${moment(start).fromNow()} did not finish in the given '${timeoutDurationInMs}' ms time it had to ( it should have finished well before ${moment(end).fromNow()}). You are likely to have to either change this timeout because it was not enough, or make some manual operations in your Elasticsearch Cluster`) 34 | }) 35 | } finally { 36 | store().delete(operation, name) 37 | } 38 | } 39 | 40 | exports.run = run 41 | 42 | exports.indexCreation = (name, task) => run('creation', name, CREATION_TIMEOUT_IN_MS, task) 43 | 44 | exports.indexUpdate = (name, task) => run('update', name, UPDATE_TIMEOUT_IN_MS, task) 45 | 46 | exports.indexDeletion = (name, task) => run('deletion', name, DELETION_TIMEOUT_IN_MS, task) 47 | -------------------------------------------------------------------------------- /lib/state.test.js: -------------------------------------------------------------------------------- 1 | const uuid = require('uuid/v1') 2 | const delay = require('delay') 3 | 4 | const { memoryStore } = require('./memoryStore') 5 | const { store, run } = require('./state') 6 | 7 | describe('store', () => { 8 | // nominal - raw Error connection, no details 9 | it(`should return a default store without params`, async () => { 10 | const stateStore = store() 11 | expect(stateStore).toBeDefined() 12 | expect(stateStore).toHaveProperty('save') 13 | expect(stateStore).toHaveProperty('delete') 14 | }) 15 | 16 | it(`should return my store if I give it in param`, async () => { 17 | const myStore = {save: jest.fn(), delete: jest.fn()} 18 | const stateStore = store(myStore) 19 | expect(stateStore).toBeDefined() 20 | expect(stateStore).toBe(myStore) 21 | }) 22 | 23 | it(`should define my store as the default one`, async () => { 24 | const myStore = {save: jest.fn(), delete: jest.fn()} 25 | const stateStore = store(myStore) 26 | expect(stateStore).toBeDefined() 27 | expect(stateStore).toBe(myStore) 28 | const defaultStore = store() 29 | expect(defaultStore).toBeDefined() 30 | expect(defaultStore).toBe(myStore) 31 | }) 32 | 33 | it(`should be able to 'run' an operation in a given timeout`, async () => { 34 | const name = uuid() 35 | store(memoryStore) // ensure a proper store 36 | expect.assertions(0) 37 | try { 38 | await run('test', name, 1000, async () => delay(100)) 39 | } catch (e) { 40 | expect(e).not.toBeDefined() 41 | } 42 | }) 43 | 44 | it(`should be able to manage the timeout of a 'run' an operation`, async () => { 45 | const name = uuid() 46 | store(memoryStore) // ensure a proper store 47 | try { 48 | await run('test', name, 100, async () => delay(400)) 49 | } catch (e) { 50 | expect(e).toBeDefined() 51 | expect(e).toHaveProperty('isBoom', true) 52 | expect(e).toHaveProperty('output.statusCode', 504) // gateway timeout 53 | } 54 | }) 55 | }) 56 | -------------------------------------------------------------------------------- /lib/validation.js: -------------------------------------------------------------------------------- 1 | const Joi = require('joi') 2 | const Boom = require('boom') 3 | const escape = require('escape-string-regexp') 4 | 5 | // https://stackoverflow.com/questions/41585392/what-are-the-rules-for-index-names-in-elastic-search#41585755 6 | const maxIndexSize = 255 7 | exports.indexName = Joi 8 | .string() 9 | .trim() 10 | .lowercase() 11 | .regex(/[#\\/*?"<>|]/, {invert: true, name: 'contain #, \\, /, *, ?, ", <, >, or |'}) 12 | .regex(/^[_\-+]/, {invert: true, name: 'start with _, - or +'}) 13 | .not(['.', '..']) 14 | .min(1) 15 | .max(maxIndexSize, 'utf8') 16 | .required() 17 | .label('index name') 18 | 19 | const indexSuffixSize = 25 // ( '-' + max of the iso data size) 20 | const maxIndexWithoutSuffixSize = maxIndexSize - indexSuffixSize 21 | exports.indexNameWithoutSuffix = exports.indexName.max(maxIndexWithoutSuffixSize, 'utf8').label('name') 22 | 23 | exports.indexTemplateStructure = Joi.object() 24 | .keys({ 25 | index_patterns: Joi.array() 26 | .min(1) 27 | .items(Joi.string().regex(new RegExp(`-*$`)).allow('*')) 28 | .single(), // indexTemplate(name) will take care of the default later 29 | settings: Joi.object().unknown(true), 30 | mappings: Joi.object().unknown(true) 31 | }) 32 | .unknown(true) 33 | .empty(Joi.object().length(0)) 34 | .label('index template') 35 | 36 | exports.indexTemplate = name => Joi.object() 37 | .keys({ 38 | index_patterns: Joi.array() 39 | .min(1) 40 | .items(Joi.string().regex(new RegExp(`^${escape(`${name}-*`)}$`)).allow('*')) 41 | .single() 42 | .default([`${name.trim().toLowerCase()}-*`]), 43 | settings: Joi.object().unknown(true), 44 | mappings: Joi.object().unknown(true) 45 | }) 46 | .unknown(true) 47 | .empty(Joi.object().length(0)) 48 | .label('index template') 49 | 50 | exports.validate = (...params) => { 51 | try { 52 | return Joi.attempt(...params) 53 | } catch (error) { 54 | throw Boom.boomify(error, {statusCode: 400}) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /lib/validation.test.js: -------------------------------------------------------------------------------- 1 | const repeat = require('lodash.repeat') 2 | 3 | const { indexName, indexNameWithoutSuffix, indexTemplate, indexTemplateStructure } = require('./validation') 4 | 5 | describe('indexName', () => { 6 | // nominal 7 | it(`should allow a basic ascii name`, () => { 8 | const validation = indexName.validate('abcdef') 9 | expect(validation.error).toBeNull() 10 | }) 11 | 12 | // not empty 13 | it(`should forbid an empty name`, () => { 14 | const validation = indexName.validate('') 15 | expect(validation.error).not.toBeNull() 16 | }) 17 | 18 | // not start with -, + or _ 19 | it(`should forbid a name starting with '-'`, () => { 20 | const validation = indexName.validate('-abcde') 21 | expect(validation.error).not.toBeNull() 22 | }) 23 | 24 | it(`should forbid a name starting with '_'`, () => { 25 | const validation = indexName.validate('_abcde') 26 | expect(validation.error).not.toBeNull() 27 | }) 28 | 29 | it(`should forbid a name starting with '+'`, () => { 30 | const validation = indexName.validate('+abcde') 31 | expect(validation.error).not.toBeNull() 32 | }) 33 | 34 | // not contain #, \, /, *, ?, ", <, >, or | 35 | it(`should forbid a name containing '#'`, () => { 36 | const validation = indexName.validate('ab#cde') 37 | expect(validation.error).not.toBeNull() 38 | }) 39 | 40 | it(`should forbid a name containing '\\'`, () => { 41 | const validation = indexName.validate('ab\\cde') 42 | expect(validation.error).not.toBeNull() 43 | }) 44 | 45 | it(`should forbid a name containing '/'`, () => { 46 | const validation = indexName.validate('ab/cde') 47 | expect(validation.error).not.toBeNull() 48 | }) 49 | 50 | it(`should forbid a name containing '*'`, () => { 51 | const validation = indexName.validate('ab*cde') 52 | expect(validation.error).not.toBeNull() 53 | }) 54 | 55 | it(`should forbid a name containing '?'`, () => { 56 | const validation = indexName.validate('ab?cde') 57 | expect(validation.error).not.toBeNull() 58 | }) 59 | 60 | it(`should forbid a name containing '"'`, () => { 61 | const validation = indexName.validate('ab"cde') 62 | expect(validation.error).not.toBeNull() 63 | }) 64 | 65 | it(`should forbid a name containing '<'`, () => { 66 | const validation = indexName.validate('ab'`, () => { 71 | const validation = indexName.validate('ab>cde') 72 | expect(validation.error).not.toBeNull() 73 | }) 74 | 75 | it(`should forbid a name containing '|'`, () => { 76 | const validation = indexName.validate('ab|cde') 77 | expect(validation.error).not.toBeNull() 78 | }) 79 | 80 | // not . or .. 81 | it(`should forbid the name '.'`, () => { 82 | const validation = indexName.validate('.') 83 | expect(validation.error).not.toBeNull() 84 | }) 85 | 86 | it(`should forbid the name '..'`, () => { 87 | const validation = indexName.validate('..') 88 | expect(validation.error).not.toBeNull() 89 | }) 90 | 91 | // max 255 binary 'char' 92 | it(`should allow a 255 characters ascii name`, () => { 93 | const validation = indexName.validate(repeat('a', 255)) 94 | expect(validation.error).toBeNull() 95 | }) 96 | 97 | it(`should allow a less than 255 'char' name for a unicode string`, () => { 98 | const validation = indexName.validate(repeat('🐭', 252 / 4)) 99 | expect(validation.error).toBeNull() 100 | }) 101 | 102 | it(`should forbid a more than 255 'char' name for an ascii string`, () => { 103 | const validation = indexName.validate(repeat('a', 256)) 104 | expect(validation.error).not.toBeNull() 105 | }) 106 | 107 | it(`should allow a nominal length unicode name`, () => { 108 | const validation = indexName.validate('👉✨🚀👍🥂') 109 | expect(validation.error).toBeNull() 110 | }) 111 | 112 | it(`should forbid a more than 255 characters unicode name`, () => { 113 | const validation = indexName.validate(repeat('a', 256)) 114 | expect(validation.error).not.toBeNull() 115 | }) 116 | 117 | it(`should forbid a more than 255 'char' name for a unicode string`, () => { 118 | const validation = indexName.validate(repeat('🐭', 256 / 4)) 119 | expect(validation.error).not.toBeNull() 120 | }) 121 | }) 122 | 123 | describe('indexNameWithoutSuffix', () => { 124 | // nominal 125 | it(`should allow a basic ascii name`, () => { 126 | const validation = indexNameWithoutSuffix.validate('abcdef') 127 | expect(validation.error).toBeNull() 128 | }) 129 | 130 | // not empty 131 | it(`should forbid an empty name`, () => { 132 | const validation = indexNameWithoutSuffix.validate('') 133 | expect(validation.error).not.toBeNull() 134 | }) 135 | 136 | // not start with -, + or _ 137 | it(`should forbid a name starting with '-'`, () => { 138 | const validation = indexNameWithoutSuffix.validate('-abcde') 139 | expect(validation.error).not.toBeNull() 140 | }) 141 | 142 | it(`should forbid a name starting with '_'`, () => { 143 | const validation = indexNameWithoutSuffix.validate('_abcde') 144 | expect(validation.error).not.toBeNull() 145 | }) 146 | 147 | it(`should forbid a name starting with '+'`, () => { 148 | const validation = indexNameWithoutSuffix.validate('+abcde') 149 | expect(validation.error).not.toBeNull() 150 | }) 151 | 152 | // not contain #, \, /, *, ?, ", <, >, or | 153 | it(`should forbid a name containing '#'`, () => { 154 | const validation = indexNameWithoutSuffix.validate('ab#cde') 155 | expect(validation.error).not.toBeNull() 156 | }) 157 | 158 | it(`should forbid a name containing '\\'`, () => { 159 | const validation = indexNameWithoutSuffix.validate('ab\\cde') 160 | expect(validation.error).not.toBeNull() 161 | }) 162 | 163 | it(`should forbid a name containing '/'`, () => { 164 | const validation = indexNameWithoutSuffix.validate('ab/cde') 165 | expect(validation.error).not.toBeNull() 166 | }) 167 | 168 | it(`should forbid a name containing '*'`, () => { 169 | const validation = indexNameWithoutSuffix.validate('ab*cde') 170 | expect(validation.error).not.toBeNull() 171 | }) 172 | 173 | it(`should forbid a name containing '?'`, () => { 174 | const validation = indexNameWithoutSuffix.validate('ab?cde') 175 | expect(validation.error).not.toBeNull() 176 | }) 177 | 178 | it(`should forbid a name containing '"'`, () => { 179 | const validation = indexNameWithoutSuffix.validate('ab"cde') 180 | expect(validation.error).not.toBeNull() 181 | }) 182 | 183 | it(`should forbid a name containing '<'`, () => { 184 | const validation = indexNameWithoutSuffix.validate('ab'`, () => { 189 | const validation = indexNameWithoutSuffix.validate('ab>cde') 190 | expect(validation.error).not.toBeNull() 191 | }) 192 | 193 | it(`should forbid a name containing '|'`, () => { 194 | const validation = indexNameWithoutSuffix.validate('ab|cde') 195 | expect(validation.error).not.toBeNull() 196 | }) 197 | 198 | // not . or .. 199 | it(`should forbid the name '.'`, () => { 200 | const validation = indexNameWithoutSuffix.validate('.') 201 | expect(validation.error).not.toBeNull() 202 | }) 203 | 204 | it(`should forbid the name '..'`, () => { 205 | const validation = indexNameWithoutSuffix.validate('..') 206 | expect(validation.error).not.toBeNull() 207 | }) 208 | 209 | // max 255 binary 'char' 210 | it(`should allow a 230 characters ascii name`, () => { 211 | const validation = indexNameWithoutSuffix.validate(repeat('a', 230)) 212 | expect(validation.error).toBeNull() 213 | }) 214 | 215 | it(`should allow a less than 230 'char' name for a unicode string`, () => { 216 | const validation = indexNameWithoutSuffix.validate(repeat('🐭', 228 / 4)) 217 | expect(validation.error).toBeNull() 218 | }) 219 | 220 | it(`should forbid a more than 230 'char' name for an ascii string`, () => { 221 | const validation = indexNameWithoutSuffix.validate(repeat('a', 236)) 222 | expect(validation.error).not.toBeNull() 223 | }) 224 | 225 | it(`should allow a nominal length unicode name`, () => { 226 | const validation = indexNameWithoutSuffix.validate('👉✨🚀👍🥂') 227 | expect(validation.error).toBeNull() 228 | }) 229 | 230 | it(`should forbid a more than 230 characters unicode name`, () => { 231 | const validation = indexNameWithoutSuffix.validate(repeat('a', 231)) 232 | expect(validation.error).not.toBeNull() 233 | }) 234 | 235 | it(`should forbid a more than 230 'char' name for a unicode string`, () => { 236 | const validation = indexNameWithoutSuffix.validate(repeat('🐭', 232 / 4)) 237 | expect(validation.error).not.toBeNull() 238 | }) 239 | }) 240 | 241 | describe('indexTemplateStructure', () => { 242 | it(`should allow a valid mapping applied to the {name} based indices`, () => { 243 | const validation = indexTemplateStructure.validate({ 244 | 'index_patterns': ['myindexname-*'], 245 | 'order': 0, 246 | 'settings': { 247 | 'number_of_shards': 1 248 | }, 249 | 'mappings': { 250 | 'type1': { 251 | '_source': { 'enabled': false } 252 | } 253 | } 254 | }) 255 | expect(validation.error).toBeNull() 256 | }) 257 | 258 | it(`should allow a valid mapping applied to all indices`, () => { 259 | const validation = indexTemplateStructure.validate({ 260 | 'index_patterns': ['*'], 261 | 'order': 0, 262 | 'settings': { 263 | 'number_of_shards': 1 264 | }, 265 | 'mappings': { 266 | 'type1': { 267 | '_source': { 'enabled': false } 268 | } 269 | } 270 | }) 271 | expect(validation.error).toBeNull() 272 | }) 273 | 274 | it(`should allow a valid mapping applied to all indices and more specifically to the {name} based indices`, () => { 275 | const validation = indexTemplateStructure.validate({ 276 | 'index_patterns': ['*', 'myindexname-*'], 277 | 'order': 0, 278 | 'settings': { 279 | 'number_of_shards': 1 280 | }, 281 | 'mappings': { 282 | 'type1': { 283 | '_source': { 'enabled': false } 284 | } 285 | } 286 | }) 287 | expect(validation.error).toBeNull() 288 | }) 289 | // empty object 290 | it(`should allow an empty object`, () => { 291 | const validation = indexTemplateStructure.validate({}) 292 | expect(validation.error).toBeNull() 293 | }) 294 | 295 | // undefined 296 | it(`should allow undefined as it is not mandatory`, () => { 297 | const validation = indexTemplateStructure.validate(undefined) 298 | expect(validation.error).toBeNull() 299 | }) 300 | }) 301 | 302 | describe('indexTemplate(name)', () => { 303 | it(`should allow a valid mapping applied to the {name} based indices`, () => { 304 | const validation = indexTemplate('myindexname').validate({ 305 | 'index_patterns': ['myindexname-*'], 306 | 'order': 0, 307 | 'settings': { 308 | 'number_of_shards': 1 309 | }, 310 | 'mappings': { 311 | 'type1': { 312 | '_source': { 'enabled': false } 313 | } 314 | } 315 | }) 316 | expect(validation.error).toBeNull() 317 | }) 318 | 319 | it(`should allow a valid mapping applied to all indices`, () => { 320 | const validation = indexTemplate('myindexname').validate({ 321 | 'index_patterns': ['*'], 322 | 'order': 0, 323 | 'settings': { 324 | 'number_of_shards': 1 325 | }, 326 | 'mappings': { 327 | 'type1': { 328 | '_source': { 'enabled': false } 329 | } 330 | } 331 | }) 332 | expect(validation.error).toBeNull() 333 | }) 334 | 335 | it(`should allow a valid mapping applied to all indices and more specifically to the {name} based indices`, () => { 336 | const validation = indexTemplate('myindexname').validate({ 337 | 'index_patterns': ['*', 'myindexname-*'], 338 | 'order': 0, 339 | 'settings': { 340 | 'number_of_shards': 1 341 | }, 342 | 'mappings': { 343 | 'type1': { 344 | '_source': { 'enabled': false } 345 | } 346 | } 347 | }) 348 | expect(validation.error).toBeNull() 349 | }) 350 | // empty object 351 | it(`should allow an empty object`, () => { 352 | const validation = indexTemplate('myindexname').validate({}) 353 | expect(validation.error).toBeNull() 354 | }) 355 | 356 | // undefined 357 | it(`should allow undefined as it is not mandatory`, () => { 358 | const validation = indexTemplate('myindexname').validate(undefined) 359 | expect(validation.error).toBeNull() 360 | }) 361 | }) 362 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "leaistic", 3 | "version": "1.0.0-development", 4 | "description": "ElasticSearch Manager", 5 | "main": "index.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "https://github.com/nearform/leaistic.git" 9 | }, 10 | "author": "florian.traverse@nearform.com", 11 | "license": "Apache-2.0", 12 | "bin": { 13 | "leaistic": "./bin/leaistic.js" 14 | }, 15 | "scripts": { 16 | "start": "npm run start:human", 17 | "start:human": "npm run start:raw | npx pino-colada", 18 | "start:verbose": "npm run start:raw | pino", 19 | "start:raw": "nodemon bin/leaistic.js", 20 | "commitmsg": "commitlint -e $GIT_PARAMS", 21 | "semantic-release": "semantic-release", 22 | "stash-unstaged": "git stash save -k 'pre-linting-stash' >> /dev/null", 23 | "unstash-unstaged": "git stash pop --index", 24 | "lint-staged": "lint-staged || (yarn run pop-stash >> /dev/null && exit 1)", 25 | "pop-stash": "git stash && git stash pop stash@{1} && git read-tree stash && git stash drop", 26 | "lint": "standard --fix --verbose | snazzy", 27 | "prees:local:start": "sysctl vm.max_map_count | grep -q 262144 || sudo sysctl -w vm.max_map_count=262144", 28 | "es:local:start": "docker-compose -f docker-compose.local.yml up", 29 | "es:local:stop": "docker-compose -f docker-compose.local.yml down -v", 30 | "es:local": "npm run es:local:start; npm run es:local:stop", 31 | "test": "npm run test:human", 32 | "test:human": "npm run test:raw | npx pino-colada", 33 | "test:verbose": "npm run test:raw | pino", 34 | "test:raw": "jest --coverage --colors", 35 | "test:debug": "npm run test:debug:human", 36 | "test:debug:human": "npm run test:debug:raw | npx pino-colada", 37 | "test:debug:verbose": "npm run test:debug:raw | pino", 38 | "test:debug:raw": "node --inspect-brk node_modules/.bin/jest --runInBand", 39 | "test:watch": "npm run test:watch:human", 40 | "test:watch:human": "npm run test:watch:raw | npx pino-colada", 41 | "test:watch:verbose": "npm run test:watch:raw | pino", 42 | "test:watch:raw": "jest --colors --watch --notify --coverage", 43 | "test:ci": "ES_URL=http://localhost:9200 jest --ci --testResultsProcessor=jest-junit --coverage --maxWorkers=4 --colors | pino" 44 | }, 45 | "pre-commit": { 46 | "run": [ 47 | "stash-unstaged", 48 | "lint-staged", 49 | "pop-stash" 50 | ] 51 | }, 52 | "lint-staged": { 53 | "*.js": [ 54 | "npm run lint", 55 | "git add", 56 | "jest --bail --findRelatedTests" 57 | ] 58 | }, 59 | "standard": { 60 | "env": [ 61 | "node", 62 | "jest" 63 | ] 64 | }, 65 | "jest-junit": { 66 | "output": ".ci-test-results/jest/results.xml" 67 | }, 68 | "dependencies": { 69 | "delay": "^2.0.0", 70 | "elasticsearch": "^15.0.0", 71 | "escape-string-regexp": "^1.0.5", 72 | "hapi": "^17.5.0", 73 | "hapi-pino": "^4.0.4", 74 | "hapi-swagger": "^9.4.2", 75 | "inert": "^5.1.0", 76 | "joi": "^13.3.0", 77 | "json-truncate": "^1.3.0", 78 | "lodash.repeat": "^4.1.0", 79 | "lodash.startcase": "^4.4.0", 80 | "make-promises-safe": "^1.1.0", 81 | "moment": "^2.22.2", 82 | "nodemon": "^1.17.4", 83 | "p-timeout": "^2.0.1", 84 | "pino": "^4.17.3", 85 | "vision": "^5.3.2" 86 | }, 87 | "devDependencies": { 88 | "@commitlint/cli": "^6.2.0", 89 | "@commitlint/config-conventional": "^6.1.3", 90 | "commitlint": "^6.2.0", 91 | "husky": "^0.14.3", 92 | "jest": "^24.3.0", 93 | "jest-junit": "^4.0.0", 94 | "lint-staged": "^8.1.5", 95 | "pre-commit": "^1.2.2", 96 | "semantic-release": "^15.13.4", 97 | "snazzy": "^7.1.1", 98 | "standard": "^11.0.1", 99 | "trace": "^3.1.0", 100 | "uuid": "^3.2.1" 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /release.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | publish: [ 3 | { 4 | path: '@semantic-release/npm', 5 | npmPublish: true 6 | }, 7 | { 8 | path: '@semantic-release/github' 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /server/failures.js: -------------------------------------------------------------------------------- 1 | const Boom = require('boom') 2 | 3 | exports.failAction = async (request, h, err) => { 4 | request.logger.warn({err}) 5 | if (err.isBoom) { 6 | throw err 7 | } 8 | 9 | if (err.statusCode) { 10 | err.isElasticSearch = true 11 | const {statusCode, ops} = err 12 | throw Boom.boomify(err, {ops, statusCode}) 13 | } 14 | 15 | if (err.isJoi) { 16 | const {name, message, details, isJoi} = err 17 | throw Boom.badRequest(`${name}: ${message}`, {name, details, isJoi}) 18 | } 19 | 20 | if (err.name && err.stack) { 21 | throw Boom.boomify(err) 22 | } 23 | 24 | throw err 25 | } 26 | -------------------------------------------------------------------------------- /server/failures.test.js: -------------------------------------------------------------------------------- 1 | const Boom = require('boom') 2 | const Joi = require('joi') 3 | 4 | const { failAction } = require('./failures') 5 | const { esError: EsError } = require('../lib/es') 6 | 7 | describe('failAction', () => { 8 | // nominal - Boom error 9 | it(`should rethrow a Boom error as is`, async () => { 10 | const err = Boom.badRequest() 11 | expect(failAction({logger: {warn: jest.fn()}}, null, err)).rejects.toBe(err) 12 | }) 13 | 14 | // nominal - ES issued error 15 | it(`should wrap and rethrow an ES error`, async () => { 16 | const err = new EsError('Fake ES error') 17 | err.statusCode = 400 18 | err.ops = {} 19 | expect.assertions(4) 20 | await failAction({logger: {warn: jest.fn()}}, null, err).catch(e => { 21 | expect(e).toHaveProperty('message', 'Fake ES error') 22 | expect(e).toHaveProperty('isBoom', true) 23 | expect(e).toHaveProperty('statusCode', 400) 24 | expect(e).toHaveProperty('ops', err.ops) 25 | }) 26 | }) 27 | 28 | // nominal - Joi validation issued error 29 | it(`should wrap a Joi validation error`, async () => { 30 | expect.assertions(4) 31 | try { 32 | await Joi.validate('abc', Joi.number()) 33 | } catch (err) { 34 | await failAction({logger: {warn: jest.fn()}}, null, err).catch(e => { 35 | expect(e).toHaveProperty('isBoom', true) 36 | expect(e).toHaveProperty('data.isJoi', true) 37 | expect(e).toHaveProperty('data.name', 'ValidationError') 38 | expect(e).toHaveProperty('output.statusCode', 400) 39 | }) 40 | } 41 | }) 42 | 43 | // nominal - Unknown error 44 | it(`should wrap and rethrow a unknown error`, async () => { 45 | const err = new Error('Unknown') 46 | expect.assertions(3) 47 | await failAction({logger: {warn: jest.fn()}}, null, err).catch(e => { 48 | expect(e).toHaveProperty('message', 'Unknown') 49 | expect(e).toHaveProperty('isBoom', true) 50 | expect(e).toHaveProperty('output.statusCode', 500) 51 | }) 52 | }) 53 | 54 | // error - not an error 55 | it(`should rethrow an unknown value`, async () => { 56 | const err = 'Not an Error' 57 | expect(failAction({logger: {warn: jest.fn()}}, null, err)).rejects.toBe(err) 58 | }) 59 | }) 60 | -------------------------------------------------------------------------------- /server/handlers/index.js: -------------------------------------------------------------------------------- 1 | const {create, update, delete: deleteIndex} = require('../../lib/indices') 2 | 3 | exports.indexCreator = async (request, h) => { 4 | const { name } = request.params 5 | const body = request.payload 6 | 7 | const {index, ops} = await create(name, {body}) 8 | return h.response({name, index, ops}) 9 | } 10 | 11 | exports.indexUpdater = async (request, h) => { 12 | const { name } = request.params 13 | const body = request.payload 14 | 15 | const {index, ops} = await update(name, {body}) 16 | return h.response({name, index, ops}) 17 | } 18 | 19 | exports.indexDeleter = async (request, h) => { 20 | const { name } = request.params 21 | 22 | const {index, ops} = await deleteIndex(name) 23 | return h.response({name, index, ops}) 24 | } 25 | -------------------------------------------------------------------------------- /server/handlers/index.test.js: -------------------------------------------------------------------------------- 1 | const uuid = require('uuid/v1') 2 | 3 | const { es } = require('../../lib/es') 4 | const { suffix } = require('../../lib/indices') 5 | const { indexCreator, indexUpdater, indexDeleter } = require('.') 6 | 7 | jest.setTimeout(60000) 8 | 9 | describe('indexCreator', () => { 10 | // nominal 11 | it(`should create an index and alias with just a (alias) name`, async () => { 12 | const name = uuid() 13 | // Note: hapi always makes payload an empty object when not defined 14 | const request = {params: {name}, payload: {}} 15 | const h = { 16 | response: jest.fn() 17 | } 18 | await indexCreator(request, h) 19 | expect(h.response).toBeDefined() 20 | expect(h.response).toHaveBeenCalled() 21 | expect(h.response.mock).toBeDefined() 22 | expect(h.response.mock.calls).toBeDefined() 23 | expect(h.response.mock.calls.length).toBe(1) 24 | expect(h.response.mock.calls[0].length).toBe(1) 25 | expect(h.response.mock.calls[0][0]).toHaveProperty('name', name) 26 | expect(h.response.mock.calls[0][0]).toHaveProperty('index') 27 | expect(h.response.mock.calls[0][0]).toHaveProperty('ops') 28 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('preChecks') 29 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('index') 30 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('alias') 31 | // more assertions in indices.test.js 32 | }) 33 | }) 34 | 35 | describe('indexUpdater', () => { 36 | // nominal 37 | it(`should create a new index, reindex old index into new one, then switch the alias, from just a (alias) name`, async () => { 38 | const name = uuid() 39 | const index = suffix(name) 40 | await es().indices.create({index}) 41 | await es().indices.putAlias({name, index}) 42 | await es().indices.refresh({index}) 43 | // Note: hapi always makes payload an empty object when not defined 44 | const request = {params: {name}, payload: {}} 45 | const h = { 46 | response: jest.fn() 47 | } 48 | await indexUpdater(request, h) 49 | expect(h.response).toBeDefined() 50 | expect(h.response).toHaveBeenCalled() 51 | expect(h.response.mock).toBeDefined() 52 | expect(h.response.mock.calls).toBeDefined() 53 | expect(h.response.mock.calls.length).toBe(1) 54 | expect(h.response.mock.calls[0].length).toBe(1) 55 | 56 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('aliasExists', true) 57 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('findAlias') 58 | 59 | expect(h.response.mock.calls[0][0].ops).not.toHaveProperty('template') 60 | 61 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('index') 62 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('reindex') 63 | 64 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('postReindex') 65 | 66 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('switchAlias') 67 | 68 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('postAliasSwitch') 69 | // more assertions in indices.test.js 70 | }) 71 | 72 | it(`should deploy a template, then create a new index, reindex old index into new one, and finally switch the alias, from just an (alias) name`, async () => { 73 | const name = uuid() 74 | const index = suffix(name) 75 | await es().indices.create({index}) 76 | await es().indices.putAlias({name, index}) 77 | await es().indices.refresh({index}) 78 | // Note: hapi always makes payload an empty object when not defined 79 | const request = { 80 | params: {name}, 81 | payload: { 82 | 'index_patterns': [`${name}-*`], 83 | 'settings': { 84 | 'number_of_shards': 1 85 | }, 86 | 'aliases': { 87 | 'alias1': {}, 88 | '{index}-alias': {} 89 | } 90 | } 91 | } 92 | const h = { 93 | response: jest.fn() 94 | } 95 | await indexUpdater(request, h) 96 | expect(h.response).toBeDefined() 97 | expect(h.response).toHaveBeenCalled() 98 | expect(h.response.mock).toBeDefined() 99 | expect(h.response.mock.calls).toBeDefined() 100 | expect(h.response.mock.calls.length).toBe(1) 101 | expect(h.response.mock.calls[0].length).toBe(1) 102 | 103 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('aliasExists', true) 104 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('findAlias') 105 | 106 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('template') 107 | 108 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('index') 109 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('reindex') 110 | 111 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('postReindex') 112 | 113 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('switchAlias') 114 | 115 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('postAliasSwitch') 116 | // more assertions in indices.test.js 117 | }) 118 | }) 119 | 120 | describe('indexDeleter', () => { 121 | // nominal 122 | it(`should create an index and alias with just a (alias) name`, async () => { 123 | const name = uuid() 124 | const index = suffix(name) 125 | await es().indices.create({index}) 126 | await es().indices.putAlias({name, index}) 127 | await es().indices.refresh({index}) 128 | // Note: hapi always makes payload an empty object when not defined 129 | const request = {params: {name}, payload: {}} 130 | const h = { 131 | response: jest.fn() 132 | } 133 | await indexDeleter(request, h) 134 | expect(h.response).toBeDefined() 135 | expect(h.response).toHaveBeenCalled() 136 | expect(h.response.mock).toBeDefined() 137 | expect(h.response.mock.calls).toBeDefined() 138 | expect(h.response.mock.calls.length).toBe(1) 139 | expect(h.response.mock.calls[0].length).toBe(1) 140 | expect(h.response.mock.calls[0][0]).toHaveProperty('name', name) 141 | expect(h.response.mock.calls[0][0]).toHaveProperty('index') 142 | expect(h.response.mock.calls[0][0]).toHaveProperty('ops') 143 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('preChecks') 144 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('findAlias') 145 | expect(h.response.mock.calls[0][0].ops).toHaveProperty('deletions') 146 | // more assertions in indices.test.js 147 | }) 148 | }) 149 | -------------------------------------------------------------------------------- /server/index.js: -------------------------------------------------------------------------------- 1 | require('make-promises-safe') 2 | const startCase = require('lodash.startcase') 3 | 4 | const Hapi = require('hapi') 5 | const Inert = require('inert') 6 | const Vision = require('vision') 7 | const HapiSwagger = require('hapi-swagger') 8 | const HapiPino = require('hapi-pino') 9 | 10 | const Pack = require('../package') 11 | const Routes = require('./routes') 12 | const {checkHost, checkPort, checkConfig} = require('./validation') 13 | 14 | const {log} = require('../lib/logger') 15 | 16 | const HOST = checkHost(process.env.HOST, 'Environment Variable HOST') 17 | // allow for -1 port = random available port 18 | const PORT = checkPort(process.env.PORT, 'Environment Variable PORT') 19 | 20 | module.exports = async (_config = { host: HOST, port: PORT }) => { 21 | const config = checkConfig(_config) 22 | 23 | log().debug(config, 'Server configuration:') 24 | const server = await new Hapi.Server(config) 25 | 26 | const swaggerOptions = { 27 | info: { 28 | title: `${startCase(Pack.name)} API Documentation`, 29 | version: Pack.version 30 | } 31 | } 32 | 33 | await server.register([ 34 | { plugin: HapiPino, options: {name: 'Leaistic Server'} }, 35 | Inert, 36 | Vision, 37 | { plugin: HapiSwagger, options: swaggerOptions } 38 | ]) 39 | 40 | try { 41 | await server.start() 42 | server.route(Routes) 43 | server.logger().info('Server running at:', server.info.uri) 44 | } catch (err) { 45 | err.message = `Could not start the server: ${err.message}` 46 | log().fatal({err}) 47 | throw err 48 | } 49 | 50 | return server 51 | } 52 | -------------------------------------------------------------------------------- /server/index.test.js: -------------------------------------------------------------------------------- 1 | const start = require('.') 2 | 3 | jest.setTimeout(60000) 4 | 5 | describe('start', () => { 6 | // nominal, random avilable port to avoid conflict 7 | it(`should always start with port '-1'`, async () => { 8 | expect(() => start({port: -1}).not.toThrow()) 9 | }) 10 | 11 | // this cannot be tested as root, may break on Windows... 12 | ;((process.getuid() || process.getgid()) ? it : it.skip)(`should never start with port '1' (reserved for system)`, async () => { 13 | expect.assertions(5) 14 | try { 15 | await start({port: 1}) 16 | } catch (err) { 17 | expect(err).toBeDefined() 18 | expect(err).toHaveProperty('code', 'EACCES') 19 | expect(err).toHaveProperty('errno', 'EACCES') 20 | expect(err).toHaveProperty('syscall', 'listen') 21 | expect(err).toHaveProperty('port', 1) 22 | } 23 | }) 24 | 25 | it(`should not start when using a bad config`, async () => { 26 | try { 27 | await start({port: -10}) 28 | } catch (err) { 29 | expect(err).toBeDefined() 30 | expect(err).toHaveProperty('name', 'ValidationError') 31 | expect(err.message).toMatch(/port/) 32 | } 33 | }) 34 | }) 35 | -------------------------------------------------------------------------------- /server/routes.js: -------------------------------------------------------------------------------- 1 | const { indexCreator, indexUpdater, indexDeleter } = require('./handlers') 2 | const { indexNameWithoutSuffix, indexTemplateStructure } = require('./../lib/validation') 3 | const { failAction } = require('./failures') 4 | 5 | module.exports = [ 6 | { 7 | method: 'GET', 8 | path: '/ping', 9 | async handler (request, h) { 10 | return h.response('OK') 11 | }, 12 | config: { 13 | auth: false, 14 | description: 'Ping', 15 | notes: 'Returns HTTP 200 if the server is up and running.', 16 | tags: [ 'api', 'monitoring' ] 17 | } 18 | }, 19 | 20 | { 21 | method: 'PUT', 22 | path: '/index/{name}', 23 | handler: indexCreator, 24 | config: { 25 | description: 'Creates an index and its alias', 26 | notes: '{name} is the alias name, the index will be {name}-$date. If a {body} is provided, it will create/update an index template for {name}-*', 27 | tags: [ 'api', 'index' ], 28 | validate: { 29 | params: { name: indexNameWithoutSuffix }, 30 | payload: indexTemplateStructure.optional().allow(null), 31 | failAction 32 | } 33 | } 34 | }, 35 | 36 | { 37 | method: 'POST', 38 | path: '/index/{name}', 39 | handler: indexUpdater, 40 | config: { 41 | description: 'Updates an index by reindexing the old one into a new one then updates the alias', 42 | notes: '{name} is the alias name, the index will be {name}-$date. If a {body} is provided, it will create/update an index template for {name}-*', 43 | tags: [ 'api', 'index' ], 44 | validate: { 45 | params: { name: indexNameWithoutSuffix }, 46 | payload: indexTemplateStructure.optional().allow(null), 47 | failAction 48 | } 49 | } 50 | }, 51 | 52 | { 53 | method: 'DELETE', 54 | path: '/index/{name}', 55 | handler: indexDeleter, 56 | config: { 57 | description: 'Deletes an index and its alias', 58 | notes: '{name} is the alias name, the index is the first one pointed to by the alias', 59 | tags: [ 'api', 'index' ], 60 | validate: { 61 | params: { name: indexNameWithoutSuffix }, 62 | failAction 63 | } 64 | } 65 | } 66 | ] 67 | -------------------------------------------------------------------------------- /server/routes.test.js: -------------------------------------------------------------------------------- 1 | const uuid = require('uuid/v1') 2 | const HapiPino = require('hapi-pino') 3 | 4 | const { es } = require('../lib/es') 5 | const { suffix } = require('../lib/indices') 6 | const [ 7 | ping, 8 | indexPut, 9 | indexPost, 10 | indexDelete 11 | ] = require('./routes') 12 | 13 | jest.setTimeout(60000) 14 | 15 | const Hapi = require('hapi') 16 | 17 | describe('ping', () => { 18 | // nominal 19 | it(`should always be OK`, async () => { 20 | const server = Hapi.server() 21 | server.route(ping) 22 | const res = await server.inject('/ping') 23 | expect(res).toBeDefined() 24 | expect(res).toHaveProperty('payload', 'OK') 25 | expect(res).toHaveProperty('result', 'OK') 26 | expect(res).toHaveProperty('statusCode', 200) 27 | }) 28 | }) 29 | 30 | describe('index PUT', () => { 31 | // nominal - no payload 32 | it(`should create a new index/alias given a proper name`, async () => { 33 | const name = uuid() 34 | const server = Hapi.server() 35 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 36 | server.route(indexPut) 37 | const res = await server.inject({ method: 'PUT', url: `/index/${name}` }) 38 | expect(res).toBeDefined() 39 | expect(res.error).not.toBeDefined() 40 | expect(res).toHaveProperty('statusCode', 200) 41 | expect(res.result).toHaveProperty('name', name) 42 | expect(res.result).toHaveProperty('index') 43 | expect(res.result).toHaveProperty('ops') 44 | expect(res.result.ops).toHaveProperty('preChecks') 45 | expect(res.result.ops).toHaveProperty('index') 46 | expect(res.result.ops).toHaveProperty('alias') 47 | // more assertions in indices.test.js 48 | }) 49 | 50 | // nominal - empty object payload 51 | it(`should create a new index/alias given a proper name`, async () => { 52 | const name = uuid() 53 | const server = Hapi.server() 54 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 55 | server.route(indexPut) 56 | const res = await server.inject({ method: 'PUT', url: `/index/${name}`, payload: {} }) 57 | expect(res).toBeDefined() 58 | expect(res.error).not.toBeDefined() 59 | expect(res).toHaveProperty('statusCode', 200) 60 | expect(res.result).toHaveProperty('name', name) 61 | expect(res.result).toHaveProperty('index') 62 | expect(res.result).toHaveProperty('ops') 63 | expect(res.result.ops).toHaveProperty('preChecks') 64 | expect(res.result.ops).toHaveProperty('index') 65 | expect(res.result.ops).toHaveProperty('alias') 66 | // more assertions in indices.test.js 67 | }) 68 | 69 | // nominal - valid index template in payload 70 | it(`should create a new index/alias given a proper name and index template`, async () => { 71 | const name = uuid() 72 | const server = Hapi.server() 73 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 74 | server.route(indexPut) 75 | const payload = { 76 | 'index_patterns': [`${name}-*`], 77 | 'settings': { 78 | 'number_of_shards': 1 79 | }, 80 | 'aliases': { 81 | 'alias1': {}, 82 | '{index}-alias': {} 83 | } 84 | } 85 | const res = await server.inject({ method: 'PUT', url: `/index/${name}`, payload }) 86 | expect(res).toBeDefined() 87 | expect(res.error).not.toBeDefined() 88 | expect(res).toHaveProperty('statusCode', 200) 89 | expect(res.result).toHaveProperty('name', name) 90 | expect(res.result).toHaveProperty('index') 91 | expect(res.result).toHaveProperty('ops') 92 | expect(res.result.ops).toHaveProperty('preChecks') 93 | expect(res.result.ops).toHaveProperty('index') 94 | expect(res.result.ops).toHaveProperty('alias') 95 | // more assertions in indices.test.js 96 | }) 97 | 98 | // validation error - invalid index name 99 | it(`should NOT create a new index/alias given an invalid name`, async () => { 100 | const name = `_${uuid()}` 101 | const server = Hapi.server() 102 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 103 | server.route(indexPut) 104 | const res = await server.inject({ method: 'PUT', url: `/index/${name}` }) 105 | expect(res).toBeDefined() 106 | expect(res.error).not.toBeDefined() 107 | expect(res).toHaveProperty('statusCode', 400) 108 | expect(res.result).toHaveProperty('error') 109 | expect(res.result).toHaveProperty('message', `child "name" fails because ["name" with value "${name}" matches the inverted start with _, - or + pattern]`) 110 | // more assertions in indices.test.js 111 | }) 112 | 113 | // validation error - invalid index template in payload 114 | it(`should NOT create a new index/alias given a proper name, with a string as payload`, async () => { 115 | const name = uuid() 116 | const server = Hapi.server() 117 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 118 | server.route(indexPut) 119 | const payload = 'hello, world!' 120 | const res = await server.inject({ method: 'PUT', url: `/index/${name}`, payload }) 121 | expect(res).toBeDefined() 122 | expect(res.error).not.toBeDefined() 123 | expect(res).toHaveProperty('statusCode', 400) 124 | expect(res.result).toHaveProperty('error') 125 | expect(res.result).toHaveProperty('message', 'Invalid request payload JSON format') 126 | // more assertions in indices.test.js 127 | }) 128 | 129 | // validation error - invalid index template in payload 130 | it(`should NOT create a new index/alias given a proper name, but a bad index template pattern`, async () => { 131 | const name = uuid() 132 | const server = Hapi.server() 133 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 134 | server.route(indexPut) 135 | const payload = { 136 | 'index_patterns': [`hello, world!`], 137 | 'settings': { 138 | 'number_of_shards': 1 139 | }, 140 | 'aliases': { 141 | 'alias1': {}, 142 | '{index}-alias': {} 143 | } 144 | } 145 | const res = await server.inject({ method: 'PUT', url: `/index/${name}`, payload }) 146 | expect(res).toBeDefined() 147 | expect(res.error).not.toBeDefined() 148 | expect(res).toHaveProperty('statusCode', 400) 149 | expect(res.result).toHaveProperty('error') 150 | expect(res.result).toHaveProperty('message') 151 | expect(res.result.message).toMatch(/^{/) // start of in context error message 152 | // more assertions in indices.test.js 153 | }) 154 | }) 155 | 156 | describe('index POST', () => { 157 | // nominal - no payload 158 | it(`should update an existing index/alias given a proper name`, async () => { 159 | const name = uuid() 160 | const server = Hapi.server() 161 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 162 | server.route(indexPost) 163 | const index = suffix(name) 164 | await es().indices.create({index}) 165 | await es().indices.putAlias({name, index}) 166 | await es().indices.refresh({index}) 167 | const res = await server.inject({ method: 'POST', url: `/index/${name}` }) 168 | expect(res).toBeDefined() 169 | expect(res.error).not.toBeDefined() 170 | expect(res).toHaveProperty('statusCode', 200) 171 | expect(res.result).toHaveProperty('name', name) 172 | expect(res.result).toHaveProperty('index') 173 | expect(res.result).toHaveProperty('ops') 174 | expect(res.result.ops).toHaveProperty('aliasExists', true) 175 | expect(res.result.ops).toHaveProperty('findAlias') 176 | expect(res.result.ops).toHaveProperty('reindex') 177 | expect(res.result.ops).toHaveProperty('postReindex') 178 | expect(res.result.ops).toHaveProperty('postAliasSwitch') 179 | // more assertions in indices.test.js 180 | }) 181 | 182 | // nominal - empty payload object 183 | it(`should update an existing index/alias given a proper name`, async () => { 184 | const name = uuid() 185 | const server = Hapi.server() 186 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 187 | server.route(indexPost) 188 | const index = suffix(name) 189 | await es().indices.create({index}) 190 | await es().indices.putAlias({name, index}) 191 | await es().indices.refresh({index}) 192 | const res = await server.inject({ method: 'POST', url: `/index/${name}`, payload: {} }) 193 | expect(res).toBeDefined() 194 | expect(res.error).not.toBeDefined() 195 | expect(res).toHaveProperty('statusCode', 200) 196 | expect(res.result).toHaveProperty('name', name) 197 | expect(res.result).toHaveProperty('index') 198 | expect(res.result).toHaveProperty('ops') 199 | expect(res.result.ops).toHaveProperty('aliasExists', true) 200 | expect(res.result.ops).toHaveProperty('findAlias') 201 | expect(res.result.ops).toHaveProperty('reindex') 202 | expect(res.result.ops).toHaveProperty('postReindex') 203 | expect(res.result.ops).toHaveProperty('postAliasSwitch') 204 | // more assertions in indices.test.js 205 | }) 206 | 207 | // nominal - valid payload 208 | it(`should update an existing index/alias given a proper name and index template`, async () => { 209 | const name = uuid() 210 | const server = Hapi.server() 211 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 212 | server.route(indexPost) 213 | const index = suffix(name) 214 | await es().indices.create({index}) 215 | await es().indices.putAlias({name, index}) 216 | await es().indices.refresh({index}) 217 | const res = await server.inject({ method: 'POST', url: `/index/${name}` }) 218 | expect(res).toBeDefined() 219 | expect(res.error).not.toBeDefined() 220 | expect(res).toHaveProperty('statusCode', 200) 221 | expect(res.result).toHaveProperty('name', name) 222 | expect(res.result).toHaveProperty('index') 223 | expect(res.result).toHaveProperty('ops') 224 | expect(res.result.ops).toHaveProperty('aliasExists', true) 225 | expect(res.result.ops).toHaveProperty('findAlias') 226 | expect(res.result.ops).toHaveProperty('reindex') 227 | expect(res.result.ops).toHaveProperty('postReindex') 228 | expect(res.result.ops).toHaveProperty('postAliasSwitch') 229 | // more assertions in indices.test.js 230 | }) 231 | 232 | // error - invalid name 233 | it(`should update an existing index/alias given an invalid name`, async () => { 234 | const name = `_${uuid()}` 235 | const server = Hapi.server() 236 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 237 | server.route(indexPost) 238 | // can't create index/alias as name is not valid 239 | const res = await server.inject({ method: 'POST', url: `/index/${name}` }) 240 | expect(res).toBeDefined() 241 | expect(res.result).toHaveProperty('error') 242 | expect(res.result).toHaveProperty('message') 243 | expect(res.result).toHaveProperty('message', `child "name" fails because ["name" with value "${name}" matches the inverted start with _, - or + pattern]`) 244 | // more assertions in indices.test.js 245 | }) 246 | 247 | // error - invalid payload 248 | it(`should NOT update an existing index/alias given a proper name, with a string as payload`, async () => { 249 | const name = uuid() 250 | const server = Hapi.server() 251 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 252 | server.route(indexPost) 253 | const index = suffix(name) 254 | await es().indices.create({index}) 255 | await es().indices.putAlias({name, index}) 256 | await es().indices.refresh({index}) 257 | const payload = 'hello, world!' 258 | const res = await server.inject({ method: 'POST', url: `/index/${name}`, payload }) 259 | expect(res).toBeDefined() 260 | expect(res).toHaveProperty('statusCode', 400) 261 | expect(res.result).toHaveProperty('error') 262 | expect(res.result).toHaveProperty('message') 263 | expect(res.result).toHaveProperty('message', 'Invalid request payload JSON format') 264 | // more assertions in indices.test.js 265 | }) 266 | 267 | // error - invalid payload 268 | it(`should NOT update an existing index/alias given a proper name, but a bad index template pattern`, async () => { 269 | const name = uuid() 270 | const server = Hapi.server() 271 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 272 | server.route(indexPost) 273 | const index = suffix(name) 274 | await es().indices.create({index}) 275 | await es().indices.putAlias({name, index}) 276 | await es().indices.refresh({index}) 277 | const payload = { 278 | 'index_patterns': ['hello, world!'], 279 | 'settings': { 280 | 'number_of_shards': 1 281 | }, 282 | 'aliases': { 283 | 'alias1': {}, 284 | '{index}-alias': {} 285 | } 286 | } 287 | const res = await server.inject({ method: 'POST', url: `/index/${name}`, payload }) 288 | expect(res).toBeDefined() 289 | expect(res).toHaveProperty('statusCode', 400) 290 | expect(res.result).toHaveProperty('error') 291 | expect(res.result).toHaveProperty('message') 292 | expect(res.result.message).toMatch(/^{/) // start of in context error message 293 | // more assertions in indices.test.js 294 | }) 295 | }) 296 | 297 | describe('index DELETE', () => { 298 | // nominal 299 | it(`should delete an existing index/alias given a proper name`, async () => { 300 | const name = uuid() 301 | const server = Hapi.server() 302 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 303 | server.route(indexDelete) 304 | const index = suffix(name) 305 | await es().indices.create({index}) 306 | await es().indices.putAlias({name, index}) 307 | await es().indices.refresh({index}) 308 | const res = await server.inject({ method: 'DELETE', url: `/index/${name}` }) 309 | expect(res).toBeDefined() 310 | expect(res.error).not.toBeDefined() 311 | expect(res).toHaveProperty('statusCode', 200) 312 | expect(res.result).toHaveProperty('name', name) 313 | expect(res.result).toHaveProperty('index') 314 | expect(res.result).toHaveProperty('ops') 315 | expect(res.result.ops).toHaveProperty('preChecks') 316 | expect(res.result.ops).toHaveProperty('findAlias') 317 | expect(res.result.ops).toHaveProperty('deletions') 318 | // more assertions in indices.test.js 319 | }) 320 | 321 | // error - invalid name 322 | it(`should delete an existing index/alias given an invalid name`, async () => { 323 | const name = `_${uuid()}` 324 | const server = Hapi.server() 325 | await server.register([{ plugin: HapiPino, options: {name: 'Leaistic Tests'} }]) 326 | server.route(indexDelete) 327 | // can't create index/alias as name is not valid 328 | const res = await server.inject({ method: 'DELETE', url: `/index/${name}` }) 329 | expect(res).toBeDefined() 330 | expect(res.result).toHaveProperty('error') 331 | expect(res.result).toHaveProperty('message') 332 | expect(res.result).toHaveProperty('message', `child "name" fails because ["name" with value "${name}" matches the inverted start with _, - or + pattern]`) 333 | // more assertions in indices.test.js 334 | }) 335 | }) 336 | -------------------------------------------------------------------------------- /server/validation.js: -------------------------------------------------------------------------------- 1 | const Joi = require('joi') 2 | 3 | const hostSchema = Joi.string().min(1).default('localhost').empty('') 4 | const portSchema = Joi.number().min(-1).max(65535).default(3000).empty('') 5 | 6 | const configSchema = Joi.object().keys({ 7 | host: hostSchema, 8 | port: portSchema 9 | }).unknown(true) 10 | 11 | exports.checkHost = (host, label = 'server "host" configuration') => Joi.attempt(host, hostSchema.label(label)) 12 | 13 | exports.checkPort = (port, label = 'server "port" configuration') => Joi.attempt(port, portSchema.label(label)) 14 | 15 | exports.checkConfig = (config, label = 'server configuration') => Joi.attempt(config, configSchema.label(label)) 16 | -------------------------------------------------------------------------------- /server/validation.test.js: -------------------------------------------------------------------------------- 1 | const { checkHost, checkPort, checkConfig } = require('./validation') 2 | 3 | describe('checkHost', () => { 4 | // nominal 5 | it(`should allow a basic name`, () => { 6 | const host = checkHost('abcdef') 7 | expect(host).toBe('abcdef') 8 | }) 9 | 10 | it(`should allow a basic name and to override label`, () => { 11 | const host = checkHost('abcdef', 'my label') 12 | expect(host).toBe('abcdef') 13 | }) 14 | 15 | it(`should allow an IP`, () => { 16 | const host = checkHost('127.0.0.1') 17 | expect(host).toBe('127.0.0.1') 18 | }) 19 | 20 | it(`should allow an IPv6`, () => { 21 | const host = checkHost('[::]') 22 | expect(host).toBe('[::]') 23 | }) 24 | 25 | it(`should allow an empty value`, () => { 26 | const host = checkHost('') 27 | expect(host).toBe('localhost') 28 | }) 29 | 30 | it(`should allow an undefined value`, () => { 31 | const host = checkHost() 32 | expect(host).toBe('localhost') 33 | }) 34 | 35 | it(`should forbid a number`, () => { 36 | expect.assertions(3) 37 | try { 38 | checkHost(123) 39 | } catch (err) { 40 | expect(err).toBeDefined() 41 | expect(err).toHaveProperty('isJoi', true) 42 | expect(err).toHaveProperty('name', 'ValidationError') 43 | } 44 | }) 45 | 46 | it(`should forbid a number and a custom label`, () => { 47 | expect.assertions(4) 48 | try { 49 | checkHost(123, 'my label') 50 | } catch (err) { 51 | expect(err).toBeDefined() 52 | expect(err).toHaveProperty('isJoi', true) 53 | expect(err).toHaveProperty('name', 'ValidationError') 54 | expect(err.message).toMatch(/my label/) 55 | } 56 | }) 57 | }) 58 | 59 | describe('checkPort', () => { 60 | // nominal 61 | it(`should allow a number below 65536`, () => { 62 | const port = checkPort(65535) 63 | expect(port).toBe(65535) 64 | }) 65 | 66 | it(`should allow 0`, () => { 67 | const port = checkPort(0) 68 | expect(port).toBe(0) 69 | }) 70 | 71 | it(`should allow -1`, () => { 72 | const port = checkPort(-1) 73 | expect(port).toBe(-1) 74 | }) 75 | 76 | it(`should should coerce a string`, () => { 77 | const port = checkPort('1234') 78 | expect(port).toBe(1234) 79 | }) 80 | 81 | it(`should allow an undefined value`, () => { 82 | const port = checkPort() 83 | expect(port).toBe(3000) 84 | }) 85 | 86 | it(`should forbid a boolean`, () => { 87 | expect.assertions(3) 88 | try { 89 | checkPort(true) 90 | } catch (err) { 91 | expect(err).toBeDefined() 92 | expect(err).toHaveProperty('isJoi', true) 93 | expect(err).toHaveProperty('name', 'ValidationError') 94 | } 95 | }) 96 | }) 97 | 98 | describe('checkConfig', () => { 99 | // nominal 100 | it(`should allow a valid config`, () => { 101 | const config = checkConfig({host: '[::]', port: 3333}) 102 | expect(config).toHaveProperty('host', '[::]') 103 | expect(config).toHaveProperty('port', 3333) 104 | }) 105 | 106 | it(`should allow unknown keys`, () => { 107 | const config = checkConfig({host: '[::]', port: 3333, hello: 'world!'}) 108 | expect(config).toHaveProperty('host', '[::]') 109 | expect(config).toHaveProperty('port', 3333) 110 | expect(config).toHaveProperty('hello', 'world!') 111 | }) 112 | 113 | it(`should forbid a number`, () => { 114 | expect.assertions(3) 115 | try { 116 | checkConfig(123) 117 | } catch (err) { 118 | expect(err).toBeDefined() 119 | expect(err).toHaveProperty('isJoi', true) 120 | expect(err).toHaveProperty('name', 'ValidationError') 121 | } 122 | }) 123 | }) 124 | --------------------------------------------------------------------------------