├── .gitignore ├── .goreleaser.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── app.go ├── export.go ├── import.go └── reshard.go ├── go.mod ├── go.sum ├── main.go ├── tools ├── export.go ├── import.go └── reshard.go └── util └── progress.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | .idea/ 3 | elastictl 4 | dist/ 5 | build/ 6 | testdata/ 7 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | before: 2 | hooks: 3 | - go mod download 4 | builds: 5 | - binary: elastictl 6 | env: 7 | - CGO_ENABLED=0 8 | goos: 9 | - linux 10 | goarch: 11 | - amd64 12 | nfpms: 13 | - 14 | package_name: elastictl 15 | file_name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Arch }}" 16 | homepage: https://heckel.io/elastictl 17 | maintainer: Philipp C. Heckel 18 | description: Simple load generating and index copying tool for Elasticsearch 19 | license: Apache 2.0 20 | formats: 21 | - deb 22 | - rpm 23 | bindir: /usr/bin 24 | archives: 25 | - replacements: 26 | 386: i386 27 | amd64: x86_64 28 | checksum: 29 | name_template: 'checksums.txt' 30 | snapshot: 31 | name_template: "{{ .Tag }}-next" 32 | changelog: 33 | sort: asc 34 | filters: 35 | exclude: 36 | - '^docs:' 37 | - '^test:' 38 | dockers: 39 | - dockerfile: Dockerfile 40 | ids: 41 | - elastictl 42 | image_templates: 43 | - "binwiederhier/elastictl:latest" 44 | - "binwiederhier/elastictl:{{ .Tag }}" 45 | - "binwiederhier/elastictl:v{{ .Major }}.{{ .Minor }}" -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu 2 | MAINTAINER Philipp C. Heckel 3 | 4 | COPY elastictl /usr/bin 5 | CMD ["elastictl"] 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GO=$(shell which go) 2 | VERSION := $(shell git describe --tag) 3 | 4 | .PHONY: 5 | 6 | help: 7 | @echo "Typical commands:" 8 | @echo " make check - Run all tests, vetting/formatting checks and linters" 9 | @echo " make fmt build-snapshot install - Build latest and install to local system" 10 | @echo 11 | @echo "Test/check:" 12 | @echo " make test - Run tests" 13 | @echo " make coverage - Run tests and show coverage" 14 | @echo " make coverage-html - Run tests and show coverage (as HTML)" 15 | @echo " make coverage-upload - Upload coverage results to codecov.io" 16 | @echo 17 | @echo "Lint/format:" 18 | @echo " make fmt - Run 'go fmt'" 19 | @echo " make fmt-check - Run 'go fmt', but don't change anything" 20 | @echo " make vet - Run 'go vet'" 21 | @echo " make lint - Run 'golint'" 22 | @echo " make staticcheck - Run 'staticcheck'" 23 | @echo 24 | @echo "Build:" 25 | @echo " make build - Build" 26 | @echo " make build-snapshot - Build snapshot" 27 | @echo " make build-simple - Build (using go build, without goreleaser)" 28 | @echo " make clean - Clean build folder" 29 | @echo 30 | @echo "Releasing (requires goreleaser):" 31 | @echo " make release - Create a release" 32 | @echo " make release-snapshot - Create a test release" 33 | @echo 34 | @echo "Install locally (requires sudo):" 35 | @echo " make install - Copy binary from dist/ to /usr/bin" 36 | @echo " make install-deb - Install .deb from dist/" 37 | @echo " make install-lint - Install golint" 38 | 39 | 40 | # Test/check targets 41 | 42 | check: test fmt-check vet lint staticcheck 43 | 44 | test: .PHONY 45 | $(GO) test ./... 46 | 47 | coverage: 48 | mkdir -p build/coverage 49 | $(GO) test -race -coverprofile=build/coverage/coverage.txt -covermode=atomic ./... 50 | $(GO) tool cover -func build/coverage/coverage.txt 51 | 52 | coverage-html: 53 | mkdir -p build/coverage 54 | $(GO) test -race -coverprofile=build/coverage/coverage.txt -covermode=atomic ./... 55 | $(GO) tool cover -html build/coverage/coverage.txt 56 | 57 | coverage-upload: 58 | cd build/coverage && (curl -s https://codecov.io/bash | bash) 59 | 60 | # Lint/formatting targets 61 | 62 | fmt: 63 | $(GO) fmt ./... 64 | 65 | fmt-check: 66 | test -z $(shell gofmt -l .) 67 | 68 | vet: 69 | $(GO) vet ./... 70 | 71 | lint: 72 | which golint || $(GO) get -u golang.org/x/lint/golint 73 | $(GO) list ./... | grep -v /vendor/ | xargs -L1 golint -set_exit_status 74 | 75 | staticcheck: .PHONY 76 | rm -rf build/staticcheck 77 | which staticcheck || go get honnef.co/go/tools/cmd/staticcheck 78 | mkdir -p build/staticcheck 79 | ln -s "$(GO)" build/staticcheck/go 80 | PATH="$(PWD)/build/staticcheck:$(PATH)" staticcheck ./... 81 | rm -rf build/staticcheck 82 | 83 | # Building targets 84 | 85 | build: .PHONY 86 | goreleaser build --rm-dist 87 | 88 | build-snapshot: 89 | goreleaser build --snapshot --rm-dist 90 | 91 | build-simple: clean 92 | mkdir -p dist/elastictl_linux_amd64 93 | $(GO) build \ 94 | -o dist/elastictl_linux_amd64/elastictl \ 95 | -ldflags \ 96 | "-s -w -X main.version=$(VERSION) -X main.commit=$(shell git rev-parse --short HEAD) -X main.date=$(shell date +%s)" 97 | 98 | clean: .PHONY 99 | rm -rf dist build dist 100 | 101 | 102 | # Releasing targets 103 | 104 | release: 105 | goreleaser release --rm-dist 106 | 107 | release-snapshot: 108 | goreleaser release --snapshot --skip-publish --rm-dist 109 | 110 | 111 | # Installing targets 112 | 113 | install: 114 | sudo rm -f /usr/bin/elastictl 115 | sudo cp -a dist/elastictl_linux_amd64/elastictl /usr/bin/elastictl 116 | 117 | install-deb: 118 | sudo systemctl stop elastictl || true 119 | sudo apt-get purge elastictl || true 120 | sudo dpkg -i dist/*.deb 121 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # elastictl 2 | 3 | Simple tool to import/export Elasticsearch indices into a file, and/or reshard an index. The tool can be used for: 4 | 5 | * Backup/restore of an Elasticsearch index 6 | * Performance test an Elasticsearch cluster (import with high concurrency, see `--workers`) 7 | * Change the shard/replica count of an index (see `reshard` subcomment) 8 | 9 | In my local cluster, I was able to import ~10k documents per second. 10 | 11 | ## Build 12 | ``` 13 | $ go build 14 | ``` 15 | 16 | Or via goreleaser: 17 | ``` 18 | $ make [build | build-snapshot] 19 | ``` 20 | 21 | ## Installation 22 | For Debian/Ubuntu 23 | ``` 24 | wget https://github.com/binwiederhier/elastictl/releases/download/v0.0.5/elastictl_0.0.5_amd64.deb 25 | dpkg -i *.deb 26 | ``` 27 | 28 | All others: 29 | ``` 30 | wget https://github.com/binwiederhier/elastictl/releases/download/v0.0.5/elastictl_0.0.5_linux_x86_64.tar.gz 31 | tar zxvf elastictl_0.0.5_linux_x86_64.tar.gz 32 | ./elastictl 33 | ``` 34 | 35 | ## Usage: 36 | 37 | ### Export/dump an index to a file 38 | The first line of the output format is the mapping, the rest are the documents. 39 | ``` 40 | # Entire index 41 | elastictl export dummy > dummy.json 42 | 43 | # Only a subset of documents 44 | elastictl export \ 45 | --search '{"query":{"bool":{"must_not":{"match":{"eventType":"Success"}}}}}' \ 46 | dummy > dummy.json 47 | ``` 48 | 49 | ### Import to new index 50 | ``` 51 | # With high concurrency 52 | cat dummy.json | elastictl import --workers 100 dummy-copy 53 | ``` 54 | 55 | ### Reshard (import/export) an index 56 | This commands export the index `dummy` to `dummy.json` and re-imports it as `dummy` using a different number of shards. 57 | This command does `DELETE` the index after exporting it! 58 | ``` 59 | elastictl reshard \ 60 | --search '{"query":{"bool":{"must_not":{"match":{"eventType":"Success"}}}}}' \ 61 | --shards 1 \ 62 | --replicas 1 \ 63 | dummy 64 | ``` 65 | -------------------------------------------------------------------------------- /cmd/app.go: -------------------------------------------------------------------------------- 1 | // Package cmd provides the elastictl CLI application 2 | package cmd 3 | 4 | import ( 5 | "github.com/urfave/cli/v2" 6 | "os" 7 | ) 8 | 9 | // New creates a new CLI application 10 | func New() *cli.App { 11 | return &cli.App{ 12 | Name: "elastictl", 13 | Usage: "Elasticsearch toolkit", 14 | UsageText: "elastictl COMMAND [OPTION..] [ARG..]", 15 | HideHelp: true, 16 | HideVersion: true, 17 | EnableBashCompletion: true, 18 | UseShortOptionHandling: true, 19 | Reader: os.Stdin, 20 | Writer: os.Stdout, 21 | ErrWriter: os.Stderr, 22 | Commands: []*cli.Command{ 23 | cmdExport, 24 | cmdBlast, 25 | cmdReshard, 26 | }, 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /cmd/export.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/urfave/cli/v2" 5 | "heckel.io/elastictl/tools" 6 | ) 7 | 8 | var cmdExport = &cli.Command{ 9 | Name: "export", 10 | Aliases: []string{"e"}, 11 | Usage: "Export an entire index to STDOUT", 12 | UsageText: "elastictl export INDEX", 13 | Action: execExport, 14 | Flags: []cli.Flag{ 15 | &cli.StringFlag{Name: "host", Aliases: []string{"H"}, Value: "localhost:9200", DefaultText: "localhost:9200", Usage: "override default host"}, 16 | &cli.StringFlag{Name: "search", Aliases: []string{"q"}, Value: "", Usage: "only dump documents matching the given ES query"}, 17 | }, 18 | } 19 | 20 | func execExport(c *cli.Context) error { 21 | host := c.String("host") 22 | search := c.String("search") 23 | if c.NArg() < 1 { 24 | return cli.Exit("invalid syntax: index missing", 1) 25 | } 26 | index := c.Args().Get(0) 27 | _, err := tools.Export(host, index, search, c.App.Writer) 28 | return err 29 | } 30 | -------------------------------------------------------------------------------- /cmd/import.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/urfave/cli/v2" 5 | "heckel.io/elastictl/tools" 6 | ) 7 | 8 | var cmdBlast = &cli.Command{ 9 | Name: "import", 10 | Aliases: []string{"i"}, 11 | Usage: "Write to ES index from STDIN", 12 | UsageText: "elastictl import INDEX", 13 | Action: execImport, 14 | Flags: []cli.Flag{ 15 | &cli.StringFlag{Name: "host", Aliases: []string{"H"}, Value: "localhost:9200", DefaultText: "localhost:9200", Usage: "override default host"}, 16 | &cli.IntFlag{Name: "workers", Aliases: []string{"w"}, Value: 50, Usage: "number of concurrent workers"}, 17 | &cli.IntFlag{Name: "shards", Aliases: []string{"s"}, Value: -1, DefaultText: "no change", Usage: "override the number of shards on index creation"}, 18 | &cli.IntFlag{Name: "replicas", Aliases: []string{"r"}, Value: -1, DefaultText: "no change", Usage: "override the number of replicas on index creation"}, 19 | &cli.BoolFlag{Name: "no-create", Aliases: []string{"N"}, Value: false, Usage: "do not create index"}, 20 | }, 21 | } 22 | 23 | func execImport(c *cli.Context) error { 24 | host := c.String("host") 25 | workers := c.Int("workers") 26 | nocreate := c.Bool("no-create") 27 | shards := c.Int("shards") 28 | replicas := c.Int("replicas") 29 | if c.NArg() < 1 { 30 | return cli.Exit("invalid syntax: index missing", 1) 31 | } 32 | index := c.Args().Get(0) 33 | _, err := tools.Import(host, index, workers, nocreate, shards, replicas, c.App.Reader, -1) 34 | return err 35 | } 36 | -------------------------------------------------------------------------------- /cmd/reshard.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/urfave/cli/v2" 5 | "heckel.io/elastictl/tools" 6 | ) 7 | 8 | var cmdReshard = &cli.Command{ 9 | Name: "reshard", 10 | Aliases: []string{"r"}, 11 | Usage: "Reshard index using different shard/replica counts", 12 | UsageText: "elastictl reshard INDEX", 13 | Action: execReshard, 14 | Flags: []cli.Flag{ 15 | &cli.StringFlag{Name: "host", Aliases: []string{"H"}, Value: "localhost:9200", DefaultText: "localhost:9200", Usage: "override default host"}, 16 | &cli.StringFlag{Name: "search", Aliases: []string{"q"}, Value: "", Usage: "only dump documents matching the given ES query"}, 17 | &cli.StringFlag{Name: "dir", Aliases: []string{"d"}, DefaultText: "current directory", Usage: "directory used to store exported index file"}, 18 | &cli.BoolFlag{Name: "no-keep", Aliases: []string{"K"}, Usage: "delete index file after successful import"}, 19 | &cli.IntFlag{Name: "workers", Aliases: []string{"w"}, Value: 50, Usage: "number of concurrent workers"}, 20 | &cli.IntFlag{Name: "shards", Aliases: []string{"s"}, Value: -1, DefaultText: "no change", Usage: "override the number of shards on index creation"}, 21 | &cli.IntFlag{Name: "replicas", Aliases: []string{"r"}, Value: -1, DefaultText: "no change", Usage: "override the number of replicas on index creation"}, 22 | }, 23 | } 24 | 25 | func execReshard(c *cli.Context) error { 26 | host := c.String("host") 27 | search := c.String("search") 28 | dir := c.String("dir") 29 | keep := !c.Bool("no-keep") 30 | workers := c.Int("workers") 31 | shards := c.Int("shards") 32 | replicas := c.Int("replicas") 33 | if c.NArg() < 1 { 34 | return cli.Exit("invalid syntax: index missing", 1) 35 | } 36 | index := c.Args().Get(0) 37 | return tools.Reshard(host, index, dir, keep, search, workers, shards, replicas) 38 | } 39 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module heckel.io/elastictl 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect 7 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 8 | github.com/tidwall/gjson v1.13.0 9 | github.com/tidwall/sjson v1.2.4 10 | github.com/urfave/cli/v2 v2.3.0 11 | ) 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 4 | github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= 5 | github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 6 | github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= 7 | github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 8 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 9 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 10 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 11 | github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= 12 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 13 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 14 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 15 | github.com/tidwall/gjson v1.6.8 h1:CTmXMClGYPAmln7652e69B7OLXfTi5ABcPPwjIWUv7w= 16 | github.com/tidwall/gjson v1.6.8/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI= 17 | github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 18 | github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M= 19 | github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 20 | github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= 21 | github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 22 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 23 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 24 | github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= 25 | github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 26 | github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= 27 | github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 28 | github.com/tidwall/sjson v1.1.5 h1:wsUceI/XDyZk3J1FUvuuYlK62zJv2HO2Pzb8A5EWdUE= 29 | github.com/tidwall/sjson v1.1.5/go.mod h1:VuJzsZnTowhSxWdOgsAnb886i4AjEyTkk7tNtsL7EYE= 30 | github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= 31 | github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= 32 | github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= 33 | github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= 34 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 35 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 36 | gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI= 37 | gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 38 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/urfave/cli/v2" 6 | "heckel.io/elastictl/cmd" 7 | "os" 8 | "runtime" 9 | ) 10 | 11 | var ( 12 | version = "dev" 13 | commit = "unknown" 14 | date = "unknown" 15 | ) 16 | 17 | func main() { 18 | cli.AppHelpTemplate += fmt.Sprintf(` 19 | Try 'elastictl COMMAND --help' for more information. 20 | 21 | elastictl %s (%s), runtime %s, built at %s 22 | Copyright (C) 2021 Philipp C. Heckel, distributed under the Apache License 2.0 23 | `, version, commit[:7], runtime.Version(), date) 24 | 25 | app := cmd.New() 26 | app.Version = version 27 | 28 | if err := app.Run(os.Args); err != nil { 29 | fmt.Fprintln(os.Stderr, err.Error()) 30 | os.Exit(1) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /tools/export.go: -------------------------------------------------------------------------------- 1 | package tools 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/tidwall/gjson" 7 | "heckel.io/elastictl/util" 8 | "io" 9 | "io/ioutil" 10 | "log" 11 | "net/http" 12 | "os" 13 | "strings" 14 | ) 15 | 16 | func Export(host string, index string, search string, w io.Writer) (int, error) { 17 | log.Printf("exporting index %s/%s", host, index) 18 | rootURI := fmt.Sprintf("http://%s", host) 19 | 20 | // Dump mapping first 21 | rootIndexURI := fmt.Sprintf("http://%s/%s", host, index) 22 | req, err := http.NewRequest("GET", rootIndexURI, nil) 23 | req.Header.Add("Content-Type", "application/json") 24 | if err != nil { 25 | return 0, err 26 | } 27 | resp, err := client.Do(req) 28 | if err != nil { 29 | return 0, err 30 | } 31 | rawMapping, err := ioutil.ReadAll(resp.Body) 32 | if err != nil { 33 | return 0, err 34 | } 35 | mapping := gjson.GetBytes(rawMapping, index).String() 36 | if _, err := fmt.Fprintln(w, mapping); err != nil { 37 | return 0, err 38 | } 39 | 40 | // Initial search request 41 | var body io.Reader 42 | if search != "" { 43 | body = strings.NewReader(search) 44 | } 45 | uri := fmt.Sprintf("%s/_search?size=10000&scroll=1m", rootIndexURI) 46 | req, err = http.NewRequest("POST", uri, body) 47 | req.Header.Add("Content-Type", "application/json") 48 | if err != nil { 49 | return 0, err 50 | } 51 | resp, err = client.Do(req) 52 | if err != nil { 53 | return 0, err 54 | } 55 | if resp.Body == nil { 56 | return 0, err 57 | } 58 | 59 | var progress *util.ProgressBar 60 | exported := 0 61 | 62 | for { 63 | body, err := ioutil.ReadAll(resp.Body) 64 | if err != nil { 65 | return 0, err 66 | } 67 | 68 | if progress == nil { 69 | total := gjson.GetBytes(body, "hits.total") 70 | if !total.Exists() { 71 | return 0, errors.New("no total") 72 | } 73 | progress = util.NewProgressBarWithTotal(os.Stderr, int(total.Int())) 74 | } 75 | 76 | scrollID := gjson.GetBytes(body, "_scroll_id") 77 | if !scrollID.Exists() { 78 | return 0, errors.New("no scroll id: " + string(body)) 79 | } 80 | 81 | hits := gjson.GetBytes(body, "hits.hits") 82 | if !hits.Exists() || !hits.IsArray() { 83 | return 0, errors.New("no hits: " + string(body)) 84 | } 85 | if len(hits.Array()) == 0 { 86 | break // we're done! 87 | } 88 | 89 | for _, hit := range hits.Array() { 90 | exported++ 91 | progress.Add(int64(len(hit.Raw))) 92 | if _, err := fmt.Fprintln(w, hit.Raw); err != nil { 93 | return 0, err 94 | } 95 | } 96 | 97 | uri := fmt.Sprintf("%s/_search/scroll", rootURI) 98 | postBody := fmt.Sprintf(`{"scroll":"1m","scroll_id":"%s"}`, scrollID.String()) 99 | req, err := http.NewRequest("POST", uri, strings.NewReader(postBody)) 100 | req.Header.Add("Content-Type", "application/json") 101 | if err != nil { 102 | return 0, err 103 | } 104 | 105 | resp, err = client.Do(req) 106 | if err != nil { 107 | return 0, err 108 | } 109 | 110 | if resp.Body == nil { 111 | return 0, err 112 | } 113 | } 114 | progress.Done() 115 | return exported, nil 116 | } 117 | -------------------------------------------------------------------------------- /tools/import.go: -------------------------------------------------------------------------------- 1 | package tools 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "fmt" 7 | "github.com/tidwall/gjson" 8 | "github.com/tidwall/sjson" 9 | "heckel.io/elastictl/util" 10 | "io" 11 | "io/ioutil" 12 | "log" 13 | "net/http" 14 | "net/url" 15 | "os" 16 | "strings" 17 | "sync" 18 | "sync/atomic" 19 | ) 20 | 21 | var ( 22 | client = &http.Client{} 23 | settingsToRemove = []string{"settings.index.creation_date", "settings.index.uuid", "settings.index.version", "settings.index.provided_name"} 24 | errTemporaryFailure = errors.New("temporary failure") 25 | ) 26 | 27 | func Import(host string, index string, workers int, nocreate bool, shards int, replicas int, r io.Reader, totalHint int) (int, error) { 28 | log.Printf("importing index %s/%s", host, index) 29 | rootURI := fmt.Sprintf("http://%s/%s", host, index) 30 | scanner := bufio.NewScanner(r) 31 | 32 | // Create index 33 | if !scanner.Scan() { 34 | return 0, errors.New("cannot read mapping") 35 | } 36 | mapping := scanner.Text() 37 | if !nocreate { 38 | var err error 39 | for _, keyToRemove := range settingsToRemove { 40 | mapping, err = sjson.Delete(mapping, keyToRemove) 41 | if err != nil { 42 | return 0, err 43 | } 44 | } 45 | if shards > 0 { 46 | mapping, err = sjson.Set(mapping, "settings.index.number_of_shards", fmt.Sprintf("%d", shards)) 47 | if err != nil { 48 | return 0, err 49 | } 50 | } 51 | if replicas > -1 { // zero replicas is allowed! 52 | mapping, err = sjson.Set(mapping, "settings.index.number_of_replicas", fmt.Sprintf("%d", replicas)) 53 | if err != nil { 54 | return 0, err 55 | } 56 | } 57 | req, err := http.NewRequest("PUT", rootURI, strings.NewReader(mapping)) 58 | req.Header.Add("Content-Type", "application/json") 59 | if err != nil { 60 | return 0, err 61 | } 62 | resp, err := client.Do(req) 63 | if err != nil { 64 | return 0, err 65 | } 66 | if resp.StatusCode == 400 || resp.StatusCode == 503 { 67 | return 0, errTemporaryFailure // special case: 400 returned when index already exists, 503 when the cluster is overloaded 68 | } else if resp.StatusCode != 201 && resp.StatusCode != 200 { 69 | return 0, fmt.Errorf("unexpected response code during index creation: %d", resp.StatusCode) 70 | } 71 | } 72 | 73 | // Start workers 74 | http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = workers // Avoid opening/closing connections 75 | 76 | wg := &sync.WaitGroup{} 77 | docsChan := make(chan string) 78 | progress := util.NewProgressBarWithTotal(os.Stderr, totalHint) 79 | imported := int64(0) 80 | for i := 0; i < workers; i++ { 81 | wg.Add(1) 82 | go importWorker(wg, docsChan, progress, client, rootURI, &imported) 83 | } 84 | 85 | go func() { 86 | for scanner.Scan() { 87 | docsChan <- scanner.Text() 88 | } 89 | close(docsChan) 90 | }() 91 | 92 | wg.Wait() 93 | progress.Done() 94 | 95 | return int(imported), nil 96 | } 97 | 98 | func importWorker(wg *sync.WaitGroup, docsChan chan string, progress *util.ProgressBar, client *http.Client, rootURI string, imported *int64) { 99 | defer wg.Done() 100 | for doc := range docsChan { 101 | id := url.QueryEscape(gjson.Get(doc, "_id").String()) 102 | dtype := gjson.Get(doc, "_type").String() 103 | source := gjson.Get(doc, "_source").String() 104 | uri := fmt.Sprintf("%s/%s/%s", rootURI, dtype, id) 105 | req, err := http.NewRequest("PUT", uri, strings.NewReader(source)) 106 | req.Header.Add("Content-Type", "application/json") 107 | if err != nil { 108 | fmt.Printf("error creating PUT request: %s\n", err.Error()) 109 | continue 110 | } 111 | resp, err := client.Do(req) 112 | if err != nil { 113 | fmt.Printf("PUT failed: %s\n", err.Error()) 114 | continue 115 | } 116 | if resp.StatusCode != 201 && resp.StatusCode != 200 { 117 | fmt.Printf("PUT returned unexpected response: %d\n", resp.StatusCode) 118 | continue 119 | } 120 | if resp.Body != nil { 121 | io.Copy(ioutil.Discard, resp.Body) 122 | resp.Body.Close() 123 | } 124 | progress.Add(int64(len(source))) 125 | atomic.AddInt64(imported, 1) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /tools/reshard.go: -------------------------------------------------------------------------------- 1 | package tools 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "log" 8 | "net/http" 9 | "os" 10 | "path/filepath" 11 | "time" 12 | ) 13 | 14 | func Reshard(host string, index string, dir string, keep bool, search string, workers int, shards int, replicas int) error { 15 | if dir == "" { 16 | var err error 17 | dir, err = os.Getwd() 18 | if err != nil { 19 | return err 20 | } 21 | } 22 | filename := filepath.Join(dir, fmt.Sprintf("%s.json", index)) 23 | file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600) 24 | if err != nil { 25 | return err 26 | } 27 | defer file.Close() 28 | exported, err := Export(host, index, search, file) 29 | if err != nil { 30 | return err 31 | } 32 | if _, err := file.Seek(0, 0); err != nil { 33 | return err 34 | } 35 | lines, err := lineCounter(file) 36 | if err != nil { 37 | return err 38 | } 39 | if exported != lines-1 { 40 | return fmt.Errorf("unexpected count: %d documents expected in exported file, got %d", exported, lines-1) 41 | } 42 | for i := 0; ; i++ { 43 | if _, err := file.Seek(0, 0); err != nil { 44 | return err 45 | } 46 | if err := deleteIndex(host, index); err != nil { 47 | return err 48 | } 49 | imported, err := Import(host, index, workers, false, shards, replicas, file, exported) 50 | if err == errTemporaryFailure && i < 10 { 51 | // retry on temporary failures up to 10 times; races on index creation do happen when 52 | // the index is busy and auto-creation of the index is turned on. 53 | time.Sleep(time.Duration(i) * time.Second) 54 | continue 55 | } else if err != nil { 56 | return err 57 | } 58 | if imported != exported { 59 | return fmt.Errorf("count mismatch: %d documents exported, but %d imported", exported, imported) 60 | } 61 | break 62 | } 63 | if !keep { 64 | file.Close() 65 | os.Remove(filename) 66 | } 67 | log.Printf("resharding complete") 68 | return nil 69 | } 70 | 71 | func deleteIndex(host, index string) error { 72 | indexURI := fmt.Sprintf("http://%s/%s", host, index) 73 | req, err := http.NewRequest("DELETE", indexURI, nil) 74 | req.Header.Add("Content-Type", "application/json") 75 | if err != nil { 76 | return err 77 | } 78 | resp, err := client.Do(req) 79 | if err != nil { 80 | return err 81 | } 82 | defer resp.Body.Close() 83 | responseBody, err := io.ReadAll(resp.Body) 84 | if err != nil { 85 | return err 86 | } 87 | if resp.StatusCode != 201 && resp.StatusCode != 200 { 88 | return fmt.Errorf("unexpected response code during delete call: %d %s\n", resp.StatusCode, string(responseBody)) 89 | } 90 | return nil 91 | } 92 | 93 | func lineCounter(r io.Reader) (int, error) { 94 | // From: https://stackoverflow.com/questions/24562942/golang-how-do-i-determine-the-number-of-lines-in-a-file-efficiently 95 | buf := make([]byte, 32*1024) 96 | count := 0 97 | lineSep := []byte{'\n'} 98 | 99 | for { 100 | c, err := r.Read(buf) 101 | count += bytes.Count(buf[:c], lineSep) 102 | 103 | switch { 104 | case err == io.EOF: 105 | return count, nil 106 | 107 | case err != nil: 108 | return count, err 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /util/progress.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | var spinner = []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"} 12 | 13 | type ProgressBar struct { 14 | started time.Time 15 | writer io.Writer 16 | count int 17 | total int 18 | size int64 19 | rendered time.Time 20 | rendercount int64 21 | prevlen int 22 | mu sync.Mutex 23 | } 24 | 25 | func NewProgressBar(writer io.Writer) *ProgressBar { 26 | return NewProgressBarWithTotal(writer, 0) 27 | } 28 | 29 | func NewProgressBarWithTotal(writer io.Writer, total int) *ProgressBar { 30 | return &ProgressBar{ 31 | started: time.Now(), 32 | writer: writer, 33 | total: total, 34 | } 35 | } 36 | 37 | func (p *ProgressBar) Add(size int64) { 38 | p.mu.Lock() 39 | defer p.mu.Unlock() 40 | p.count++ 41 | p.size += size 42 | if time.Since(p.rendered) > 65*time.Millisecond { 43 | p.render(false) 44 | } 45 | } 46 | 47 | func (p *ProgressBar) Done() { 48 | p.mu.Lock() 49 | defer p.mu.Unlock() 50 | p.render(true) 51 | } 52 | 53 | func (p *ProgressBar) render(done bool) { 54 | spin := spinner[p.rendercount%int64(len(spinner))] 55 | count := p.count 56 | countPerSec := float64(p.count) / time.Since(p.started).Seconds() 57 | size := bytesToHuman(p.size) 58 | sizePerSec := bytesToHuman(int64(float64(p.size) / time.Since(p.started).Seconds())) 59 | now := time.Now().Format("2006/01/02 15:04:05") 60 | if done { 61 | line := fmt.Sprintf("\r%s complete: %d docs (%.1f docs/s), %s (%s/s)", now, count, countPerSec, size, sizePerSec) 62 | fmt.Fprint(p.writer, line) 63 | if p.prevlen > len(line) { 64 | fmt.Fprint(p.writer, strings.Repeat(" ", p.prevlen-len(line))) 65 | } 66 | fmt.Fprintln(p.writer) 67 | p.prevlen = len(line) 68 | } else { 69 | percent := "" 70 | if p.total > 0 { 71 | percent = fmt.Sprintf(" %.1f%%,", float64(count)/float64(p.total)*100) 72 | } 73 | line := fmt.Sprintf("\r%s %s processing:%s %d docs (%.1f docs/s), %s (%s/s)", now, spin, percent, count, countPerSec, size, sizePerSec) 74 | fmt.Fprint(p.writer, line) 75 | if p.prevlen > len(line) { 76 | fmt.Fprint(p.writer, strings.Repeat(" ", p.prevlen-len(line))) 77 | } 78 | p.prevlen = len(line) 79 | } 80 | p.rendered = time.Now() 81 | p.rendercount++ 82 | } 83 | 84 | func bytesToHuman(b int64) string { 85 | // From: https://yourbasic.org/golang/formatting-byte-size-to-human-readable-format/ 86 | const unit = 1024 87 | if b < unit { 88 | return fmt.Sprintf("%d B", b) 89 | } 90 | div, exp := int64(unit), 0 91 | for n := b / unit; n >= unit; n /= unit { 92 | div *= unit 93 | exp++ 94 | } 95 | return fmt.Sprintf("%.1f %cB", 96 | float64(b)/float64(div), "kMGTPE"[exp]) 97 | } 98 | --------------------------------------------------------------------------------