├── .editorconfig ├── .github └── workflows │ └── test.yml ├── .gitignore ├── LICENSE ├── README.md ├── diff ├── differ.go ├── differ_test.go └── printer │ ├── json.go │ ├── printer.go │ └── std.go ├── docker-compose.kibana.yml ├── docker-compose.yml ├── elastic ├── client.go ├── config │ └── config.go ├── v5 │ └── client.go ├── v6 │ └── client.go └── v7 │ └── client.go ├── etc ├── elasticsearch5 │ └── elasticsearch.yml ├── elasticsearch6 │ └── elasticsearch.yml ├── elasticsearch7 │ └── elasticsearch.yml ├── kibana5 │ └── kibana.yml ├── kibana6 │ └── kibana.yml └── kibana7 │ └── kibana.yml ├── go.mod ├── go.sum ├── main.go └── seed └── 01.sh /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [Makefile**] 13 | indent_style = tab 14 | indent_size = 2 15 | 16 | [*.md] 17 | trim_trailing_whitespace = false 18 | 19 | [*.go] 20 | indent_style = tab 21 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: [push, pull_request] 3 | jobs: 4 | test: 5 | strategy: 6 | matrix: 7 | go: [1.18.x] 8 | os: [ubuntu-latest] 9 | name: Run ${{ matrix.go }} on ${{ matrix.os }} 10 | runs-on: ${{ matrix.os }} 11 | steps: 12 | - name: Prepare tests 13 | run: | 14 | sudo apt-get install -y netcat 15 | sudo sysctl -w vm.max_map_count=262144 16 | 17 | - name: Setup Go ${{ matrix.go }} 18 | uses: actions/setup-go@v2 19 | with: 20 | go-version: ${{ matrix.go }} 21 | 22 | - name: Checkout code 23 | uses: actions/checkout@v2 24 | 25 | - uses: actions/cache@v2 26 | with: 27 | path: | 28 | ~/.cache/go-build 29 | ~/go/pkg/mod 30 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 31 | restore-keys: | 32 | ${{ runner.os }}-go- 33 | 34 | - name: Run Docker containers 35 | timeout-minutes: 1 36 | run: docker compose up -d --wait 37 | 38 | - name: Check Docker containers 39 | run: docker ps -a 40 | 41 | - name: Get dependencies 42 | run: | 43 | go get -u github.com/google/go-cmp/cmp 44 | go get -u github.com/fortytw2/leaktest 45 | go mod tidy 46 | 47 | - name: Run the tests 48 | timeout-minutes: 5 49 | run: | 50 | go test -race -v ./... 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | .DS_Store 20 | 21 | _testmain.go 22 | 23 | *.exe 24 | *.test 25 | *.prof 26 | *.pid 27 | *.coverprofile 28 | *.log 29 | .DS_Store 30 | .idea/ 31 | .vscode/ 32 | 33 | /data 34 | /*.json 35 | /esdiff 36 | /tmp/ 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright © 2018-present Oliver Eilhard 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the “Software”), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included 12 | in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 20 | IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Diff for Elasticsearch 2 | 3 | **Warning: This is a work-in-progress. Things might break without warning.** 4 | 5 | The `esdiff` tool iterates over two indices in Elasticsearch 5.x, 6.x or 7.x 6 | and performs a diff between the documents in those indices. 7 | 8 | It does so by scrolling over the indices. To allow for a stable sort 9 | order, it uses `_id` by default (`_uid` in ES 5.x). 10 | 11 | You need Go 1.11 or later to compile. Install with: 12 | 13 | ```sh 14 | go install github.com/olivere/esdiff@latest 15 | ``` 16 | 17 | ## Example usage 18 | 19 | First, we need to setup two Elasticsearch clusters for testing, 20 | then seed a few documents. 21 | 22 | ```sh 23 | $ mkdir -p data 24 | 25 | # Create an Elasticsearch 5.x cluster on http://localhost:19200 26 | # Create an Elasticsearch 6.x cluster on http://localhost:29200 27 | # Create an Elasticsearch 7.x cluster on http://localhost:39200 28 | 29 | # Increase your docker memory limit (6.0GiB) in Docker App > Preferences > Advanced. 30 | $ docker-compose up -d 31 | 32 | Creating esdiff_elasticsearch5_1 ... done 33 | Creating esdiff_elasticsearch6_1 ... done 34 | Creating esdiff_elasticsearch7_1 ... done 35 | 36 | # Check docker containers 37 | $ docker-compose ps 38 | Name Command State Ports 39 | ---------------------------------------------------------------------------------------------------- 40 | esdiff_elasticsearch5_1 /bin/bash bin/es-docker Up 0.0.0.0:19200->9200/tcp, 9300/tcp 41 | esdiff_elasticsearch6_1 /usr/local/bin/docker-entr ... Up 0.0.0.0:29200->9200/tcp, 9300/tcp 42 | esdiff_elasticsearch7_1 /usr/local/bin/docker-entr ... Up 0.0.0.0:39200->9200/tcp, 9300/tcp 43 | 44 | # Check docker container logs 45 | $ docker-compose logs -f elasticsearch5 46 | Attaching to esdiff_elasticsearch5_1 47 | elasticsearch5_1 | [2019-07-02T14:17:33,351][WARN ][o.e.b.JNANatives ] Unable to lock JVM Memory: error=12, reason=Cannot allocate memory 48 | elasticsearch5_1 | [2019-07-02T14:17:33,355][WARN ][o.e.b.JNANatives ] This can result in part of the JVM being swapped out. 49 | elasticsearch5_1 | [2019-07-02T14:17:33,355][WARN ][o.e.b.JNANatives ] Increase RLIMIT_MEMLOCK, soft limit: 83968000, hard limit: 83968000 50 | elasticsearch5_1 | [2019-07-02T14:17:33,356][WARN ][o.e.b.JNANatives ] These can be adjusted by modifying /etc/security/limits.conf, for example: 51 | elasticsearch5_1 | # allow user 'elasticsearch' mlockall 52 | ........ 53 | 54 | # Add some documents 55 | $ ./seed/01.sh 56 | 57 | # Compile 58 | $ go build 59 | ``` 60 | 61 | Let's make a simple diff: 62 | 63 | ### Examples 64 | 65 | Same cluster and same documents should return only unchanged documents: 66 | 67 | ```sh 68 | $ ./esdiff -u=true 'http://localhost:19200/index01/tweet' 'http://localhost:19200/index01/tweet' 69 | Unchanged 1 70 | Unchanged 2 71 | Unchanged 3 72 | ``` 73 | 74 | The following example will return a diff between indices in ES 5.x and ES 6.x: 75 | 76 | ```sh 77 | $ ./esdiff -u=true 'http://localhost:19200/index01/tweet' 'http://localhost:29200/index01/_doc' 78 | Unchanged 1 79 | Deleted 2 80 | Updated 3 {*diff.Document}.Source["message"]: 81 | -: "Playing the piano is fun as well" 82 | +: "Playing the guitar is fun as well" 83 | 84 | Created 4 {*diff.Document}: 85 | -: (*diff.Document)(nil) 86 | +: &diff.Document{ID: "4", Source: map[string]interface {}{"message": "Climbed that mountain", "user": "sandrae"}} 87 | ``` 88 | 89 | ES 5.x and ES 7.x—different documents—again: 90 | 91 | ```sh 92 | $ ./esdiff -u=true 'http://localhost:19200/index01/tweet' 'http://localhost:39200/index01/_doc' 93 | Unchanged 1 94 | Deleted 2 95 | Updated 3 {*diff.Document}.Source["message"]: 96 | -: "Playing the piano is fun as well" 97 | +: "Playing the flute, oh boy" 98 | 99 | Created 5 {*diff.Document}: 100 | -: (*diff.Document)(nil) 101 | +: &diff.Document{ID: "5", Source: map[string]interface {}{"message": "Ran that marathon", "user": "sandrae"}} 102 | ``` 103 | 104 | ### Output options 105 | 106 | Notice that you can pass additional options to filter for 107 | the kind of modes that you're interested in. E.g. if you also 108 | want to see all unchanged documents but not those that were 109 | deleted, use `-u=true -d=false`: 110 | 111 | ```sh 112 | $ ./esdiff -u=true -d=false 'http://localhost:19200/index01/tweet' 'http://localhost:29200/index01/_doc' 113 | Unchanged 1 114 | Updated 3 {*diff.Document}.Source["message"]: 115 | -: "Playing the piano is fun as well" 116 | +: "Playing the guitar is fun as well" 117 | 118 | Created 4 {*diff.Document}: 119 | -: (*diff.Document)(nil) 120 | +: &diff.Document{ID: "4", Source: map[string]interface {}{"message": "Climbed that mountain", "user": "sandrae"}} 121 | ``` 122 | 123 | ### Formatting options 124 | 125 | Use JSON as output format instead. Together with 126 | [`jq`](https://stedolan.github.io/jq/) 127 | and 128 | [`jiq`](https://github.com/fiatjaf/jiq) 129 | this is quite powerful 130 | (among [other jq-related tools](https://github.com/fiatjaf/awesome-jq)). 131 | 132 | ```sh 133 | $ ./esdiff -o=json 'http://localhost:29200/index01/_doc' 'http://localhost:39200/index01/_doc' | jq 'select(.mode | contains("deleted"))' 134 | { 135 | "mode": "deleted", 136 | "_id": "4", 137 | "src": { 138 | "_id": "4", 139 | "_source": { 140 | "message": "Climbed that mountain", 141 | "user": "sandrae" 142 | } 143 | }, 144 | "dst": null 145 | } 146 | ``` 147 | 148 | ### Filtering options 149 | 150 | You can also pass a query to filter the source and/or the destination, 151 | using the `-sf` and `-df` args respectively: 152 | 153 | ```sh 154 | $ $ ./esdiff -o=json -sf='{"term":{"user":"olivere"}}' 'http://localhost:29200/index01/_doc' 'http://localhost:19200/index01/_doc' 155 | {"mode":"deleted","_id":"1","src":{"_id":"1","_source":{"message":"Welcome to Golang","user":"olivere"}},"dst":null} 156 | ``` 157 | 158 | ### All options 159 | 160 | Use `-h` to display all options: 161 | 162 | ```sh 163 | $ ./esdiff -h 164 | General usage: 165 | 166 | esdiff [flags] 167 | 168 | General flags: 169 | -a Print added docs (default true) 170 | -c Print changed docs (default true) 171 | -d Print deleted docs (default true) 172 | -df string 173 | Raw query for filtering the destination, e.g. {"term":{"name.keyword":"Oliver"}} 174 | -dsort string 175 | Field to sort the destination, e.g. "id" or "-id" (prepend with - for descending) 176 | -exclude string 177 | Raw source filter for excluding certain fields from the source, e.g. "hash_value,sub.*" 178 | -include string 179 | Raw source filter for including certain fields from the source, e.g. "obj.*" 180 | -o string 181 | Output format, e.g. json 182 | -sf string 183 | Raw query for filtering the source, e.g. {"term":{"user":"olivere"}} 184 | -size int 185 | Batch size (default 100) 186 | -ssort string 187 | Field to sort the source, e.g. "id" or "-id" (prepend with - for descending) 188 | -u Print unchanged docs 189 | -replace-with string 190 | Replace the id in the document with the unique field you need from the source,e.g. "unique_key" 191 | ``` 192 | 193 | ## License 194 | 195 | MIT. See [LICENSE](https://github.com/olivere/esdiff/blob/master/LICENSE). 196 | -------------------------------------------------------------------------------- /diff/differ.go: -------------------------------------------------------------------------------- 1 | package diff 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | ) 8 | 9 | // Document is a generic document retrieved from Elasticsearch. 10 | type Document struct { 11 | ID string `json:"_id,omitempty"` 12 | Source map[string]interface{} `json:"_source,omitempty"` 13 | } 14 | 15 | // Mode describes the outcome of comparing two documents. 16 | type Mode int 17 | 18 | const ( 19 | // Unchanged means that a document has not been changed between 20 | // source and destination index. 21 | Unchanged Mode = iota 22 | // Created means that a document has been added to the destination 23 | // that didn't exist in the source index. 24 | Created 25 | // Updated means that a document has been found both in the source 26 | // and destination index, but its contents (_source) has changed. 27 | Updated 28 | // Deleted means that a document has been found in the source index 29 | // but it doesn't exist in the destination index. 30 | Deleted 31 | ) 32 | 33 | // Mode returns a string represenation for a mode. 34 | func (m Mode) String() string { 35 | switch m { 36 | case Unchanged: 37 | return "Unchanged" 38 | case Created: 39 | return "Created" 40 | case Updated: 41 | return "Updated" 42 | case Deleted: 43 | return "Deleted" 44 | default: 45 | return "" 46 | } 47 | } 48 | 49 | // Diff is the outcome of comparing two documents in source and 50 | // destination index. 51 | type Diff struct { 52 | Mode Mode 53 | Src *Document 54 | Dst *Document 55 | } 56 | 57 | // Differ compares the documents in the source index to those in 58 | // the destination index. It returns the outcomes via a Diff structure, 59 | // one by one. 60 | func Differ( 61 | ctx context.Context, 62 | srcCh <-chan *Document, 63 | dstCh <-chan *Document, 64 | ) (<-chan Diff, <-chan error) { 65 | diffCh := make(chan Diff) 66 | errCh := make(chan error) 67 | 68 | go func() { 69 | defer func() { 70 | close(diffCh) 71 | close(errCh) 72 | }() 73 | 74 | // Both src and dst are nil => no diffs 75 | if srcCh == nil && dstCh == nil { 76 | return 77 | } 78 | 79 | // No src => return all from dst as Created 80 | if srcCh == nil && dstCh != nil { 81 | for { 82 | select { 83 | case doc, ok := <-dstCh: 84 | if !ok { 85 | return 86 | } 87 | diffCh <- Diff{Mode: Created, Dst: doc} 88 | case <-ctx.Done(): 89 | errCh <- ctx.Err() 90 | return 91 | } 92 | } 93 | } 94 | 95 | // No dst => return all from src as Deleted 96 | if srcCh != nil && dstCh == nil { 97 | for { 98 | select { 99 | case doc, ok := <-srcCh: 100 | if !ok { 101 | return 102 | } 103 | diffCh <- Diff{Mode: Deleted, Src: doc} 104 | case <-ctx.Done(): 105 | errCh <- ctx.Err() 106 | return 107 | } 108 | } 109 | } 110 | 111 | // Read first document from both channels 112 | srcDoc, dstDoc := <-srcCh, <-dstCh 113 | 114 | // Main loop 115 | for { 116 | // Stop early because context might be canceled. 117 | select { 118 | default: 119 | case <-ctx.Done(): 120 | errCh <- ctx.Err() 121 | return 122 | } 123 | 124 | // No more documents from the channels => done 125 | if srcDoc == nil && dstDoc == nil { 126 | break 127 | } 128 | 129 | // No more from dst => everything in src has to be deleted 130 | if srcDoc != nil && dstDoc == nil { 131 | diffCh <- Diff{Mode: Deleted, Src: srcDoc} 132 | for { 133 | select { 134 | case doc, ok := <-dstCh: 135 | if !ok { 136 | return 137 | } 138 | diffCh <- Diff{Mode: Deleted, Src: doc} 139 | case <-ctx.Done(): 140 | errCh <- ctx.Err() 141 | return 142 | } 143 | } 144 | } 145 | 146 | // No more from src => everything in dst has to be created 147 | if srcDoc == nil && dstDoc != nil { 148 | diffCh <- Diff{Mode: Created, Dst: dstDoc} 149 | for { 150 | select { 151 | case doc, ok := <-dstCh: 152 | if !ok { 153 | return 154 | } 155 | diffCh <- Diff{Mode: Created, Dst: doc} 156 | case <-ctx.Done(): 157 | errCh <- ctx.Err() 158 | return 159 | } 160 | } 161 | } 162 | 163 | // We have two to compare 164 | if srcDoc.ID > dstDoc.ID { 165 | diffCh <- Diff{Mode: Created, Dst: dstDoc} 166 | dstDoc = nil 167 | var stop bool 168 | for !stop { 169 | select { 170 | case doc, ok := <-dstCh: 171 | if !ok { 172 | stop = true 173 | break 174 | } 175 | dstDoc = doc 176 | if srcDoc.ID <= dstDoc.ID { 177 | stop = true 178 | break 179 | } 180 | diffCh <- Diff{Mode: Created, Dst: dstDoc} 181 | dstDoc = nil 182 | case <-ctx.Done(): 183 | errCh <- ctx.Err() 184 | return 185 | } 186 | } 187 | } else if srcDoc.ID < dstDoc.ID { 188 | diffCh <- Diff{Mode: Deleted, Src: srcDoc} 189 | srcDoc = nil 190 | var stop bool 191 | for !stop { 192 | select { 193 | case doc, ok := <-srcCh: 194 | if !ok { 195 | stop = true 196 | break 197 | } 198 | srcDoc = doc 199 | if srcDoc.ID >= dstDoc.ID { 200 | stop = true 201 | break 202 | } 203 | diffCh <- Diff{Mode: Deleted, Src: srcDoc} 204 | srcDoc = nil 205 | case <-ctx.Done(): 206 | errCh <- ctx.Err() 207 | return 208 | } 209 | } 210 | } else { 211 | // srcDoc.ID == dstDoc.ID 212 | if cmp.Equal(srcDoc.Source, dstDoc.Source) { 213 | diffCh <- Diff{Mode: Unchanged, Src: srcDoc, Dst: dstDoc} 214 | } else { 215 | diffCh <- Diff{Mode: Updated, Src: srcDoc, Dst: dstDoc} 216 | } 217 | srcDoc, dstDoc = <-srcCh, <-dstCh 218 | } 219 | } 220 | }() 221 | 222 | return diffCh, errCh 223 | } 224 | -------------------------------------------------------------------------------- /diff/differ_test.go: -------------------------------------------------------------------------------- 1 | package diff 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/google/go-cmp/cmp" 9 | 10 | "github.com/fortytw2/leaktest" 11 | ) 12 | 13 | var differTests = []struct { 14 | Srcs, Dsts []*Document 15 | Errs []error 16 | Diffs []Diff 17 | }{ 18 | // #0 19 | { 20 | Srcs: nil, 21 | Dsts: nil, 22 | Errs: nil, 23 | Diffs: nil, 24 | }, 25 | // #1 26 | { 27 | Srcs: nil, 28 | Dsts: []*Document{ 29 | {ID: "1", Source: map[string]interface{}{"Name": "One"}}, 30 | }, 31 | Errs: nil, 32 | Diffs: []Diff{ 33 | { 34 | Mode: Created, 35 | Src: nil, 36 | Dst: &Document{ID: "1", Source: map[string]interface{}{"Name": "One"}}, 37 | }, 38 | }, 39 | }, 40 | // #2 41 | { 42 | Srcs: []*Document{ 43 | {ID: "1", Source: map[string]interface{}{"Name": "One"}}, 44 | }, 45 | Dsts: nil, 46 | Errs: nil, 47 | Diffs: []Diff{ 48 | { 49 | Mode: Deleted, 50 | Src: &Document{ID: "1", Source: map[string]interface{}{"Name": "One"}}, 51 | Dst: nil, 52 | }, 53 | }, 54 | }, 55 | // #3 56 | { 57 | Dsts: []*Document{ 58 | {ID: "1", Source: map[string]interface{}{"Name": "One"}}, 59 | {ID: "2", Source: map[string]interface{}{"Name": "Two"}}, 60 | }, 61 | Srcs: []*Document{ 62 | {ID: "1", Source: map[string]interface{}{"Name": "One"}}, 63 | {ID: "3", Source: map[string]interface{}{"Name": "Three"}}, 64 | }, 65 | Errs: nil, 66 | Diffs: []Diff{ 67 | { 68 | Mode: Unchanged, 69 | Src: &Document{ID: "1", Source: map[string]interface{}{"Name": "One"}}, 70 | Dst: &Document{ID: "1", Source: map[string]interface{}{"Name": "One"}}, 71 | }, 72 | { 73 | Mode: Created, 74 | Src: nil, 75 | Dst: &Document{ID: "2", Source: map[string]interface{}{"Name": "Two"}}, 76 | }, 77 | { 78 | Mode: Deleted, 79 | Src: &Document{ID: "3", Source: map[string]interface{}{"Name": "Three"}}, 80 | Dst: nil, 81 | }, 82 | }, 83 | }, 84 | // #4 85 | { 86 | Srcs: []*Document{ 87 | {ID: "2", Source: map[string]interface{}{"Name": "Two"}}, 88 | {ID: "3", Source: map[string]interface{}{"Name": "Three"}}, 89 | {ID: "4", Source: map[string]interface{}{"Name": "Four", "Value": 3}}, 90 | {ID: "5", Source: map[string]interface{}{"Name": "Five"}}, 91 | {ID: "6", Source: map[string]interface{}{"Name": "Six"}}, 92 | }, 93 | Dsts: []*Document{ 94 | {ID: "1", Source: map[string]interface{}{"Name": "One"}}, 95 | {ID: "4", Source: map[string]interface{}{"Name": "Four", "Value": 4}}, 96 | {ID: "6", Source: map[string]interface{}{"Name": "Six"}}, 97 | }, 98 | Errs: nil, 99 | Diffs: []Diff{ 100 | { 101 | Mode: Created, 102 | Src: nil, 103 | Dst: &Document{ID: "1", Source: map[string]interface{}{"Name": "One"}}, 104 | }, 105 | { 106 | Mode: Deleted, 107 | Src: &Document{ID: "2", Source: map[string]interface{}{"Name": "Two"}}, 108 | Dst: nil, 109 | }, 110 | { 111 | Mode: Deleted, 112 | Src: &Document{ID: "3", Source: map[string]interface{}{"Name": "Three"}}, 113 | Dst: nil, 114 | }, 115 | { 116 | Mode: Updated, 117 | Src: &Document{ID: "4", Source: map[string]interface{}{"Name": "Four", "Value": 3}}, 118 | Dst: &Document{ID: "4", Source: map[string]interface{}{"Name": "Four", "Value": 4}}, 119 | }, 120 | { 121 | Mode: Deleted, 122 | Src: &Document{ID: "5", Source: map[string]interface{}{"Name": "Five"}}, 123 | Dst: nil, 124 | }, 125 | { 126 | Mode: Unchanged, 127 | Src: &Document{ID: "6", Source: map[string]interface{}{"Name": "Six"}}, 128 | Dst: &Document{ID: "6", Source: map[string]interface{}{"Name": "Six"}}, 129 | }, 130 | }, 131 | }, 132 | // #5 133 | { 134 | Srcs: []*Document{ 135 | {ID: "1", Source: map[string]interface{}{"Name": "One", "Price": 599.00}}, 136 | {ID: "3", Source: map[string]interface{}{"Name": "Three", "Price": 2599.00}}, 137 | }, 138 | Dsts: []*Document{ 139 | {ID: "1", Source: map[string]interface{}{"Name": "One", "Price": 599.00}}, 140 | {ID: "2", Source: map[string]interface{}{"Name": "Two", "Price": 2599.00}}, 141 | }, 142 | Errs: nil, 143 | Diffs: []Diff{ 144 | { 145 | Mode: Unchanged, 146 | Src: &Document{ID: "1", Source: map[string]interface{}{"Name": "One", "Price": 599.00}}, 147 | Dst: &Document{ID: "1", Source: map[string]interface{}{"Name": "One", "Price": 599.00}}, 148 | }, 149 | { 150 | Mode: Created, 151 | Src: nil, 152 | Dst: &Document{ID: "2", Source: map[string]interface{}{"Name": "Two", "Price": 2599.00}}, 153 | }, 154 | { 155 | Mode: Deleted, 156 | Src: &Document{ID: "3", Source: map[string]interface{}{"Name": "Three", "Price": 2599.00}}, 157 | Dst: nil, 158 | }, 159 | }, 160 | }, 161 | // #6 162 | { 163 | Srcs: []*Document{ 164 | {ID: "239473748", Source: map[string]interface{}{"Name": "Same Document"}}, 165 | }, 166 | Dsts: []*Document{ 167 | {ID: "239473748", Source: map[string]interface{}{"Name": "Same Document"}}, 168 | {ID: "34", Source: map[string]interface{}{"Name": "New Document"}}, 169 | {ID: "32", Source: map[string]interface{}{"Name": "New Document 2"}}, 170 | }, 171 | Errs: nil, 172 | Diffs: []Diff{ 173 | { 174 | Mode: Unchanged, 175 | Src: &Document{ID: "239473748", Source: map[string]interface{}{"Name": "Same Document"}}, 176 | Dst: &Document{ID: "239473748", Source: map[string]interface{}{"Name": "Same Document"}}, 177 | }, 178 | { 179 | Mode: Created, 180 | Src: nil, 181 | Dst: &Document{ID: "34", Source: map[string]interface{}{"Name": "New Document"}}, 182 | }, 183 | { 184 | Mode: Created, 185 | Src: nil, 186 | Dst: &Document{ID: "32", Source: map[string]interface{}{"Name": "New Document 2"}}, 187 | }, 188 | }, 189 | }, 190 | } 191 | 192 | func TestDiffer(t *testing.T) { 193 | for i, tt := range differTests { 194 | ctx := context.Background() 195 | done := make(chan struct{}) 196 | var errs []error 197 | var diffs []Diff 198 | 199 | // Generator for src 200 | srcCh := make(chan *Document) 201 | go func() { 202 | defer close(srcCh) 203 | for _, doc := range tt.Srcs { 204 | srcCh <- doc 205 | } 206 | }() 207 | 208 | // Generator for dst 209 | dstCh := make(chan *Document) 210 | go func() { 211 | defer close(dstCh) 212 | for _, doc := range tt.Dsts { 213 | dstCh <- doc 214 | } 215 | }() 216 | 217 | // Process diffs 218 | go func() { 219 | defer close(done) 220 | diffCh, errCh := Differ(ctx, srcCh, dstCh) 221 | var done bool 222 | for !done { 223 | select { 224 | case d, ok := <-diffCh: 225 | if !ok { 226 | return 227 | } 228 | diffs = append(diffs, d) 229 | case err, ok := <-errCh: 230 | if !ok { 231 | return 232 | } 233 | errs = append(errs, err) 234 | } 235 | } 236 | }() 237 | 238 | // Wait until we are done or we get a timeout 239 | select { 240 | case <-done: 241 | case <-time.After(5 * time.Second): 242 | t.Fatal("timeout") 243 | } 244 | leaktest.Check(t)() 245 | 246 | if want, have := len(tt.Errs), len(errs); want != have { 247 | t.Fatalf("#%d: len(Errors): want %d, have %d\n%v", i, want, have, cmp.Diff(tt.Errs, errs)) 248 | } 249 | for k := 0; k < len(errs); k++ { 250 | if want, have := tt.Errs[k], errs[k]; want != have { 251 | t.Fatalf("#%d: Error[%d]: want %v, have %v", i, k, want, have) 252 | } 253 | } 254 | if want, have := len(tt.Diffs), len(diffs); want != have { 255 | t.Fatalf("#%d: len(Diffs): want %d, have %d\n%v", i, want, have, cmp.Diff(tt.Diffs, diffs)) 256 | } 257 | for k := 0; k < len(diffs); k++ { 258 | if want, have := tt.Diffs[k].Mode, diffs[k].Mode; want != have { 259 | t.Fatalf("#%d: Diffs[%d].Mode: want %v, have %v\n%v", i, k, want, have, cmp.Diff(tt.Diffs[k], diffs[k])) 260 | } 261 | if want, have := tt.Diffs[k].Src, diffs[k].Src; !cmp.Equal(want, have) { 262 | t.Fatalf("#%d: Diffs[%d].Src: %v", i, k, cmp.Diff(want, have)) 263 | } 264 | if want, have := tt.Diffs[k].Dst, diffs[k].Dst; !cmp.Equal(want, have) { 265 | t.Fatalf("#%d: Diffs[%d].Dst: %v", i, k, cmp.Diff(want, have)) 266 | } 267 | switch diffs[k].Mode { 268 | case Unchanged: 269 | case Created: 270 | case Deleted: 271 | if have := diffs[k].Src; have == nil { 272 | t.Fatalf("#%d: Diffs[%d].Src: want %v, have %v", i, k, nil, have) 273 | } 274 | if have := diffs[k].Dst; have != nil { 275 | t.Fatalf("#%d: Diffs[%d].Dst: want != %v, have %v", i, k, nil, have) 276 | } 277 | case Updated: 278 | } 279 | } 280 | } 281 | } 282 | -------------------------------------------------------------------------------- /diff/printer/json.go: -------------------------------------------------------------------------------- 1 | package printer 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | 7 | "github.com/olivere/esdiff/diff" 8 | ) 9 | 10 | // JSONPrinter prints diffs as JSON, making it easily parseable 11 | // for tools like jq or jiq. 12 | type JSONPrinter struct { 13 | w io.Writer 14 | enc *json.Encoder 15 | unchanged bool 16 | updated bool 17 | created bool 18 | deleted bool 19 | } 20 | 21 | // NewJSONPrinter creates a new JSONPrinter. 22 | func NewJSONPrinter(w io.Writer, unchanged, updated, created, deleted bool) *JSONPrinter { 23 | return &JSONPrinter{ 24 | w: w, 25 | enc: json.NewEncoder(w), 26 | unchanged: unchanged, 27 | updated: updated, 28 | created: created, 29 | deleted: deleted, 30 | } 31 | } 32 | 33 | // Print prints a diff as JSON. 34 | func (p *JSONPrinter) Print(d diff.Diff) error { 35 | type rowType struct { 36 | Mode string `json:"mode"` 37 | ID string `json:"_id"` 38 | Src interface{} `json:"src,omitempty"` 39 | Dst interface{} `json:"dst,omitempty"` 40 | // Diff interface{} `json:"diff,omitempty"` 41 | } 42 | 43 | ok := false 44 | 45 | row := rowType{ 46 | Src: d.Src, 47 | Dst: d.Dst, 48 | } 49 | 50 | switch d.Mode { 51 | case diff.Unchanged: 52 | row.Mode = "unchanged" 53 | row.ID = d.Src.ID 54 | ok = p.unchanged 55 | case diff.Created: 56 | row.Mode = "created" 57 | row.ID = d.Dst.ID 58 | ok = p.created 59 | case diff.Updated: 60 | row.Mode = "updated" 61 | row.ID = d.Src.ID 62 | ok = p.updated 63 | case diff.Deleted: 64 | row.Mode = "deleted" 65 | row.ID = d.Src.ID 66 | ok = p.deleted 67 | } 68 | 69 | if ok { 70 | return p.enc.Encode(row) 71 | } 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /diff/printer/printer.go: -------------------------------------------------------------------------------- 1 | package printer 2 | 3 | import ( 4 | "github.com/olivere/esdiff/diff" 5 | ) 6 | 7 | // Printer prints a diff using a specific output format, e.g. JSON. 8 | type Printer interface { 9 | Print(diff.Diff) error 10 | } 11 | -------------------------------------------------------------------------------- /diff/printer/std.go: -------------------------------------------------------------------------------- 1 | package printer 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/google/go-cmp/cmp" 8 | 9 | "github.com/olivere/esdiff/diff" 10 | ) 11 | 12 | // StdPrinter uses a textual description for diffs. 13 | type StdPrinter struct { 14 | w io.Writer 15 | unchanged bool 16 | updated bool 17 | created bool 18 | deleted bool 19 | } 20 | 21 | // NewStdPrinter creates a new StdPrinter. 22 | func NewStdPrinter(w io.WriteCloser, unchanged, updated, created, deleted bool) *StdPrinter { 23 | return &StdPrinter{ 24 | w: w, 25 | unchanged: unchanged, 26 | updated: updated, 27 | created: created, 28 | deleted: deleted, 29 | } 30 | } 31 | 32 | // Print prints a diff in a textual form. It e.g. doesn't print 33 | // a diff for unchanged documents. 34 | func (p *StdPrinter) Print(d diff.Diff) error { 35 | switch d.Mode { 36 | case diff.Unchanged: 37 | if p.unchanged { 38 | fmt.Fprintf(p.w, "Unchanged\t%v\t%v\n", d.Src.ID, cmp.Diff(d.Src, d.Dst)) 39 | } 40 | case diff.Created: 41 | if p.created { 42 | fmt.Fprintf(p.w, "Created\t%v\t%v\n", d.Dst.ID, cmp.Diff(d.Src, d.Dst)) 43 | } 44 | case diff.Updated: 45 | if p.updated { 46 | fmt.Fprintf(p.w, "Updated\t%v\t%v\n", d.Src.ID, cmp.Diff(d.Src, d.Dst)) 47 | } 48 | case diff.Deleted: 49 | if p.deleted { 50 | fmt.Fprintf(p.w, "Deleted\t%v\n", d.Src.ID) 51 | } 52 | } 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /docker-compose.kibana.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | kibana5: 5 | image: docker.elastic.co/kibana/kibana:5.6.16 6 | hostname: kibana5 7 | environment: 8 | - "ES_JAVA_OPTS=-Xms1g -Xmx1g" 9 | - cluster.name=mia5 10 | volumes: 11 | - ./etc/kibana5/kibana.yml:/usr/share/kibana/config/kibana.yml:ro 12 | ports: 13 | - 19201:5601 14 | 15 | kibana6: 16 | image: docker.elastic.co/kibana/kibana:6.8.23 17 | hostname: kibana6 18 | volumes: 19 | - ./etc/kibana6/kibana.yml:/usr/share/kibana/config/kibana.yml:ro 20 | ports: 21 | - 29201:5601 22 | 23 | kibana7: 24 | image: docker.elastic.co/kibana/kibana:7.17.5 25 | hostname: kibana7 26 | volumes: 27 | - ./etc/kibana7/kibana.yml:/usr/share/kibana/config/kibana.yml:ro 28 | ports: 29 | - 39201:5601 30 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | 3 | services: 4 | elasticsearch5: 5 | image: docker.elastic.co/elasticsearch/elasticsearch:5.6.16 6 | hostname: elasticsearch5 7 | environment: 8 | - "ES_JAVA_OPTS=-Xms1g -Xmx1g" 9 | - cluster.name=mia5 10 | - bootstrap.memory_lock=true 11 | - xpack.security.enabled=false 12 | - xpack.monitoring.enabled=false 13 | - xpack.ml.enabled=false 14 | - xpack.graph.enabled=false 15 | - xpack.watcher.enabled=false 16 | volumes: 17 | - ./data/elasticsearch5:/usr/share/elasticsearch/data 18 | - ./etc/elasticsearch5/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro 19 | ports: 20 | - 19200:9200 21 | 22 | elasticsearch6: 23 | image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23 24 | hostname: elasticsearch6 25 | environment: 26 | - bootstrap.memory_lock=true 27 | - discovery.type=single-node 28 | - network.publish_host=127.0.0.1 29 | - logger.org.elasticsearch=warn 30 | - "ES_JAVA_OPTS=-Xms1g -Xmx1g" 31 | ulimits: 32 | nproc: 65536 33 | nofile: 34 | soft: 65536 35 | hard: 65536 36 | memlock: 37 | soft: -1 38 | hard: -1 39 | volumes: 40 | - ./data/elasticsearch6:/usr/share/elasticsearch/data 41 | # - ./etc/elasticsearch6/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro 42 | ports: 43 | - 29200:9200 44 | 45 | elasticsearch7: 46 | image: docker.elastic.co/elasticsearch/elasticsearch:7.17.5 47 | hostname: elasticsearch7 48 | environment: 49 | - bootstrap.memory_lock=true 50 | - discovery.type=single-node 51 | - network.publish_host=127.0.0.1 52 | - logger.org.elasticsearch=warn 53 | - xpack.security.enabled=false 54 | - "ES_JAVA_OPTS=-Xms1g -Xmx1g" 55 | ulimits: 56 | nproc: 65536 57 | nofile: 58 | soft: 65536 59 | hard: 65536 60 | memlock: 61 | soft: -1 62 | hard: -1 63 | volumes: 64 | - ./data/elasticsearch7:/usr/share/elasticsearch/data 65 | # - ./etc/elasticsearch7/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro 66 | ports: 67 | - 39200:9200 68 | -------------------------------------------------------------------------------- /elastic/client.go: -------------------------------------------------------------------------------- 1 | package elastic 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/olivere/esdiff/diff" 7 | ) 8 | 9 | // Client encapsulates access to an Elasticsearch cluster. 10 | type Client interface { 11 | Iterate(context.Context, *IterateRequest) (<-chan *diff.Document, <-chan error) 12 | } 13 | 14 | // IterateRequest specifies a request for the Iterate function. 15 | type IterateRequest struct { 16 | RawQuery string 17 | SortField string 18 | ReplaceField string 19 | SourceFilterInclude []string 20 | SourceFilterExclude []string 21 | } 22 | 23 | // ClientWithBatchSize should be implemented by clients that 24 | // support setting the batch size for scrolling. 25 | type ClientWithBatchSize interface { 26 | SetBatchSize(int) 27 | } 28 | 29 | // ClientOption specifies the signature for setting a generic 30 | // option for a Client. 31 | type ClientOption func(Client) 32 | 33 | // WithBatchSize allows setting the batch size for scrolling through 34 | // the documents (for clients that support this). 35 | func WithBatchSize(size int) ClientOption { 36 | return func(client Client) { 37 | c, ok := client.(ClientWithBatchSize) 38 | if ok { 39 | if size > 0 { 40 | c.SetBatchSize(size) 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /elastic/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | // Config represents an Elasticsearch configuration. 11 | type Config struct { 12 | URL string 13 | Index string 14 | Type string 15 | Username string 16 | Password string 17 | Shards int 18 | Replicas int 19 | Sniff bool 20 | Infolog string 21 | Errorlog string 22 | Tracelog string 23 | } 24 | 25 | // Parse returns the Elasticsearch configuration by extracting it 26 | // from the URL, its path, and its query string. 27 | // 28 | // Example: 29 | // http://127.0.0.1:9200/index/type?shards=1&replicas=0&sniff=false&tracelog=elastic.trace.log 30 | // 31 | // The code above will return a URL of http://127.0.0.1:9200, an index name 32 | // of store-blobs, and the related settings from the query string. 33 | func Parse(elasticURL string) (*Config, error) { 34 | cfg := &Config{ 35 | Shards: 1, 36 | Replicas: 0, 37 | Sniff: false, // sniffing disabled by default 38 | } 39 | 40 | uri, err := url.Parse(elasticURL) 41 | if err != nil { 42 | return nil, fmt.Errorf("error parsing elastic parameter %q: %v", elasticURL, err) 43 | } 44 | indexAndType := strings.Trim(uri.Path, "/") 45 | if indexAndType == "" { 46 | return nil, fmt.Errorf("missing index and/or type in elastic parameter %q", elasticURL) 47 | } 48 | parts := strings.SplitN(indexAndType, "/", 2) 49 | cfg.Index = parts[0] 50 | cfg.Type = parts[1] 51 | if cfg.Index == "" { 52 | return nil, fmt.Errorf("missing index and in elastic parameter %q", elasticURL) 53 | } 54 | if uri.User != nil { 55 | cfg.Username = uri.User.Username() 56 | cfg.Password, _ = uri.User.Password() 57 | } 58 | uri.User = nil 59 | 60 | if i, err := strconv.Atoi(uri.Query().Get("shards")); err == nil { 61 | cfg.Shards = i 62 | } 63 | if i, err := strconv.Atoi(uri.Query().Get("replicas")); err == nil { 64 | cfg.Replicas = i 65 | } 66 | if s := uri.Query().Get("sniff"); s != "" { 67 | if b, err := strconv.ParseBool(s); err == nil { 68 | cfg.Sniff = b 69 | } 70 | } 71 | if s := uri.Query().Get("infolog"); s != "" { 72 | cfg.Infolog = s 73 | } 74 | if s := uri.Query().Get("errorlog"); s != "" { 75 | cfg.Errorlog = s 76 | } 77 | if s := uri.Query().Get("tracelog"); s != "" { 78 | cfg.Tracelog = s 79 | } 80 | 81 | uri.Path = "" 82 | uri.RawQuery = "" 83 | cfg.URL = uri.String() 84 | 85 | return cfg, nil 86 | } 87 | -------------------------------------------------------------------------------- /elastic/v5/client.go: -------------------------------------------------------------------------------- 1 | package v5 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io" 7 | "log" 8 | "os" 9 | "strconv" 10 | 11 | "github.com/pkg/errors" 12 | elasticv5 "gopkg.in/olivere/elastic.v5" 13 | 14 | "github.com/olivere/esdiff/diff" 15 | "github.com/olivere/esdiff/elastic" 16 | "github.com/olivere/esdiff/elastic/config" 17 | ) 18 | 19 | // Client implements an Elasticsearch 5.x client. 20 | type Client struct { 21 | c *elasticv5.Client 22 | index string 23 | typ string 24 | size int 25 | } 26 | 27 | // NewClient creates a new Client. 28 | func NewClient(cfg *config.Config) (*Client, error) { 29 | var options []elasticv5.ClientOptionFunc 30 | if cfg != nil { 31 | if cfg.URL != "" { 32 | options = append(options, elasticv5.SetURL(cfg.URL)) 33 | } 34 | if cfg.Errorlog != "" { 35 | f, err := os.OpenFile(cfg.Errorlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 36 | if err != nil { 37 | return nil, errors.Wrap(err, "unable to initialize error log") 38 | } 39 | l := log.New(f, "", 0) 40 | options = append(options, elasticv5.SetErrorLog(l)) 41 | } 42 | if cfg.Tracelog != "" { 43 | f, err := os.OpenFile(cfg.Tracelog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 44 | if err != nil { 45 | return nil, errors.Wrap(err, "unable to initialize trace log") 46 | } 47 | l := log.New(f, "", 0) 48 | options = append(options, elasticv5.SetTraceLog(l)) 49 | } 50 | if cfg.Infolog != "" { 51 | f, err := os.OpenFile(cfg.Infolog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 52 | if err != nil { 53 | return nil, errors.Wrap(err, "unable to initialize info log") 54 | } 55 | l := log.New(f, "", 0) 56 | options = append(options, elasticv5.SetInfoLog(l)) 57 | } 58 | if cfg.Username != "" || cfg.Password != "" { 59 | options = append(options, elasticv5.SetBasicAuth(cfg.Username, cfg.Password)) 60 | } 61 | options = append(options, elasticv5.SetSniff(cfg.Sniff)) 62 | } 63 | cli, err := elasticv5.NewClient(options...) 64 | if err != nil { 65 | return nil, err 66 | } 67 | c := &Client{ 68 | c: cli, 69 | index: cfg.Index, 70 | typ: cfg.Type, 71 | size: 100, 72 | } 73 | return c, nil 74 | } 75 | 76 | // SetBatchSize specifies the size of a single scroll operation. 77 | func (c *Client) SetBatchSize(size int) { 78 | c.size = size 79 | } 80 | 81 | // Iterate iterates over the index. 82 | func (c *Client) Iterate(ctx context.Context, req *elastic.IterateRequest) (<-chan *diff.Document, <-chan error) { 83 | docCh := make(chan *diff.Document, 1) 84 | errCh := make(chan error, 1) 85 | 86 | go func() { 87 | defer func() { 88 | close(docCh) 89 | close(errCh) 90 | }() 91 | 92 | // Sorting 93 | var sorter elasticv5.Sorter 94 | if req.SortField == "" { 95 | sorter = elasticv5.NewFieldSort("_uid").Asc() 96 | } else { 97 | field := req.SortField 98 | asc := true 99 | if field[0] == '-' { 100 | field = field[1:] 101 | asc = false 102 | } 103 | sorter = elasticv5.NewFieldSort(field).Order(asc) 104 | } 105 | 106 | svc := c.c.Scroll(c.index).Type(c.typ).Size(c.size).SortBy(sorter) 107 | if req.RawQuery != "" { 108 | q := elasticv5.NewRawStringQuery(req.RawQuery) 109 | svc = svc.Query(q) 110 | } 111 | if len(req.SourceFilterInclude)+len(req.SourceFilterExclude) > 0 { 112 | fsc := elasticv5.NewFetchSourceContext(true). 113 | Include(req.SourceFilterInclude...). 114 | Exclude(req.SourceFilterExclude...) 115 | svc = svc.FetchSourceContext(fsc) 116 | } 117 | 118 | for { 119 | res, err := svc.Do(ctx) 120 | if err == io.EOF { 121 | return 122 | } 123 | if err != nil { 124 | errCh <- err 125 | return 126 | } 127 | if res == nil { 128 | errCh <- errors.New("unexpected nil document") 129 | return 130 | } 131 | if res.Hits == nil { 132 | errCh <- errors.New("unexpected nil hits") 133 | return 134 | } 135 | for _, hit := range res.Hits.Hits { 136 | doc := new(diff.Document) 137 | err := json.Unmarshal(*hit.Source, &doc.Source) 138 | if err != nil { 139 | errCh <- err 140 | return 141 | } 142 | // Replace ID field with some other field from the document? 143 | if req.ReplaceField != "" { 144 | if val, ok := doc.Source[req.ReplaceField]; ok { 145 | switch v := val.(type) { 146 | case string: 147 | doc.ID = v 148 | case int: 149 | doc.ID = strconv.Itoa(v) 150 | case int32: 151 | doc.ID = strconv.FormatInt(int64(v), 10) 152 | case int64: 153 | doc.ID = strconv.FormatInt(v, 10) 154 | case float32: 155 | doc.ID = strconv.Itoa(int(v)) 156 | case float64: 157 | doc.ID = strconv.Itoa(int(v)) 158 | default: 159 | doc.ID = val.(string) 160 | } 161 | } else { 162 | errCh <- errors.New("unexpected replace-with field") 163 | return 164 | } 165 | } else { 166 | doc.ID = hit.Id 167 | } 168 | docCh <- doc 169 | } 170 | } 171 | }() 172 | 173 | return docCh, errCh 174 | } 175 | -------------------------------------------------------------------------------- /elastic/v6/client.go: -------------------------------------------------------------------------------- 1 | package v6 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io" 7 | "log" 8 | "os" 9 | "strconv" 10 | 11 | elasticv6 "github.com/olivere/elastic" 12 | "github.com/pkg/errors" 13 | 14 | "github.com/olivere/esdiff/diff" 15 | "github.com/olivere/esdiff/elastic" 16 | "github.com/olivere/esdiff/elastic/config" 17 | ) 18 | 19 | // Client implements an Elasticsearch 6.x client. 20 | type Client struct { 21 | c *elasticv6.Client 22 | index string 23 | typ string 24 | size int 25 | } 26 | 27 | // NewClient creates a new Client. 28 | func NewClient(cfg *config.Config) (*Client, error) { 29 | var options []elasticv6.ClientOptionFunc 30 | if cfg != nil { 31 | if cfg.URL != "" { 32 | options = append(options, elasticv6.SetURL(cfg.URL)) 33 | } 34 | if cfg.Errorlog != "" { 35 | f, err := os.OpenFile(cfg.Errorlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 36 | if err != nil { 37 | return nil, errors.Wrap(err, "unable to initialize error log") 38 | } 39 | l := log.New(f, "", 0) 40 | options = append(options, elasticv6.SetErrorLog(l)) 41 | } 42 | if cfg.Tracelog != "" { 43 | f, err := os.OpenFile(cfg.Tracelog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 44 | if err != nil { 45 | return nil, errors.Wrap(err, "unable to initialize trace log") 46 | } 47 | l := log.New(f, "", 0) 48 | options = append(options, elasticv6.SetTraceLog(l)) 49 | } 50 | if cfg.Infolog != "" { 51 | f, err := os.OpenFile(cfg.Infolog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 52 | if err != nil { 53 | return nil, errors.Wrap(err, "unable to initialize info log") 54 | } 55 | l := log.New(f, "", 0) 56 | options = append(options, elasticv6.SetInfoLog(l)) 57 | } 58 | if cfg.Username != "" || cfg.Password != "" { 59 | options = append(options, elasticv6.SetBasicAuth(cfg.Username, cfg.Password)) 60 | } 61 | options = append(options, elasticv6.SetSniff(cfg.Sniff)) 62 | } 63 | cli, err := elasticv6.NewClient(options...) 64 | if err != nil { 65 | return nil, err 66 | } 67 | c := &Client{ 68 | c: cli, 69 | index: cfg.Index, 70 | typ: cfg.Type, 71 | size: 100, 72 | } 73 | return c, nil 74 | } 75 | 76 | // SetBatchSize specifies the size of a single scroll operation. 77 | func (c *Client) SetBatchSize(size int) { 78 | c.size = size 79 | } 80 | 81 | // Iterate iterates over the index. 82 | func (c *Client) Iterate(ctx context.Context, req *elastic.IterateRequest) (<-chan *diff.Document, <-chan error) { 83 | docCh := make(chan *diff.Document, 1) 84 | errCh := make(chan error, 1) 85 | 86 | go func() { 87 | defer func() { 88 | close(docCh) 89 | close(errCh) 90 | }() 91 | 92 | // Sorting 93 | var sorter elasticv6.Sorter 94 | if req.SortField == "" { 95 | sorter = elasticv6.NewFieldSort("_id").Asc() 96 | } else { 97 | field := req.SortField 98 | asc := true 99 | if field[0] == '-' { 100 | field = field[1:] 101 | asc = false 102 | } 103 | sorter = elasticv6.NewFieldSort(field).Order(asc) 104 | } 105 | 106 | svc := c.c.Scroll(c.index).Type(c.typ).Size(c.size).SortBy(sorter) 107 | if req.RawQuery != "" { 108 | q := elasticv6.NewRawStringQuery(req.RawQuery) 109 | svc = svc.Query(q) 110 | } 111 | if len(req.SourceFilterInclude)+len(req.SourceFilterExclude) > 0 { 112 | fsc := elasticv6.NewFetchSourceContext(true). 113 | Include(req.SourceFilterInclude...). 114 | Exclude(req.SourceFilterExclude...) 115 | svc = svc.FetchSourceContext(fsc) 116 | } 117 | for { 118 | res, err := svc.Do(ctx) 119 | if err == io.EOF { 120 | return 121 | } 122 | if err != nil { 123 | errCh <- err 124 | return 125 | } 126 | if res == nil { 127 | errCh <- errors.New("unexpected nil document") 128 | return 129 | } 130 | if res.Hits == nil { 131 | errCh <- errors.New("unexpected nil hits") 132 | return 133 | } 134 | for _, hit := range res.Hits.Hits { 135 | doc := new(diff.Document) 136 | err := json.Unmarshal(*hit.Source, &doc.Source) 137 | if err != nil { 138 | errCh <- err 139 | return 140 | } 141 | // Replace ID field with some other field from the document? 142 | if req.ReplaceField != "" { 143 | if val, ok := doc.Source[req.ReplaceField]; ok { 144 | switch v := val.(type) { 145 | case string: 146 | doc.ID = v 147 | case int: 148 | doc.ID = strconv.Itoa(v) 149 | case int32: 150 | doc.ID = strconv.FormatInt(int64(v), 10) 151 | case int64: 152 | doc.ID = strconv.FormatInt(v, 10) 153 | case float32: 154 | doc.ID = strconv.Itoa(int(v)) 155 | case float64: 156 | doc.ID = strconv.Itoa(int(v)) 157 | default: 158 | doc.ID = val.(string) 159 | } 160 | } else { 161 | errCh <- errors.New("unexpected replace-with field") 162 | return 163 | } 164 | } else { 165 | doc.ID = hit.Id 166 | } 167 | docCh <- doc 168 | } 169 | } 170 | }() 171 | 172 | return docCh, errCh 173 | } 174 | -------------------------------------------------------------------------------- /elastic/v7/client.go: -------------------------------------------------------------------------------- 1 | package v7 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io" 7 | "log" 8 | "os" 9 | "strconv" 10 | 11 | elastic7 "github.com/olivere/elastic/v7" 12 | "github.com/pkg/errors" 13 | 14 | "github.com/olivere/esdiff/diff" 15 | "github.com/olivere/esdiff/elastic" 16 | "github.com/olivere/esdiff/elastic/config" 17 | ) 18 | 19 | // Client implements an Elasticsearch 7.x client. 20 | type Client struct { 21 | c *elastic7.Client 22 | index string 23 | typ string 24 | size int 25 | } 26 | 27 | // NewClient creates a new Client. 28 | func NewClient(cfg *config.Config) (*Client, error) { 29 | var options []elastic7.ClientOptionFunc 30 | if cfg != nil { 31 | if cfg.URL != "" { 32 | options = append(options, elastic7.SetURL(cfg.URL)) 33 | } 34 | if cfg.Errorlog != "" { 35 | f, err := os.OpenFile(cfg.Errorlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 36 | if err != nil { 37 | return nil, errors.Wrap(err, "unable to initialize error log") 38 | } 39 | l := log.New(f, "", 0) 40 | options = append(options, elastic7.SetErrorLog(l)) 41 | } 42 | if cfg.Tracelog != "" { 43 | f, err := os.OpenFile(cfg.Tracelog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 44 | if err != nil { 45 | return nil, errors.Wrap(err, "unable to initialize trace log") 46 | } 47 | l := log.New(f, "", 0) 48 | options = append(options, elastic7.SetTraceLog(l)) 49 | } 50 | if cfg.Infolog != "" { 51 | f, err := os.OpenFile(cfg.Infolog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 52 | if err != nil { 53 | return nil, errors.Wrap(err, "unable to initialize info log") 54 | } 55 | l := log.New(f, "", 0) 56 | options = append(options, elastic7.SetInfoLog(l)) 57 | } 58 | if cfg.Username != "" || cfg.Password != "" { 59 | options = append(options, elastic7.SetBasicAuth(cfg.Username, cfg.Password)) 60 | } 61 | options = append(options, elastic7.SetSniff(cfg.Sniff)) 62 | } 63 | cli, err := elastic7.NewClient(options...) 64 | if err != nil { 65 | return nil, err 66 | } 67 | c := &Client{ 68 | c: cli, 69 | index: cfg.Index, 70 | typ: cfg.Type, 71 | size: 100, 72 | } 73 | return c, nil 74 | } 75 | 76 | // SetBatchSize specifies the size of a single scroll operation. 77 | func (c *Client) SetBatchSize(size int) { 78 | c.size = size 79 | } 80 | 81 | // Iterate iterates over the index. 82 | func (c *Client) Iterate(ctx context.Context, req *elastic.IterateRequest) (<-chan *diff.Document, <-chan error) { 83 | docCh := make(chan *diff.Document, 1) 84 | errCh := make(chan error, 1) 85 | 86 | go func() { 87 | defer func() { 88 | close(docCh) 89 | close(errCh) 90 | }() 91 | 92 | // Sorting 93 | var sorter elastic7.Sorter 94 | if req.SortField == "" { 95 | sorter = elastic7.NewFieldSort("_id").Asc() 96 | } else { 97 | field := req.SortField 98 | asc := true 99 | if field[0] == '-' { 100 | field = field[1:] 101 | asc = false 102 | } 103 | sorter = elastic7.NewFieldSort(field).Order(asc) 104 | } 105 | 106 | svc := c.c.Scroll(c.index).Type(c.typ).Size(c.size).SortBy(sorter) 107 | 108 | if req.RawQuery != "" { 109 | q := elastic7.NewRawStringQuery(req.RawQuery) 110 | svc = svc.Query(q) 111 | } 112 | 113 | if len(req.SourceFilterInclude)+len(req.SourceFilterExclude) > 0 { 114 | fsc := elastic7.NewFetchSourceContext(true). 115 | Include(req.SourceFilterInclude...). 116 | Exclude(req.SourceFilterExclude...) 117 | svc = svc.FetchSourceContext(fsc) 118 | } 119 | 120 | for { 121 | res, err := svc.Do(ctx) 122 | if err == io.EOF { 123 | return 124 | } 125 | if err != nil { 126 | errCh <- err 127 | return 128 | } 129 | if res == nil { 130 | errCh <- errors.New("unexpected nil document") 131 | return 132 | } 133 | 134 | if res.Hits == nil { 135 | errCh <- errors.New("unexpected nil hits") 136 | return 137 | } 138 | 139 | for _, hit := range res.Hits.Hits { 140 | doc := new(diff.Document) 141 | err := json.Unmarshal(hit.Source, &doc.Source) 142 | if err != nil { 143 | errCh <- err 144 | return 145 | } 146 | // Replace ID field with some other field from the document? 147 | if req.ReplaceField != "" { 148 | if val, ok := doc.Source[req.ReplaceField]; ok { 149 | switch v := val.(type) { 150 | case string: 151 | doc.ID = v 152 | case int: 153 | doc.ID = strconv.Itoa(v) 154 | case int32: 155 | doc.ID = strconv.FormatInt(int64(v), 10) 156 | case int64: 157 | doc.ID = strconv.FormatInt(v, 10) 158 | case float32: 159 | doc.ID = strconv.Itoa(int(v)) 160 | case float64: 161 | doc.ID = strconv.Itoa(int(v)) 162 | default: 163 | doc.ID = val.(string) 164 | } 165 | } else { 166 | errCh <- errors.New("unexpected replace-with field") 167 | return 168 | } 169 | } else { 170 | doc.ID = hit.Id 171 | } 172 | docCh <- doc 173 | 174 | } 175 | } 176 | }() 177 | 178 | return docCh, errCh 179 | } 180 | -------------------------------------------------------------------------------- /etc/elasticsearch5/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Elasticsearch configuration from elasticsearch-docker. 3 | ## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml 4 | # 5 | cluster.name: "mia5" 6 | network.host: 0.0.0.0 7 | 8 | # minimum_master_nodes need to be explicitly set when bound on a public IP 9 | # set to 1 to allow single node clusters 10 | # Details: https://github.com/elastic/elasticsearch/pull/17288 11 | discovery.zen.minimum_master_nodes: 1 12 | 13 | ## Use single node discovery in order to disable production mode and avoid bootstrap checks 14 | ## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html 15 | # 16 | discovery.type: single-node 17 | -------------------------------------------------------------------------------- /etc/elasticsearch6/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Elasticsearch configuration from elasticsearch-docker. 3 | ## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml 4 | # 5 | cluster.name: "mia6" 6 | network.host: 0.0.0.0 7 | 8 | # minimum_master_nodes need to be explicitly set when bound on a public IP 9 | # set to 1 to allow single node clusters 10 | # Details: https://github.com/elastic/elasticsearch/pull/17288 11 | discovery.zen.minimum_master_nodes: 1 12 | 13 | ## Use single node discovery in order to disable production mode and avoid bootstrap checks 14 | ## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html 15 | # 16 | discovery.type: single-node 17 | -------------------------------------------------------------------------------- /etc/elasticsearch7/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Elasticsearch configuration from elasticsearch-docker. 3 | ## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml 4 | # 5 | cluster.name: "mia7" 6 | network.host: 0.0.0.0 7 | 8 | # minimum_master_nodes need to be explicitly set when bound on a public IP 9 | # set to 1 to allow single node clusters 10 | # Details: https://github.com/elastic/elasticsearch/pull/17288 11 | discovery.zen.minimum_master_nodes: 1 12 | 13 | ## Use single node discovery in order to disable production mode and avoid bootstrap checks 14 | ## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html 15 | # 16 | discovery.type: single-node 17 | -------------------------------------------------------------------------------- /etc/kibana5/kibana.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Default Kibana configuration from kibana-docker. 3 | 4 | server.name: kibana5 5 | server.host: "0" 6 | elasticsearch.url: http://elasticsearch5:9200 7 | # elasticsearch7.username: elastic 8 | # elasticsearch7.password: changeme 9 | # xpack.monitoring.ui.container.elasticsearch7.enabled: false 10 | -------------------------------------------------------------------------------- /etc/kibana6/kibana.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Default Kibana configuration from kibana-docker. 3 | 4 | server.name: kibana6 5 | server.host: "0" 6 | elasticsearch.url: http://elasticsearch6:9200 7 | # elasticsearch6.username: elastic 8 | # elasticsearch6.password: changeme 9 | # xpack.monitoring.ui.container.elasticsearch6.enabled: false 10 | -------------------------------------------------------------------------------- /etc/kibana7/kibana.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Default Kibana configuration from kibana-docker. 3 | 4 | server.name: kibana7 5 | server.host: "0" 6 | elasticsearch.hosts: http://elasticsearch7:9200 7 | # elasticsearch7.username: elastic 8 | # elasticsearch7.password: changeme 9 | # xpack.monitoring.ui.container.elasticsearch7.enabled: false 10 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/olivere/esdiff 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/Masterminds/semver v1.5.0 7 | github.com/fortytw2/leaktest v1.3.0 8 | github.com/google/go-cmp v0.5.6 9 | github.com/olivere/elastic v6.2.37+incompatible 10 | github.com/olivere/elastic/v7 v7.0.31 11 | github.com/pkg/errors v0.9.1 12 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 13 | gopkg.in/olivere/elastic.v5 v5.0.86 14 | ) 15 | 16 | require ( 17 | github.com/josharian/intern v1.0.0 // indirect 18 | github.com/mailru/easyjson v0.7.7 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= 4 | github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= 5 | github.com/aws/aws-sdk-go v1.29.11/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= 6 | github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= 7 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 8 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 9 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 10 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 13 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 14 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 15 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 16 | github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= 17 | github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= 18 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 19 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 20 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 21 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 22 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 23 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 24 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 25 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 26 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 27 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 28 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 29 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 30 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 31 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 32 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 33 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 34 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 35 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 36 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 37 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 38 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 39 | github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 40 | github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= 41 | github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 42 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 43 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 44 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 45 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 46 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 47 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 48 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 49 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 50 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 51 | github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= 52 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= 53 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 54 | github.com/olivere/elastic v6.2.37+incompatible h1:UfSGJem5czY+x/LqxgeCBgjDn6St+z8OnsCuxwD3L0U= 55 | github.com/olivere/elastic v6.2.37+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= 56 | github.com/olivere/elastic/v7 v7.0.12/go.mod h1:14rWX28Pnh3qCKYRVnSGXWLf9MbLonYS/4FDCY3LAPo= 57 | github.com/olivere/elastic/v7 v7.0.31 h1:VJu9/zIsbeiulwlRCfGQf6Tzsr++uo+FeUgj5oj+xKk= 58 | github.com/olivere/elastic/v7 v7.0.31/go.mod h1:idEQxe7Es+Wr4XAuNnJdKeMZufkA9vQprOIFck061vg= 59 | github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 60 | github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= 61 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 62 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 63 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 64 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 65 | github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= 66 | github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= 67 | github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= 68 | github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ= 69 | github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak= 70 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 71 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 72 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 73 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 74 | go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 75 | go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= 76 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 77 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 78 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 79 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 80 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 81 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 82 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 83 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 84 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 85 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 86 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 87 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 88 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 89 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 90 | golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 91 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 92 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 93 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 94 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 95 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 96 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 97 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 98 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 99 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 100 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 101 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 102 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 103 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 104 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 105 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 106 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 107 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 108 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 109 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 110 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 111 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 112 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 113 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 114 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 115 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 116 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 117 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 118 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 119 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 120 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 121 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 122 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 123 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 124 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 125 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 126 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 127 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 128 | google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= 129 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 130 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 131 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 132 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 133 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 134 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 135 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 136 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 137 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 138 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 139 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 140 | gopkg.in/olivere/elastic.v5 v5.0.86 h1:xFy6qRCGAmo5Wjx96srho9BitLhZl2fcnpuidPwduXM= 141 | gopkg.in/olivere/elastic.v5 v5.0.86/go.mod h1:M3WNlsF+WhYn7api4D87NIflwTV/c0iVs8cqfWhK+68= 142 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 143 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 144 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 145 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 146 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 147 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | "os" 11 | "path" 12 | "strings" 13 | 14 | "github.com/Masterminds/semver" 15 | "github.com/pkg/errors" 16 | "golang.org/x/sync/errgroup" 17 | 18 | "github.com/olivere/esdiff/diff" 19 | "github.com/olivere/esdiff/diff/printer" 20 | "github.com/olivere/esdiff/elastic" 21 | "github.com/olivere/esdiff/elastic/config" 22 | v5 "github.com/olivere/esdiff/elastic/v5" 23 | v6 "github.com/olivere/esdiff/elastic/v6" 24 | v7 "github.com/olivere/esdiff/elastic/v7" 25 | ) 26 | 27 | func main() { 28 | var ( 29 | outputFormat = flag.String("o", "", "Output format, e.g. json") 30 | size = flag.Int("size", 100, "Batch size") 31 | rawSrcQuery = flag.String("sf", "", `Raw query for filtering the source, e.g. {"term":{"user":"olivere"}}`) 32 | rawDstQuery = flag.String("df", "", `Raw query for filtering the destination, e.g. {"term":{"name.keyword":"Oliver"}}`) 33 | srcSort = flag.String("ssort", "", `Field to sort the source, e.g. "id" or "-id" (prepend with - for descending)`) 34 | dstSort = flag.String("dsort", "", `Field to sort the destination, e.g. "id" or "-id" (prepend with - for descending)`) 35 | srcFilterInclude = flag.String("include", "", `Raw source filter for including certain fields from the source, e.g. "obj.*"`) 36 | srcFilterExclude = flag.String("exclude", "", `Raw source filter for excluding certain fields from the source, e.g. "hash_value,sub.*"`) 37 | unchanged = flag.Bool("u", false, `Print unchanged docs`) 38 | updated = flag.Bool("c", true, `Print changed docs`) 39 | changed = flag.Bool("a", true, `Print added docs`) 40 | deleted = flag.Bool("d", true, `Print deleted docs`) 41 | replaceWithAnotherField = flag.String("replace-with", "", `replace id field to other field you want`) 42 | ) 43 | 44 | log.SetFlags(0) 45 | flag.Usage = usage 46 | flag.Parse() 47 | 48 | if flag.NArg() != 2 { 49 | usage() 50 | os.Exit(1) 51 | } 52 | 53 | var srcFilterIncludes []string 54 | if *srcFilterInclude != "" { 55 | srcFilterIncludes = strings.Split(*srcFilterInclude, ",") 56 | } 57 | var srcFilterExcludes []string 58 | if *srcFilterExclude != "" { 59 | srcFilterExcludes = strings.Split(*srcFilterExclude, ",") 60 | } 61 | 62 | options := []elastic.ClientOption{ 63 | elastic.WithBatchSize(*size), 64 | } 65 | 66 | src, err := newClient(flag.Arg(0), options...) 67 | if err != nil { 68 | log.Fatal(err) 69 | } 70 | srcIterReq := &elastic.IterateRequest{ 71 | RawQuery: *rawSrcQuery, 72 | SortField: *srcSort, 73 | ReplaceField: *replaceWithAnotherField, 74 | SourceFilterInclude: srcFilterIncludes, 75 | SourceFilterExclude: srcFilterExcludes, 76 | } 77 | dst, err := newClient(flag.Arg(1), options...) 78 | if err != nil { 79 | log.Fatal(err) 80 | } 81 | dstIterReq := &elastic.IterateRequest{ 82 | RawQuery: *rawDstQuery, 83 | SortField: *dstSort, 84 | ReplaceField: *replaceWithAnotherField, 85 | SourceFilterInclude: srcFilterIncludes, 86 | SourceFilterExclude: srcFilterExcludes, 87 | } 88 | var p printer.Printer 89 | { 90 | switch *outputFormat { 91 | default: 92 | p = printer.NewStdPrinter(os.Stdout, *unchanged, *updated, *changed, *deleted) 93 | case "json": 94 | p = printer.NewJSONPrinter(os.Stdout, *unchanged, *updated, *changed, *deleted) 95 | } 96 | } 97 | 98 | g, ctx := errgroup.WithContext(context.Background()) 99 | srcDocCh, srcErrCh := src.Iterate(ctx, srcIterReq) 100 | dstDocCh, dstErrCh := dst.Iterate(ctx, dstIterReq) 101 | diffCh, errCh := diff.Differ(ctx, srcDocCh, dstDocCh) 102 | g.Go(func() error { 103 | for { 104 | select { 105 | case d, ok := <-diffCh: 106 | if !ok { 107 | return nil 108 | } 109 | if err := p.Print(d); err != nil { 110 | return err 111 | } 112 | case <-ctx.Done(): 113 | return ctx.Err() 114 | } 115 | } 116 | }) 117 | g.Go(func() error { 118 | return <-srcErrCh 119 | }) 120 | g.Go(func() error { 121 | return <-dstErrCh 122 | }) 123 | g.Go(func() error { 124 | return <-errCh 125 | }) 126 | if err = g.Wait(); err != nil { 127 | log.Fatal(err) 128 | } 129 | } 130 | 131 | func usage() { 132 | fmt.Fprintf(os.Stderr, "General usage:\n\n") 133 | fmt.Fprintf(os.Stderr, "\t%s [flags] \n\n", path.Base(os.Args[0])) 134 | fmt.Fprintf(os.Stderr, "General flags:\n") 135 | flag.PrintDefaults() 136 | } 137 | 138 | // newClient will create a new Elasticsearch client, 139 | // matching the supported version. 140 | func newClient(url string, opts ...elastic.ClientOption) (elastic.Client, error) { 141 | cfg, err := config.Parse(url) 142 | if err != nil { 143 | return nil, err 144 | } 145 | v, major, _, _, err := elasticsearchVersion(cfg) 146 | if err != nil { 147 | return nil, err 148 | } 149 | switch major { 150 | case 5: 151 | c, err := v5.NewClient(cfg) 152 | if err != nil { 153 | return nil, err 154 | } 155 | for _, opt := range opts { 156 | opt(c) 157 | } 158 | return c, nil 159 | case 6: 160 | c, err := v6.NewClient(cfg) 161 | if err != nil { 162 | return nil, err 163 | } 164 | for _, opt := range opts { 165 | opt(c) 166 | } 167 | return c, nil 168 | case 7: 169 | c, err := v7.NewClient(cfg) 170 | if err != nil { 171 | return nil, err 172 | } 173 | for _, opt := range opts { 174 | opt(c) 175 | } 176 | return c, nil 177 | default: 178 | return nil, errors.Errorf("unsupported Elasticsearch version %s", v) 179 | } 180 | } 181 | 182 | // elasticsearchVersion determines the Elasticsearch option. 183 | func elasticsearchVersion(cfg *config.Config) (string, int64, int64, int64, error) { 184 | type infoType struct { 185 | Name string `json:"name"` 186 | Version struct { 187 | Number string `json:"number"` // e.g. "6.2.4" 188 | } `json:"version"` 189 | } 190 | req, err := http.NewRequest("GET", cfg.URL, nil) 191 | if err != nil { 192 | return "", 0, 0, 0, err 193 | } 194 | if cfg.Username != "" || cfg.Password != "" { 195 | req.SetBasicAuth(cfg.Username, cfg.Password) 196 | } 197 | res, err := http.DefaultClient.Do(req) 198 | if err != nil { 199 | return "", 0, 0, 0, err 200 | } 201 | defer res.Body.Close() 202 | var info infoType 203 | if err = json.NewDecoder(res.Body).Decode(&info); err != nil { 204 | return "", 0, 0, 0, err 205 | } 206 | v, err := semver.NewVersion(info.Version.Number) 207 | if err != nil { 208 | return info.Version.Number, 0, 0, 0, err 209 | } 210 | return info.Version.Number, v.Major(), v.Minor(), v.Patch(), nil 211 | } 212 | -------------------------------------------------------------------------------- /seed/01.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | curl -H 'Content-Type: application/json' -XDELETE 'localhost:19200/index01' 3 | curl -H 'Content-Type: application/json' -XDELETE 'localhost:29200/index01' 4 | curl -H 'Content-Type: application/json' -XDELETE 'localhost:39200/index01' 5 | 6 | # Create mappings 7 | curl -X PUT "localhost:19200/index01" -H 'Content-Type: application/json' -d' 8 | { 9 | "mappings": { 10 | "tweet": { 11 | "properties": { 12 | "user": { "type": "keyword" }, 13 | "message": { "type": "keyword" } 14 | } 15 | } 16 | } 17 | } 18 | ' 19 | 20 | curl -X PUT "localhost:29200/index01" -H 'Content-Type: application/json' -d' 21 | { 22 | "mappings": { 23 | "_doc": { 24 | "properties": { 25 | "user": { "type": "keyword" }, 26 | "message": { "type": "keyword" } 27 | } 28 | } 29 | } 30 | } 31 | ' 32 | 33 | curl -X PUT "localhost:39200/index01" -H 'Content-Type: application/json' -d' 34 | { 35 | "mappings": { 36 | "properties": { 37 | "user": { "type": "keyword" }, 38 | "message": { "type": "keyword" } 39 | } 40 | } 41 | } 42 | ' 43 | 44 | # Add documents 45 | curl -H 'Content-Type: application/json' -XPUT 'localhost:19200/index01/tweet/1' -d '{"user":"olivere","message":"Welcome to Golang"}' 46 | curl -H 'Content-Type: application/json' -XPUT 'localhost:19200/index01/tweet/2' -d '{"user":"olivere","message":"Running is fun"}' 47 | curl -H 'Content-Type: application/json' -XPUT 'localhost:19200/index01/tweet/3' -d '{"user":"sandrae","message":"Playing the piano is fun as well"}' 48 | 49 | curl -H 'Content-Type: application/json' -XPUT 'localhost:29200/index01/_doc/1' -d '{"user":"olivere","message":"Welcome to Golang"}' 50 | curl -H 'Content-Type: application/json' -XPUT 'localhost:29200/index01/_doc/3' -d '{"user":"sandrae","message":"Playing the guitar is fun as well"}' 51 | curl -H 'Content-Type: application/json' -XPUT 'localhost:29200/index01/_doc/4' -d '{"user":"sandrae","message":"Climbed that mountain"}' 52 | 53 | curl -H 'Content-Type: application/json' -XPUT 'localhost:39200/index01/_doc/1' -d '{"user":"olivere","message":"Welcome to Golang"}' 54 | curl -H 'Content-Type: application/json' -XPUT 'localhost:39200/index01/_doc/3' -d '{"user":"sandrae","message":"Playing the flute, oh boy"}' 55 | curl -H 'Content-Type: application/json' -XPUT 'localhost:39200/index01/_doc/5' -d '{"user":"sandrae","message":"Ran that marathon"}' 56 | --------------------------------------------------------------------------------