├── .github ├── FUNDING.yml └── workflows │ ├── go.yml │ └── goreleaser.yml ├── .gitignore ├── .goreleaser.yml ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── export └── export.go ├── flags └── flags.go ├── formats ├── csv.go ├── csv_test.go ├── json.go └── raw.go ├── go.mod ├── go.sum └── main.go /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: pteich 2 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ "*" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | go-version: [ '1.21.x', '1.22.x', '1.23.x' ] 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | with: 20 | fetch-depth: 0 21 | 22 | - name: Setup Go ${{ matrix.go-version }} 23 | uses: actions/setup-go@v5 24 | with: 25 | go-version: ${{ matrix.go-version }} 26 | 27 | - name: Download dependencies 28 | run: go mod tidy 29 | 30 | - name: Build 31 | run: go build -v ./... 32 | 33 | - name: Test 34 | run: go test -v ./... 35 | -------------------------------------------------------------------------------- /.github/workflows/goreleaser.yml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | permissions: 9 | contents: write 10 | 11 | jobs: 12 | goreleaser: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - 16 | name: Checkout 17 | uses: actions/checkout@v4 18 | with: 19 | fetch-depth: 0 20 | - 21 | name: Set up Go 22 | uses: actions/setup-go@v5 23 | 24 | - 25 | name: Login to Docker Hub 26 | uses: docker/login-action@v3 27 | with: 28 | registry: ghcr.io 29 | username: $GITHUB_ACTOR 30 | password: ${{ secrets.GH_GORELEASER }} 31 | 32 | - 33 | name: Download Syft 34 | uses: anchore/sbom-action/download-syft@v0.17.4 35 | 36 | - 37 | name: Run GoReleaser 38 | uses: goreleaser/goreleaser-action@v6 39 | with: 40 | distribution: goreleaser 41 | version: '~> v2' 42 | args: release --clean 43 | env: 44 | GITHUB_TOKEN: ${{ secrets.GH_GORELEASER }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.csv 3 | vendor 4 | bin 5 | dist 6 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://goreleaser.com/static/schema.json 2 | version: 2 3 | before: 4 | hooks: 5 | # You may remove this if you don't use go modules. 6 | - go mod tidy 7 | universal_binaries: 8 | - replace: true 9 | builds: 10 | - env: 11 | - CGO_ENABLED=0 12 | goos: 13 | - linux 14 | - windows 15 | - darwin 16 | - freebsd 17 | goarch: 18 | - amd64 19 | - arm64 20 | ldflags: 21 | - -s -w 22 | - -X main.version={{.Version}} 23 | - -X main.commit={{.ShortCommit}} 24 | - -X main.Date={{.CommitDate}} 25 | 26 | archives: 27 | - format: tar.gz 28 | format_overrides: 29 | - goos: windows 30 | format: zip 31 | - goos: darwin 32 | format: zip 33 | 34 | checksum: 35 | name_template: 'checksums.txt' 36 | 37 | snapshot: 38 | version_template: "{{ incpatch .Version }}-next" 39 | 40 | changelog: 41 | use: github 42 | sort: asc 43 | filters: 44 | exclude: 45 | - '^docs:' 46 | - typo 47 | - Goreleaser 48 | - Dockerfile 49 | - CI 50 | - '^test:' 51 | 52 | sboms: 53 | - artifacts: archive 54 | 55 | nfpms: 56 | - file_name_template: "{{ .ConventionalFileName }}" 57 | formats: 58 | - deb 59 | - rpm 60 | - apk 61 | - archlinux 62 | dependencies: 63 | - "bash" 64 | maintainer: "Peter Teich " 65 | vendor: "Peter Teich" 66 | homepage: "https://github.com/pteich" 67 | description: "Export Data from ElasticSearch to CSV/JSON using a Lucene Query (e.g. from Kibana) or a raw JSON Query string" 68 | license: "MIT" 69 | 70 | dockers: 71 | - image_templates: ["ghcr.io/pteich/elastic-query-export:{{ .Version }}"] 72 | dockerfile: Dockerfile 73 | build_flag_templates: 74 | - --label=org.opencontainers.image.title={{ .ProjectName }} 75 | - --label=org.opencontainers.image.description={{ .ProjectName }} 76 | - --label=org.opencontainers.image.url=https://github.com/pteich/elastic-query-export 77 | - --label=org.opencontainers.image.source=https://github.com/pteich/elastic-query-export 78 | - --label=org.opencontainers.image.version={{ .Version }} 79 | - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} 80 | - --label=org.opencontainers.image.revision={{ .FullCommit }} 81 | - --label=org.opencontainers.image.licenses=MIT 82 | 83 | brews: 84 | - name: elastic-query-export 85 | homepage: https://github.com/pteich/elastic-query-export 86 | repository: 87 | owner: pteich 88 | name: homebrew-tap 89 | 90 | release: 91 | draft: true 92 | replace_existing_draft: true 93 | replace_existing_artifacts: true -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.14 4 | env: 5 | - GO111MODULE="on" 6 | before_install: 7 | - go get -u github.com/mitchellh/gox 8 | install: 9 | - go mod download 10 | script: 11 | - go test -v -covermode=count -coverprofile=coverage.out ./... 12 | after_success: 13 | - make build-gox 14 | deploy: 15 | provider: releases 16 | api_key: 17 | secure: oxxAVwvp7iB+XXQsDmyK9SBnmIiSzepaoq0Yk0NVYA+CDhMHuiaqjwhpil31qhKp1pieX9MY9hyX6+WMbJuwv3Vm3H2Ut4GtlK83y/mrcsPi+zTPII6l8ThRjVCrD7vFTMX7R+3N9PK6BTFe6tND7nRcAnrHm8P9QGfFDuupRTna7W5k7TEPVg29S6dT2FCiWdApl+b/h5Y/0VzaS63XZ4Fap/rdcyCg1UKzLtnGL85wYXJPAjH05ZZNyvY1Fk+vj7Rwk1RbaIxi/gAmA4WOb1hscgxtXNI2OCUfg70m/KOVSjUkEamKcbqsQecQgoiuCUNQL9jkhHAXGmSBKiGWMMEMKxCaVaJizY5NjjZbtn1IOkOu44wUYiHCyHeL2keA1vY2EDf9igyTdYFL6Ee9mBf9JSMNERAHhN1jRj6ZpQ2g8u57+l1bMw9E0B1lvP2HolOq9lvEdu0aHghHslNJYRTdi6qVeIUmamZ4UdXJLwdixxmCroZpR/DqmzA5u9NckRjFINTiU+CJtKYVSg4ENrLn3F2tN5cEFTi5RWzoHjNzOMP+yL+zZqzmiwLKcKH42b1kSItYTwAvt/4EFk3xpQVDqpe/vsDYlkJwNYVH0ylyM1yaiXYpBIWpFleKwmG2kn1S+I6GKmZJlLNGkBkr8EBfR2Xh182q65rxO9Jy5cE= 18 | file_glob: true 19 | file: bin/* 20 | skip_cleanup: true 21 | on: 22 | tags: true 23 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:latest 2 | LABEL org.opencontainers.image.source https://github.com/pteich/elastic-query-export 3 | 4 | COPY elastic-query-export /usr/bin/elastic-query-export 5 | 6 | ENTRYPOINT ["/usr/bin/elastic-query-export"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Peter Teich 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BINARY=es-query-csv 2 | VERSION=v1.4.0 3 | BUILD_TIME=`date +%FT%T%z` 4 | GOX_OSARCH="darwin/amd64 darwin/arm64 linux/386 linux/amd64 windows/386 windows/amd64" 5 | 6 | default: build 7 | 8 | clean: 9 | rm -rf ./bin 10 | 11 | build: 12 | CGO_ENABLED=0 \ 13 | go build -a -o ./bin/${BINARY}-${VERSION} *.go 14 | 15 | build-linux: 16 | CGO_ENABLED=0 \ 17 | GOARCH=amd64 \ 18 | GOOS=linux \ 19 | go build -ldflags "-X main.Version=${VERSION}" -a -o ./bin/${BINARY}-${VERSION} *.go 20 | 21 | build-gox: 22 | gox -ldflags "-X main.Version=${VERSION}" -osarch=${GOX_OSARCH} -output="bin/${VERSION}/{{.Dir}}_{{.OS}}_{{.Arch}}" 23 | 24 | deps: 25 | dep ensure; 26 | 27 | test: 28 | go test 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # elastic-query-export 2 | 3 | Export Data from ElasticSearch to CSV by Raw or Lucene Query (e.g. from Kibana). 4 | Works with ElasticSearch 6+ (OpenSearch works too) and makes use of ElasticSearch's Scroll API and Go's 5 | concurrency features to work as fast as possible. 6 | 7 | ## Install 8 | 9 | Download a pre-compiled binary for your operating system from here: https://github.com/pteich/elastic-query-export/releases 10 | You need just this binary. It works on OSX (Darwin), Linux and Windows. 11 | 12 | There are also prebuilt RPM, DEB and APK packages for your Linux distribution. 13 | 14 | ### Brew 15 | 16 | Use Brew to install: 17 | ```shell 18 | brew tap pteich/tap 19 | brew install elastic-query-export 20 | ``` 21 | 22 | ### Arch AUR 23 | 24 | ```shell 25 | yay -S elastic-query-export-bin 26 | ``` 27 | 28 | ### Docker 29 | 30 | A Docker image is available here: https://github.com/pteich/elastic-query-export/pkgs/container/elastic-query-export 31 | It can be used just like the locally installed binary: 32 | 33 | ```shell 34 | docker run ghcr.io/pteich/elastic-query-export:1.6.2 -h 35 | ``` 36 | 37 | 38 | ## General usage 39 | 40 | ````shell 41 | es-query-export -c "http://localhost:9200" -i "logstash-*" --start="2019-04-04T12:15:00" --fields="RemoteHost,RequestTime,Timestamp,RequestUri,RequestProtocol,Agent" -q "RequestUri:*export*" 42 | ```` 43 | 44 | ## CLI Options 45 | 46 | | Flag | Default | | 47 | |------------------|-----------------------|---------------------------------------------------------------------------------------------------------| 48 | | `-h --help` | | show help | 49 | | `-v --version` | | show version | 50 | | `-c --connect` | http://localhost:9200 | URI to ElasticSearch instance | 51 | | `-i --index` | logs-* | name of index to use, use globbing characters * to match multiple | 52 | | `-q --query` | | Lucene query to match documents (same as in Kibana) | 53 | | ` --fields` | | define a comma separated list of fields to export | 54 | | `-o --outfile` | output.csv | name of output file, you can use `-` as filename to output data to stdout and pipe it to other commands | 55 | | `-f --outformat` | csv | format of the output data: possible values csv, json, raw | 56 | | `-r --rawquery` | | optional raw ElasticSearch query JSON string | 57 | | `-s --start` | | optional start date - Format: YYYY-MM-DDThh:mm:ss.SSSZ. or any other Elasticsearch default format | 58 | | `-e --end` | | optional end date - Format: YYYY-MM-DDThh:mm:ss.SSSZ. or any other Elasticsearch default format | 59 | | `--timefield` | | optional time field to use, default to @timestamp | 60 | | `--verifySSL` | false | optional define how to handle SSL certificates | 61 | | `--user` | | optional username | 62 | | `--pass` | | optional password | 63 | | `--size` | 1000 | size of the scroll window, the more the faster the export works but it adds more pressure on your nodes | 64 | | `--trace` | false | enable trace mode to debug queries send to ElasticSearch | 65 | 66 | ## Output Formats 67 | 68 | - `csv` - all or selected fields separated by comma (,) with field names in the first line 69 | - `json` - all or selected fields as JSON objects, one per line 70 | - `raw` - JSON dump of matching documents including id, index and _source field containing the document data. One document as JSON object per line. 71 | 72 | ## Pipe output to other commands 73 | 74 | Since v1.6.0 you can provide `-` as filename and send output to stdout. This can be used to pipe it to other commands like so: 75 | 76 | ```shell 77 | es-query-export -start="2019-04-04T12:15:00" -q "RequestUri:*export*" -outfile - | aws s3 cp - s3://mybucket/stream.csv 78 | ``` -------------------------------------------------------------------------------- /export/export.go: -------------------------------------------------------------------------------- 1 | package export 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "errors" 7 | "io" 8 | "log" 9 | "net/http" 10 | "os" 11 | "strings" 12 | "time" 13 | 14 | "github.com/olivere/elastic/v7" 15 | "gopkg.in/cheggaaa/pb.v2" 16 | 17 | "github.com/pteich/elastic-query-export/flags" 18 | "github.com/pteich/elastic-query-export/formats" 19 | ) 20 | 21 | const workers = 8 22 | 23 | // Formatter defines how an output formatter has to look like 24 | type Formatter interface { 25 | Run(context.Context, <-chan *elastic.SearchHit) error 26 | } 27 | 28 | // Run starts the export of Elastic data 29 | func Run(ctx context.Context, conf *flags.Flags) { 30 | tlsCfg := &tls.Config{ 31 | InsecureSkipVerify: !conf.ElasticVerifySSL, 32 | } 33 | 34 | if conf.ElasticClientCrt != "" && conf.ElasticClientKey != "" { 35 | cert, err := tls.LoadX509KeyPair(conf.ElasticClientCrt, conf.ElasticClientKey) 36 | if err != nil { 37 | log.Fatalf("Error loading client certificate: %s", err) 38 | } 39 | tlsCfg.Certificates = []tls.Certificate{cert} 40 | 41 | } 42 | 43 | tr := &http.Transport{ 44 | TLSClientConfig: tlsCfg, 45 | } 46 | httpClient := &http.Client{Transport: tr} 47 | 48 | esOpts := make([]elastic.ClientOptionFunc, 0) 49 | esOpts = append(esOpts, 50 | elastic.SetHttpClient(httpClient), 51 | elastic.SetURL(conf.ElasticURL), 52 | elastic.SetSniff(false), 53 | elastic.SetHealthcheckInterval(60*time.Second), 54 | elastic.SetErrorLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags)), 55 | ) 56 | 57 | if conf.Trace { 58 | esOpts = append(esOpts, elastic.SetTraceLog(log.New(os.Stderr, "ELASTIC ", log.LstdFlags))) 59 | } 60 | 61 | if conf.ElasticUser != "" && conf.ElasticPass != "" { 62 | esOpts = append(esOpts, elastic.SetBasicAuth(conf.ElasticUser, conf.ElasticPass)) 63 | } 64 | 65 | client, err := elastic.NewClient(esOpts...) 66 | if err != nil { 67 | log.Fatalf("Error connecting to ElasticSearch: %s", err) 68 | } 69 | defer client.Stop() 70 | 71 | if conf.Fieldlist != "" { 72 | conf.Fields = strings.Split(conf.Fieldlist, ",") 73 | } 74 | 75 | var outfile *os.File 76 | 77 | if conf.Outfile == "-" { 78 | outfile = os.Stdout 79 | } else { 80 | outfile, err = os.Create(conf.Outfile) 81 | if err != nil { 82 | log.Fatalf("Error creating output file - %s", err) 83 | } 84 | defer outfile.Close() 85 | } 86 | 87 | var rangeQuery *elastic.RangeQuery 88 | 89 | esQuery := elastic.NewBoolQuery() 90 | 91 | if conf.StartDate != "" && conf.EndDate != "" { 92 | rangeQuery = elastic.NewRangeQuery(conf.Timefield).Gte(conf.StartDate).Lte(conf.EndDate) 93 | } else if conf.StartDate != "" { 94 | rangeQuery = elastic.NewRangeQuery(conf.Timefield).Gte(conf.StartDate) 95 | } else if conf.EndDate != "" { 96 | rangeQuery = elastic.NewRangeQuery(conf.Timefield).Lte(conf.EndDate) 97 | } else { 98 | rangeQuery = nil 99 | } 100 | 101 | if rangeQuery != nil { 102 | esQuery = esQuery.Filter(rangeQuery) 103 | } 104 | 105 | if conf.RAWQuery != "" { 106 | esQuery = esQuery.Must(elastic.NewRawStringQuery(conf.RAWQuery)) 107 | } else if conf.Query != "" { 108 | esQuery = esQuery.Must(elastic.NewQueryStringQuery(conf.Query)) 109 | } else { 110 | esQuery = esQuery.Must(elastic.NewMatchAllQuery()) 111 | } 112 | 113 | // Count total and setup progress 114 | total, err := client.Count(conf.Index).Query(esQuery).Do(ctx) 115 | if err != nil { 116 | log.Fatalf("Error counting ElasticSearch documents: %s", err) 117 | } 118 | bar := pb.StartNew(int(total)) 119 | 120 | hits := make(chan *elastic.SearchHit) 121 | 122 | go func() { 123 | defer close(hits) 124 | 125 | scroll := client.Scroll(conf.Index).Size(conf.ScrollSize).Query(esQuery) 126 | defer scroll.Clear(ctx) 127 | 128 | // include selected fields otherwise export all 129 | if conf.Fields != nil { 130 | fetchSource := elastic.NewFetchSourceContext(true) 131 | for _, field := range conf.Fields { 132 | fetchSource.Include(field) 133 | } 134 | scroll = scroll.FetchSourceContext(fetchSource) 135 | } 136 | 137 | for { 138 | results, err := scroll.Do(ctx) 139 | if err != nil { 140 | if errors.Is(err, io.EOF) { 141 | return // all results retrieved 142 | } 143 | 144 | log.Println(err) 145 | return // something went wrong 146 | } 147 | 148 | // Send the hits to the hits channel 149 | for _, hit := range results.Hits.Hits { 150 | // Check if we need to terminate early 151 | select { 152 | case hits <- hit: 153 | case <-ctx.Done(): 154 | return 155 | } 156 | } 157 | } 158 | }() 159 | 160 | var output Formatter 161 | switch conf.OutFormat { 162 | case flags.FormatJSON: 163 | output = formats.JSON{ 164 | Outfile: outfile, 165 | ProgessBar: bar, 166 | } 167 | case flags.FormatRAW: 168 | output = formats.Raw{ 169 | Outfile: outfile, 170 | ProgessBar: bar, 171 | } 172 | default: 173 | output = formats.CSV{ 174 | Conf: conf, 175 | Outfile: outfile, 176 | Workers: workers, 177 | ProgessBar: bar, 178 | } 179 | } 180 | 181 | err = output.Run(ctx, hits) 182 | if err != nil { 183 | log.Printf("Failed to write output: %s", err) 184 | } 185 | 186 | bar.Finish() 187 | } 188 | -------------------------------------------------------------------------------- /flags/flags.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | const ( 4 | FormatCSV = "csv" 5 | FormatJSON = "json" 6 | FormatRAW = "raw" 7 | ) 8 | 9 | type Flags struct { 10 | ElasticURL string `cli:"connect" cliAlt:"c" usage:"ElasticSearch URL"` 11 | ElasticUser string `cli:"user" usage:"ElasticSearch Username"` 12 | ElasticPass string `cli:"pass" usage:"ElasticSearch Password"` 13 | ElasticVerifySSL bool `cli:"verifySSL" usage:"Verify SSL certificate"` 14 | ElasticClientCrt string `cli:"clientCRT" usage:"Path to client certificate"` 15 | ElasticClientKey string `cli:"clientKey" usage:"Path to client certificate key"` 16 | Index string `cli:"index" cliAlt:"i" usage:"ElasticSearch Index (or Index Prefix)"` 17 | RAWQuery string `cli:"rawquery" cliAlt:"r" usage:"ElasticSearch raw query string"` 18 | Query string `cli:"query" cliAlt:"q" usage:"Lucene query same that is used in Kibana search input"` 19 | OutFormat string `cli:"outformat" cliAlt:"f" usage:"Format of the output data. [json|csv]"` 20 | Outfile string `cli:"outfile" cliAlt:"o" usage:"Path to output file"` 21 | StartDate string `cli:"start" cliAlt:"s" usage:"Start date for included documents"` 22 | EndDate string `cli:"end" cliAlt:"e" usage:"End date for included documents"` 23 | ScrollSize int `cli:"size" usage:"Number of documents that will be returned per shard"` 24 | Timefield string `cli:"timefield" usage:"Field name to use for start and end date query"` 25 | Fieldlist string `cli:"fields" usage:"Fields to include in export as comma separated list"` 26 | Trace bool `cli:"trace" usage:"Enable debug output"` 27 | Fields []string 28 | } 29 | -------------------------------------------------------------------------------- /formats/csv.go: -------------------------------------------------------------------------------- 1 | package formats 2 | 3 | import ( 4 | "context" 5 | "encoding/csv" 6 | "encoding/json" 7 | "fmt" 8 | "log" 9 | "os" 10 | "regexp" 11 | "sync" 12 | 13 | "github.com/olivere/elastic/v7" 14 | "golang.org/x/sync/errgroup" 15 | "gopkg.in/cheggaaa/pb.v2" 16 | 17 | "github.com/pteich/elastic-query-export/flags" 18 | ) 19 | 20 | type CSV struct { 21 | Conf *flags.Flags 22 | Outfile *os.File 23 | Workers int 24 | ProgessBar *pb.ProgressBar 25 | } 26 | 27 | func (c CSV) Run(ctx context.Context, hits <-chan *elastic.SearchHit) error { 28 | g, ctx := errgroup.WithContext(ctx) 29 | 30 | csvout := make(chan []string, c.Workers) 31 | defer close(csvout) 32 | 33 | go func() { 34 | w := csv.NewWriter(c.Outfile) 35 | 36 | for csvdata := range csvout { 37 | if err := w.Write(csvdata); err != nil { 38 | log.Printf("Error writing CSV data - %v", err) 39 | } 40 | 41 | w.Flush() 42 | c.ProgessBar.Increment() 43 | } 44 | 45 | }() 46 | 47 | sendHeader := sync.Once{} 48 | fields := c.Conf.Fields 49 | headerSent := make(chan struct{}) 50 | 51 | for i := 0; i < c.Workers; i++ { 52 | g.Go(func() error { 53 | 54 | for hit := range hits { 55 | var document map[string]interface{} 56 | var csvdata []string 57 | var outdata string 58 | 59 | if err := json.Unmarshal(hit.Source, &document); err != nil { 60 | log.Printf("Error unmarshal JSON from ElasticSearch - %v", err) 61 | } 62 | 63 | document = flatten(document) 64 | 65 | sendHeader.Do(func() { 66 | if c.Conf.Fields == nil { 67 | for key := range document { 68 | fields = append(fields, key) 69 | } 70 | } 71 | csvout <- fields 72 | close(headerSent) 73 | }) 74 | 75 | <-headerSent 76 | 77 | for _, field := range fields { 78 | if val, ok := document[field]; ok { 79 | if val == nil { 80 | csvdata = append(csvdata, "") 81 | continue 82 | } 83 | 84 | // this type switch is probably not really needed anymore 85 | switch val := val.(type) { 86 | case int64: 87 | outdata = fmt.Sprintf("%d", val) 88 | case float64: 89 | d := int(val) 90 | if val == float64(d) { 91 | outdata = fmt.Sprintf("%d", d) 92 | } else { 93 | outdata = fmt.Sprintf("%f", val) 94 | } 95 | default: 96 | outdata = removeLBR(fmt.Sprintf("%v", val)) 97 | } 98 | 99 | csvdata = append(csvdata, outdata) 100 | } else { 101 | csvdata = append(csvdata, "") 102 | } 103 | } 104 | 105 | // send string array to csv output 106 | csvout <- csvdata 107 | 108 | select { 109 | default: 110 | case <-ctx.Done(): 111 | return ctx.Err() 112 | } 113 | } 114 | return nil 115 | }) 116 | } 117 | 118 | return g.Wait() 119 | } 120 | 121 | func flatten(document map[string]interface{}) map[string]interface{} { 122 | result := map[string]interface{}{} 123 | 124 | for key, value := range document { 125 | result[key] = value 126 | 127 | childDocument, ok := value.(map[string]interface{}) 128 | if !ok { 129 | continue 130 | } 131 | 132 | for subKey, subValue := range flatten(childDocument) { 133 | result[key+"."+subKey] = subValue 134 | } 135 | } 136 | 137 | return result 138 | } 139 | 140 | func removeLBR(text string) string { 141 | re := regexp.MustCompile(`\x{000D}\x{000A}|[\x{000A}\x{000B}\x{000C}\x{000D}\x{0085}\x{2028}\x{2029}]`) 142 | return re.ReplaceAllString(text, ``) 143 | } 144 | -------------------------------------------------------------------------------- /formats/csv_test.go: -------------------------------------------------------------------------------- 1 | package formats 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func Test_flatten(t *testing.T) { 9 | tests := []struct { 10 | name string 11 | document map[string]interface{} 12 | want map[string]interface{} 13 | }{ 14 | { 15 | "simple", 16 | map[string]interface{}{ 17 | "string": "1", 18 | "int": 2, 19 | "float": 3.0, 20 | "bool": true, 21 | "slice": []string{"a", "b", "c"}, 22 | }, 23 | map[string]interface{}{ 24 | "string": "1", 25 | "int": 2, 26 | "float": 3.0, 27 | "bool": true, 28 | "slice": []string{"a", "b", "c"}, 29 | }, 30 | }, 31 | { 32 | "nested", 33 | map[string]interface{}{ 34 | "string": "value1", 35 | "map": map[string]interface{}{ 36 | "string": "value2", 37 | "int": 2, 38 | "float": 3.0, 39 | "bool": true, 40 | "slice": []string{"a", "b", "c"}, 41 | }, 42 | }, 43 | map[string]interface{}{ 44 | "string": "value1", 45 | "map": map[string]interface{}{ 46 | "string": "value2", 47 | "int": 2, 48 | "float": 3.0, 49 | "bool": true, 50 | "slice": []string{"a", "b", "c"}, 51 | }, 52 | "map.string": "value2", 53 | "map.int": 2, 54 | "map.float": 3.0, 55 | "map.bool": true, 56 | "map.slice": []string{"a", "b", "c"}, 57 | }, 58 | }, 59 | { 60 | "very nested", 61 | map[string]interface{}{ 62 | "string": "value1", 63 | "map": map[string]interface{}{ 64 | "string": "value2", 65 | "int": 2, 66 | "float": 3.0, 67 | "bool": true, 68 | "slice": []string{"a", "b", "c"}, 69 | "map": map[string]interface{}{ 70 | "string": "value3", 71 | "int": 2, 72 | "float": 3.0, 73 | "bool": true, 74 | "slice": []string{"a", "b", "c"}, 75 | }, 76 | }, 77 | }, 78 | map[string]interface{}{ 79 | "string": "value1", 80 | "map": map[string]interface{}{ 81 | "string": "value2", 82 | "int": 2, 83 | "float": 3.0, 84 | "bool": true, 85 | "slice": []string{"a", "b", "c"}, 86 | "map": map[string]interface{}{ 87 | "string": "value3", 88 | "int": 2, 89 | "float": 3.0, 90 | "bool": true, 91 | "slice": []string{"a", "b", "c"}, 92 | }, 93 | }, 94 | "map.string": "value2", 95 | "map.int": 2, 96 | "map.float": 3.0, 97 | "map.bool": true, 98 | "map.slice": []string{"a", "b", "c"}, 99 | "map.map": map[string]interface{}{ 100 | "string": "value3", 101 | "int": 2, 102 | "float": 3.0, 103 | "bool": true, 104 | "slice": []string{"a", "b", "c"}, 105 | }, 106 | "map.map.string": "value3", 107 | "map.map.int": 2, 108 | "map.map.float": 3.0, 109 | "map.map.bool": true, 110 | "map.map.slice": []string{"a", "b", "c"}, 111 | }, 112 | }, 113 | } 114 | for _, tt := range tests { 115 | t.Run(tt.name, func(t *testing.T) { 116 | if got := flatten(tt.document); !reflect.DeepEqual(got, tt.want) { 117 | t.Errorf("flatten() = %v, want %v", got, tt.want) 118 | } 119 | }) 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /formats/json.go: -------------------------------------------------------------------------------- 1 | package formats 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/olivere/elastic/v7" 9 | "gopkg.in/cheggaaa/pb.v2" 10 | ) 11 | 12 | type JSON struct { 13 | Outfile *os.File 14 | ProgessBar *pb.ProgressBar 15 | } 16 | 17 | func (j JSON) Run(ctx context.Context, hits <-chan *elastic.SearchHit) error { 18 | for hit := range hits { 19 | fmt.Fprintln(j.Outfile, string(hit.Source)) 20 | j.ProgessBar.Increment() 21 | 22 | select { 23 | case <-ctx.Done(): 24 | return ctx.Err() 25 | default: 26 | } 27 | } 28 | 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /formats/raw.go: -------------------------------------------------------------------------------- 1 | package formats 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "os" 9 | 10 | "github.com/olivere/elastic/v7" 11 | "gopkg.in/cheggaaa/pb.v2" 12 | ) 13 | 14 | type Raw struct { 15 | Outfile *os.File 16 | ProgessBar *pb.ProgressBar 17 | } 18 | 19 | func (r Raw) Run(ctx context.Context, hits <-chan *elastic.SearchHit) error { 20 | for hit := range hits { 21 | data, err := json.Marshal(hit) 22 | if err != nil { 23 | log.Println(err) 24 | continue 25 | } 26 | 27 | fmt.Fprintln(r.Outfile, string(data)) 28 | r.ProgessBar.Increment() 29 | } 30 | 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/pteich/elastic-query-export 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/olivere/elastic/v7 v7.0.32 7 | github.com/pteich/configstruct v1.6.0 8 | golang.org/x/sync v0.10.0 9 | gopkg.in/cheggaaa/pb.v2 v2.0.7 10 | ) 11 | 12 | require ( 13 | github.com/josharian/intern v1.0.0 // indirect 14 | github.com/mailru/easyjson v0.9.0 // indirect 15 | github.com/mattn/go-colorable v0.1.14 // indirect 16 | github.com/mattn/go-isatty v0.0.20 // indirect 17 | github.com/pkg/errors v0.9.1 // indirect 18 | golang.org/x/sys v0.29.0 // indirect 19 | gopkg.in/VividCortex/ewma.v1 v1.1.1 // indirect 20 | gopkg.in/fatih/color.v1 v1.7.0 // indirect 21 | gopkg.in/mattn/go-colorable.v0 v0.1.0 // indirect 22 | gopkg.in/mattn/go-isatty.v0 v0.0.4 // indirect 23 | gopkg.in/mattn/go-runewidth.v0 v0.0.4 // indirect 24 | gopkg.in/yaml.v3 v3.0.1 // indirect 25 | ) 26 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/c-bata/go-prompt v0.2.6/go.mod h1:/LMAke8wD2FsNu9EXNdHxNLbd9MedkPnCdfpU9wwHfY= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= 6 | github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= 7 | github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= 8 | github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= 9 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 10 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 11 | github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= 12 | github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= 13 | github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 14 | github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 15 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 16 | github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 17 | github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 18 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 19 | github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= 20 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 21 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 22 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 23 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 24 | github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 25 | github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 26 | github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 27 | github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 28 | github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= 29 | github.com/mattn/go-tty v0.0.5/go.mod h1:u5GGXBtZU6RQoKV8gY5W6UhMudbR5vXnUe7j3pxse28= 30 | github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E= 31 | github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k= 32 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 33 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 34 | github.com/pkg/term v1.2.0-beta.2/go.mod h1:E25nymQcrSllhX42Ok8MRm1+hyBdHY0dCeiKZ9jpNGw= 35 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 36 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 37 | github.com/pteich/configstruct v1.6.0 h1:F5/PuWiukfRoUXJCfD/+20DPgaxRfB8uLvX3LSRRkfo= 38 | github.com/pteich/configstruct v1.6.0/go.mod h1:G6MAPCHmkYMn3Fo7NILgnAJ5ZjQwOTKrHa7CHkVMCYI= 39 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 40 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 41 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 42 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 43 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 44 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 45 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 46 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 47 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 48 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 49 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 50 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 51 | golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= 52 | golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 53 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 54 | golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 55 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 56 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 57 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 58 | golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 59 | golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 60 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 61 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 62 | golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 63 | golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= 64 | golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 65 | gopkg.in/VividCortex/ewma.v1 v1.1.1 h1:tWHEKkKq802K/JT9RiqGCBU5fW3raAPnJGTE9ostZvg= 66 | gopkg.in/VividCortex/ewma.v1 v1.1.1/go.mod h1:TekXuFipeiHWiAlO1+wSS23vTcyFau5u3rxXUSXj710= 67 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 68 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 69 | gopkg.in/cheggaaa/pb.v2 v2.0.7 h1:beaAg8eacCdMQS9Y7obFEtkY7gQl0uZ6Zayb3ry41VY= 70 | gopkg.in/cheggaaa/pb.v2 v2.0.7/go.mod h1:0CiZ1p8pvtxBlQpLXkHuUTpdJ1shm3OqCF1QugkjHL4= 71 | gopkg.in/fatih/color.v1 v1.7.0 h1:bYGjb+HezBM6j/QmgBfgm1adxHpzzrss6bj4r9ROppk= 72 | gopkg.in/fatih/color.v1 v1.7.0/go.mod h1:P7yosIhqIl/sX8J8UypY5M+dDpD2KmyfP5IRs5v/fo0= 73 | gopkg.in/mattn/go-colorable.v0 v0.1.0 h1:WYuADWvfvYC07fm8ygYB3LMcsc5CunpxfMGKawHkAos= 74 | gopkg.in/mattn/go-colorable.v0 v0.1.0/go.mod h1:BVJlBXzARQxdi3nZo6f6bnl5yR20/tOL6p+V0KejgSY= 75 | gopkg.in/mattn/go-isatty.v0 v0.0.4 h1:NtS1rQGQr4IaFWBGz4Cz4BhB///gyys4gDVtKA7hIsc= 76 | gopkg.in/mattn/go-isatty.v0 v0.0.4/go.mod h1:wt691ab7g0X4ilKZNmMII3egK0bTxl37fEn/Fwbd8gc= 77 | gopkg.in/mattn/go-runewidth.v0 v0.0.4 h1:r0P71TnzQDlNIcizCqvPSSANoFa3WVGtcNJf3TWurcY= 78 | gopkg.in/mattn/go-runewidth.v0 v0.0.4/go.mod h1:BmXejnxvhwdaATwiJbB1vZ2dtXkQKZGu9yLFCZb4msQ= 79 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 80 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 81 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 82 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "syscall" 9 | 10 | "github.com/pteich/configstruct" 11 | 12 | "github.com/pteich/elastic-query-export/export" 13 | "github.com/pteich/elastic-query-export/flags" 14 | ) 15 | 16 | var Version string 17 | 18 | func main() { 19 | conf := flags.Flags{ 20 | ElasticURL: "http://localhost:9200", 21 | ElasticVerifySSL: false, 22 | Index: "logs-*", 23 | Query: "*", 24 | OutFormat: flags.FormatCSV, 25 | Outfile: "output.csv", 26 | ScrollSize: 1000, 27 | Timefield: "@timestamp", 28 | } 29 | 30 | ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) 31 | defer cancel() 32 | 33 | cmd := configstruct.NewCommand( 34 | "", 35 | "CLI tool to export data from ElasticSearch into a CSV or JSON file. https://github.com/pteich/elastic-query-export", 36 | &conf, 37 | func(c *configstruct.Command, cfg interface{}) error { 38 | export.Run(ctx, cfg.(*flags.Flags)) 39 | return nil 40 | }, 41 | ) 42 | 43 | err := cmd.ParseAndRun(os.Args) 44 | if err != nil { 45 | fmt.Println(err) 46 | os.Exit(1) 47 | } 48 | } 49 | --------------------------------------------------------------------------------