├── .github ├── draft-release-notes-config.yml └── workflows │ ├── draft-release-notes-workflow.yml │ ├── odfe-cli-publish-upload-artifacts.yml │ └── odfe-cli-test-build-workflow.yml ├── .gitignore ├── .goreleaser.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── client ├── http.go └── mocks │ └── mock_http.go ├── codecov.yml ├── commands ├── ad.go ├── ad_create.go ├── ad_delete.go ├── ad_get.go ├── ad_start_stop.go ├── ad_update.go ├── completion.go ├── curl.go ├── curl_delete.go ├── curl_get.go ├── curl_post.go ├── curl_put.go ├── file_operation_unix.go ├── file_operation_windows.go ├── knn.go ├── knn_test.go ├── profile.go ├── profile_test.go ├── root.go ├── root_test.go └── testdata │ └── config.yaml ├── controller ├── ad │ ├── ad.go │ ├── ad_test.go │ ├── mocks │ │ └── mock_ad.go │ └── testdata │ │ ├── create_failed_response.json │ │ ├── create_request.json │ │ ├── create_response.json │ │ ├── get_response.json │ │ └── search_response.json ├── config │ ├── config.go │ ├── config_test.go │ ├── mocks │ │ └── mock_config.go │ └── testdata │ │ └── config.yaml ├── es │ ├── es.go │ ├── es_test.go │ ├── mocks │ │ └── mock_es.go │ └── testdata │ │ └── search_result.json ├── knn │ ├── knn.go │ ├── knn_test.go │ └── mocks │ │ └── mock_knn.go └── profile │ ├── mocks │ └── mock_profile.go │ ├── profile.go │ └── profile_test.go ├── docker-compose.yml ├── docs ├── dev │ ├── images │ │ └── design.png │ ├── rfc │ │ ├── odfe-cli-rest-api-as-commands.md │ │ └── odfe-cli-rfc.md │ └── tutorial.md └── guide │ ├── release.md │ └── usage.md ├── entity ├── ad │ ├── ad.go │ └── ad_test.go ├── config.go ├── es │ ├── es.go │ └── request_error.go ├── knn │ └── knn.go └── profile.go ├── gateway ├── ad │ ├── ad.go │ ├── ad_test.go │ ├── mocks │ │ └── mock_ad.go │ └── testdata │ │ ├── create_result.json │ │ ├── get_result.json │ │ └── search_result.json ├── aws │ └── signer │ │ ├── aws.go │ │ └── aws_test.go ├── es │ ├── es.go │ ├── es_test.go │ ├── mocks │ │ └── mock_es.go │ └── testdata │ │ └── search_result.json ├── gateway.go ├── gateway_test.go └── knn │ ├── knn.go │ ├── knn_test.go │ └── mocks │ └── mock_knn.go ├── go.mod ├── go.sum ├── handler ├── ad │ ├── ad.go │ ├── ad_test.go │ └── testdata │ │ ├── create.json │ │ ├── invalid.txt │ │ └── update.json ├── es │ ├── es.go │ └── es_test.go └── knn │ ├── knn.go │ └── knn_test.go ├── it ├── ad_test.go ├── es │ ├── es_get_test.go │ └── testdata │ │ ├── bulk-user-request │ │ ├── sample-index │ │ └── sample-index-compressed.gz ├── helper.go ├── knn_test.go └── testdata │ ├── ecommerce │ ├── knn-sample-index │ └── knn-sample-index-mapping ├── main.go ├── mapper ├── ad │ ├── ad.go │ ├── ad_test.go │ └── testdata │ │ └── search_response.json ├── es │ ├── es.go │ ├── es_test.go │ └── testdata │ │ └── index.json └── mapper.go └── release-notes └── opendistro-odfe-cli-release-notes-1.1.0.md /.github/draft-release-notes-config.yml: -------------------------------------------------------------------------------- 1 | # The overall template of the release notes 2 | template: | 3 | $CHANGES 4 | 5 | # Setting the formatting and sorting for the release notes body 6 | name-template: Version (set version here) 7 | change-template: '* $TITLE (#$NUMBER)' 8 | sort-by: merged_at 9 | sort-direction: ascending 10 | replacers: 11 | - search: '##' 12 | replace: '###' 13 | 14 | # Organizing the tagged PRs into categories 15 | categories: 16 | - title: 'Breaking Changes' 17 | labels: 18 | - 'Breaking Changes' 19 | - title: 'Features' 20 | labels: 21 | - 'Features' 22 | - title: 'Enhancements' 23 | labels: 24 | - 'Enhancements' 25 | - title: 'Bug Fixes' 26 | labels: 27 | - 'Bug Fixes' 28 | - title: 'Infrastructure' 29 | labels: 30 | - 'Infrastructure' 31 | - title: 'Documentation' 32 | labels: 33 | - 'Documentation' 34 | - title: 'Maintenance' 35 | labels: 36 | - 'Maintenance' 37 | - title: 'Refactoring' 38 | labels: 39 | - 'Refactoring' -------------------------------------------------------------------------------- /.github/workflows/draft-release-notes-workflow.yml: -------------------------------------------------------------------------------- 1 | name: Release Drafter 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | update_release_draft: 10 | name: Update draft release notes 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Update draft release notes 14 | uses: release-drafter/release-drafter@v5 15 | with: 16 | config-name: draft-release-notes-config.yml 17 | name: Version (set here) 18 | tag: (None) 19 | env: 20 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 21 | -------------------------------------------------------------------------------- /.github/workflows/odfe-cli-publish-upload-artifacts.yml: -------------------------------------------------------------------------------- 1 | name: Publish and Upload odfe-cli 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Get Version 12 | run: | 13 | TAG_NAME=${GITHUB_REF#refs/*/} 14 | echo "RELEASE_VERSION=${TAG_NAME:1}" >> $GITHUB_ENV 15 | 16 | - name: Set up Go ubuntu-latest 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: 1.16.2 20 | 21 | - name: Check out source code 22 | uses: actions/checkout@v2 23 | 24 | - name: Run GoReleaser 25 | uses: goreleaser/goreleaser-action@v2 26 | with: 27 | version: latest 28 | args: --snapshot --skip-publish 29 | env: 30 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 31 | 32 | - name: Upload macOS(amd64) Artifact 33 | uses: actions/upload-artifact@v2 34 | with: 35 | name: opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-macos-x64 36 | path: dist/odfe-cli_darwin_amd64/odfe-cli 37 | 38 | - name: Upload Linux(amd64) Artifact 39 | uses: actions/upload-artifact@v2 40 | with: 41 | name: opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-linux-x64 42 | path: dist/odfe-cli_linux_amd64/odfe-cli 43 | 44 | - name: Upload Linux(arm64) Artifact 45 | uses: actions/upload-artifact@v2 46 | with: 47 | name: opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-linux-arm64 48 | path: dist/odfe-cli_linux_arm64/odfe-cli 49 | 50 | - name: Upload Windows(i386) Artifact 51 | uses: actions/upload-artifact@v2 52 | with: 53 | name: opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-windows-x86 54 | path: dist/odfe-cli_windows_386/odfe-cli.exe 55 | 56 | - name: Upload Windows(amd64) Artifact 57 | uses: actions/upload-artifact@v2 58 | with: 59 | name: opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-windows-x64 60 | path: dist/odfe-cli_windows_amd64/odfe-cli.exe 61 | 62 | 63 | - name: Configure AWS credentials 64 | uses: aws-actions/configure-aws-credentials@v1 65 | with: 66 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 67 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 68 | aws-region: ${{ secrets.AWS_REGION }} 69 | 70 | - name: Zip and copy macOS(amd64) artifacts to S3 bucket with the AWS CLI 71 | run: | 72 | cd dist/odfe-cli_darwin_amd64/ 73 | zip opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-macos-x64.zip odfe-cli 74 | aws s3 cp opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-macos-x64.zip ${{ secrets.S3_MACOS_URI }} 75 | 76 | - name: Zip and copy macOS(arm64) artifacts to S3 bucket with the AWS CLI 77 | run: | 78 | cd dist/odfe-cli_darwin_arm64/ 79 | zip opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-macos-arm64.zip odfe-cli 80 | aws s3 cp opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-macos-arm64.zip ${{ secrets.S3_MACOS_URI }} 81 | 82 | - name: Zip and copy linux(amd64) artifacts to S3 bucket with the AWS CLI 83 | run: | 84 | cd dist/odfe-cli_linux_amd64 85 | zip opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-linux-x64.zip odfe-cli 86 | aws s3 cp opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-linux-x64.zip ${{ secrets.S3_LINUX_URI }} 87 | 88 | - name: Zip and copy linux(arm64) artifacts to S3 bucket with the AWS CLI 89 | run: | 90 | cd dist/odfe-cli_linux_arm64/ 91 | zip opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-linux-arm64.zip odfe-cli 92 | aws s3 cp opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-linux-arm64.zip ${{ secrets.S3_LINUX_URI }} 93 | 94 | - name: Zip and copy windows(x386) artifacts to S3 bucket with the AWS CLI 95 | run: | 96 | cd dist/odfe-cli_windows_386 97 | zip opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-windows-x86.zip odfe-cli.exe 98 | aws s3 cp opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-windows-x86.zip ${{ secrets.S3_WINDOWS_URI }} 99 | 100 | - name: Zip and copy windows(amd64) artifacts to S3 bucket with the AWS CLI 101 | run: | 102 | cd dist/odfe-cli_windows_amd64/ 103 | zip opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-windows-x64.zip odfe-cli.exe 104 | aws s3 cp opendistro-odfe-cli-${{ env.RELEASE_VERSION }}-windows-x64.zip ${{ secrets.S3_WINDOWS_URI }} 105 | -------------------------------------------------------------------------------- /.github/workflows/odfe-cli-test-build-workflow.yml: -------------------------------------------------------------------------------- 1 | name: Build and Test odfe-cli 2 | on: 3 | push: 4 | branches: 5 | - main 6 | - opendistro-* 7 | pull_request: 8 | branches: 9 | - main 10 | - opendistro-* 11 | 12 | jobs: 13 | test: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Set up Go ubuntu-latest 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: 1.16.2 20 | 21 | - name: Install goimports 22 | run: go get golang.org/x/tools/cmd/goimports 23 | 24 | - name: Check out source code 25 | uses: actions/checkout@v2 26 | 27 | - name: Format check 28 | run: goimports -w . 29 | 30 | - name: Check for modified files 31 | id: git-check 32 | run: | 33 | echo ::set-output name=modified::$(if git diff-index --quiet HEAD --; then echo "false"; else echo "true"; fi) 34 | 35 | - name: Display unformated changes and fail if any 36 | if: steps.git-check.outputs.modified == 'true' 37 | run: | 38 | echo "Found some files are dirty. Please add changes to index and ammend commit". 39 | git diff 40 | exit 1 41 | 42 | - name: Lint check 43 | uses: golangci/golangci-lint-action@v2 44 | with: 45 | version: v1.29 46 | 47 | - name: Run Unit Tests 48 | env: 49 | GOPROXY: "https://proxy.golang.org" 50 | run: | 51 | go test ./... -coverprofile=coverage.out 52 | go tool cover -func=coverage.out 53 | 54 | - name: Run Docker Image 55 | run: | 56 | make docker.start.components 57 | sleep 60 58 | 59 | - name: Run Integration Tests 60 | env: 61 | GOPROXY: "https://proxy.golang.org" 62 | run: make test.integration 63 | 64 | - name: Upload coverage to Codecov 65 | uses: codecov/codecov-action@v1.0.3 66 | with: 67 | token: ${{secrets.CODECOV_TOKEN}} 68 | file: coverage.out 69 | flags: odfe-cli 70 | name: codecov-umbrella 71 | 72 | - name: Stop and Clean Docker Components 73 | run: | 74 | make docker.stop 75 | make docker.clean 76 | 77 | build: 78 | strategy: 79 | matrix: 80 | platform: [ ubuntu-latest, macos-latest, windows-latest ] 81 | go-version: [ 1.16.2 ] 82 | runs-on: ${{ matrix.platform }} 83 | steps: 84 | - name: Set up Go ${{ matrix.platform }} 85 | uses: actions/setup-go@v2 86 | with: 87 | go-version: ${{ matrix.go-version }} 88 | 89 | - name: Check out source code 90 | uses: actions/checkout@v2 91 | 92 | - name: Build for ${{ matrix.platform }}-${{ matrix.go-version }} 93 | env: 94 | GOPROXY: "https://proxy.golang.org" 95 | run: go build . -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # IDE Settings 9 | .idea/* 10 | 11 | #OS Settings 12 | .DS_Store 13 | 14 | # Test binary, built with `go test -c` 15 | *.test 16 | 17 | bin 18 | .go 19 | .push-* 20 | .container-* 21 | .dockerfile-* 22 | 23 | /coverage.out 24 | 25 | odfe-cli 26 | odfe-cli.exe 27 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # This is an example goreleaser.yaml file with some sane defaults. 2 | # Make sure to check the documentation at http://goreleaser.com 3 | project_name: odfe-cli 4 | dist: ./dist 5 | before: 6 | hooks: 7 | # You may remove this if you don't use go modules. 8 | - go mod download 9 | builds: 10 | - env: 11 | - CGO_ENABLED=0 12 | goos: 13 | - linux 14 | - windows 15 | - darwin 16 | goarch: 17 | - 386 18 | - amd64 19 | - arm 20 | - arm64 21 | ignore: 22 | - goos: darwin 23 | goarch: 386 24 | archives: 25 | - 26 | name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' 27 | replacements: 28 | darwin: Darwin 29 | linux: Linux 30 | windows: Windows 31 | 386: i386 32 | amd64: x86_64 33 | format_overrides: 34 | - goos: windows 35 | format: zip 36 | checksum: 37 | name_template: '{{ .ProjectName }}_checksums.txt' 38 | snapshot: 39 | name_template: '{{ .ProjectName }}_{{ .Version }}' 40 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | 61 | We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. 62 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # we will put our integration testing in this path 2 | INTEGRATION_TEST_PATH=./it/... 3 | 4 | # set of env variables that you need for testing 5 | ENV_LOCAL_TEST=\ 6 | ODFE_ENDPOINT="https://localhost:9200" \ 7 | ODFE_USER="admin" \ 8 | ODFE_PASSWORD="admin" 9 | 10 | # this command will start a docker components that we set in docker-compose.yml 11 | docker.start.components: 12 | docker-compose up -d; 13 | 14 | # shutting down docker components 15 | docker.stop: 16 | docker-compose down; 17 | 18 | # clean up docker 19 | docker.clean: 20 | docker system prune --volumes --force; 21 | 22 | # this command will trigger integration test 23 | # INTEGRATION_TEST_SUITE_PATH is used for run specific test in Golang 24 | test.integration: 25 | go clean -testcache; 26 | $(ENV_LOCAL_TEST) go test -tags=integration $(INTEGRATION_TEST_PATH); 27 | 28 | test.unit: 29 | go clean -testcache; 30 | go test ./...; 31 | 32 | # format project using goimports tool 33 | format: 34 | goimports -w .; 35 | 36 | # generate odfe-cli file in current directory 37 | # update GOOS / GOARCH if you want to build for other os and architecture 38 | build: 39 | go build . 40 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Open Distro for Elasticsearch Command Line Interface 2 | Copyright 2019-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. -------------------------------------------------------------------------------- /client/http.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package client 17 | 18 | import ( 19 | "crypto/tls" 20 | "net/http" 21 | "time" 22 | 23 | "github.com/hashicorp/go-retryablehttp" 24 | ) 25 | 26 | const defaultTimeout = 10 27 | 28 | //Client is an Abstraction for actual client 29 | type Client struct { 30 | HTTPClient *retryablehttp.Client 31 | } 32 | 33 | //NewDefaultClient return new instance of client 34 | func NewDefaultClient(tripper http.RoundTripper) (*Client, error) { 35 | 36 | client := retryablehttp.NewClient() 37 | client.HTTPClient.Transport = tripper 38 | client.HTTPClient.Timeout = defaultTimeout * time.Second 39 | client.Logger = nil 40 | return &Client{ 41 | HTTPClient: client, 42 | }, nil 43 | } 44 | 45 | //New takes transport and uses accordingly 46 | func New(tripper http.RoundTripper) (*Client, error) { 47 | if tripper == nil { 48 | tripper = &http.Transport{ 49 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 50 | } 51 | } 52 | return NewDefaultClient(tripper) 53 | } 54 | -------------------------------------------------------------------------------- /client/mocks/mock_http.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package mocks 17 | 18 | import ( 19 | "fmt" 20 | "net/http" 21 | "odfe-cli/client" 22 | "os" 23 | ) 24 | 25 | // RoundTripFunc . 26 | type RoundTripFunc func(req *http.Request) *http.Response 27 | 28 | // RoundTrip . 29 | func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { 30 | return f(req), nil 31 | } 32 | 33 | //NewTestClient returns *http.Client with Transport replaced to avoid making real calls 34 | func NewTestClient(fn RoundTripFunc) *client.Client { 35 | c, err := client.New(fn) 36 | if err != nil { 37 | fmt.Println("Fatal: failed to get client") 38 | os.Exit(1) 39 | } 40 | return c 41 | } 42 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | threshold: 5% 6 | target: auto 7 | odfe-cli: 8 | paths: 9 | - "odfe-cli/" 10 | flags: 11 | - odfe-cli 12 | patch: off 13 | comment: 14 | layout: "reach, diff, flags, files" 15 | behavior: default 16 | require_changes: false 17 | require_base: no 18 | require_head: no 19 | branches: null 20 | flags: 21 | odfe-cli: 22 | carryforward: true -------------------------------------------------------------------------------- /commands/ad.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | "odfe-cli/client" 20 | adctrl "odfe-cli/controller/ad" 21 | esctrl "odfe-cli/controller/es" 22 | adgateway "odfe-cli/gateway/ad" 23 | esgateway "odfe-cli/gateway/es" 24 | handler "odfe-cli/handler/ad" 25 | "os" 26 | 27 | "github.com/spf13/cobra" 28 | ) 29 | 30 | const ( 31 | adCommandName = "ad" 32 | ) 33 | 34 | //adCommand is base command for Anomaly Detection plugin. 35 | var adCommand = &cobra.Command{ 36 | Use: adCommandName, 37 | Short: "Manage the Anomaly Detection plugin", 38 | Long: "Use the Anomaly Detection commands to create, configure, and manage detectors.", 39 | } 40 | 41 | func init() { 42 | adCommand.Flags().BoolP("help", "h", false, "Help for Anomaly Detection") 43 | GetRoot().AddCommand(adCommand) 44 | } 45 | 46 | //GetADCommand returns AD base command, since this will be needed for subcommands 47 | //to add as parent later 48 | func GetADCommand() *cobra.Command { 49 | return adCommand 50 | } 51 | 52 | //GetADHandler returns handler by wiring the dependency manually 53 | func GetADHandler() (*handler.Handler, error) { 54 | c, err := client.New(nil) 55 | if err != nil { 56 | return nil, err 57 | } 58 | profile, err := GetProfile() 59 | if err != nil { 60 | return nil, err 61 | } 62 | g := adgateway.New(c, profile) 63 | esg := esgateway.New(c, profile) 64 | esc := esctrl.New(esg) 65 | ctr := adctrl.New(os.Stdin, esc, g) 66 | return handler.New(ctr), nil 67 | } 68 | -------------------------------------------------------------------------------- /commands/ad_create.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | "fmt" 20 | handler "odfe-cli/handler/ad" 21 | 22 | "github.com/spf13/cobra" 23 | ) 24 | 25 | const ( 26 | createDetectorsCommandName = "create" 27 | generate = "generate-template" 28 | ) 29 | 30 | //createCmd creates detectors with configuration from input file, if interactive mode is on, 31 | //this command will prompt for confirmation on number of detectors will be created on executions. 32 | var createCmd = &cobra.Command{ 33 | Use: createDetectorsCommandName + " json-file-path ...", 34 | Short: "Create detectors based on JSON files", 35 | Long: "Create detectors based on a local JSON file\n" + 36 | "To begin, use `odfe-cli ad create --generate-template` to generate a sample configuration. Save this template locally and update it for your use case. Then use `odfe-cli ad create file-path` to create detector.", 37 | Run: func(cmd *cobra.Command, args []string) { 38 | generate, _ := cmd.Flags().GetBool(generate) 39 | if generate { 40 | generateTemplate() 41 | return 42 | } 43 | //If no args, display usage 44 | if len(args) < 1 { 45 | fmt.Println(cmd.Usage()) 46 | return 47 | } 48 | err := createDetectors(args) 49 | DisplayError(err, createDetectorsCommandName) 50 | }, 51 | } 52 | 53 | //generateTemplate prints sample detector configuration 54 | func generateTemplate() { 55 | detector, _ := handler.GenerateAnomalyDetector() 56 | fmt.Println(string(detector)) 57 | } 58 | 59 | func init() { 60 | GetADCommand().AddCommand(createCmd) 61 | createCmd.Flags().BoolP(generate, "g", false, "Output sample detector configuration") 62 | createCmd.Flags().BoolP("help", "h", false, "Help for "+createDetectorsCommandName) 63 | 64 | } 65 | 66 | //createDetectors create detectors based on configurations from fileNames 67 | func createDetectors(fileNames []string) error { 68 | 69 | commandHandler, err := GetADHandler() 70 | if err != nil { 71 | return err 72 | } 73 | for _, name := range fileNames { 74 | err = handler.CreateAnomalyDetector(commandHandler, name) 75 | if err != nil { 76 | return err 77 | } 78 | } 79 | return nil 80 | } 81 | -------------------------------------------------------------------------------- /commands/ad_delete.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | handler "odfe-cli/handler/ad" 20 | 21 | "github.com/spf13/cobra" 22 | ) 23 | 24 | const ( 25 | deleteDetectorsCommandName = "delete" 26 | deleteDetectorIDFlagName = "id" 27 | detectorForceDeletionFlagName = "force" 28 | ) 29 | 30 | //deleteDetectorsCmd deletes detectors based on id, name or name regex pattern. 31 | //default input is name pattern, one can change this format to be id by passing --id flag 32 | var deleteDetectorsCmd = &cobra.Command{ 33 | Use: deleteDetectorsCommandName + " detector_name ..." + " [flags] ", 34 | Short: "Delete detectors based on a list of IDs, names, or name regex patterns", 35 | Long: "Delete detectors based on list of IDs, names, or name regex patterns.\n" + 36 | "Wrap regex patterns in quotation marks to prevent the terminal from matching patterns against the files in the current directory.\nThe default input is detector name. Use the `--id` flag if input is detector ID instead of name", 37 | 38 | Args: cobra.MinimumNArgs(1), 39 | Run: func(cmd *cobra.Command, args []string) { 40 | force, _ := cmd.Flags().GetBool(detectorForceDeletionFlagName) 41 | detectorID, _ := cmd.Flags().GetBool(deleteDetectorIDFlagName) 42 | action := handler.DeleteAnomalyDetectorByNamePattern 43 | if detectorID { 44 | action = handler.DeleteAnomalyDetectorByID 45 | } 46 | err := deleteDetectors(args, force, action) 47 | DisplayError(err, deleteDetectorsCommandName) 48 | }, 49 | } 50 | 51 | func init() { 52 | GetADCommand().AddCommand(deleteDetectorsCmd) 53 | deleteDetectorsCmd.Flags().BoolP(detectorForceDeletionFlagName, "f", false, "Delete the detector even if it is running") 54 | deleteDetectorsCmd.Flags().BoolP(deleteDetectorIDFlagName, "", false, "Input is detector ID") 55 | deleteDetectorsCmd.Flags().BoolP("help", "h", false, "Help for "+deleteDetectorsCommandName) 56 | } 57 | 58 | //deleteDetectors deletes detectors with force by calling delete method provided 59 | func deleteDetectors(detectors []string, force bool, f func(*handler.Handler, string, bool) error) error { 60 | commandHandler, err := GetADHandler() 61 | if err != nil { 62 | return err 63 | } 64 | for _, detector := range detectors { 65 | err = f(commandHandler, detector, force) 66 | if err != nil { 67 | return err 68 | } 69 | } 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /commands/ad_get.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | "encoding/json" 20 | "fmt" 21 | "io" 22 | entity "odfe-cli/entity/ad" 23 | "odfe-cli/handler/ad" 24 | "os" 25 | 26 | "github.com/spf13/cobra" 27 | ) 28 | 29 | const ( 30 | getDetectorsCommandName = "get" 31 | getDetectorIDFlagName = "id" 32 | ) 33 | 34 | //getDetectorsCmd prints detectors configuration based on id, name or name regex pattern. 35 | //default input is name pattern, one can change this format to be id by passing --id flag 36 | var getDetectorsCmd = &cobra.Command{ 37 | Use: getDetectorsCommandName + " detector_name ..." + " [flags] ", 38 | Short: "Get detectors based on a list of IDs, names, or name regex patterns", 39 | Long: "Get detectors based on a list of IDs, names, or name regex patterns.\n" + 40 | "Wrap regex patterns in quotation marks to prevent the terminal from matching patterns against the files in the current directory.\nThe default input is detector name. Use the `--id` flag if input is detector ID instead of name", 41 | Args: cobra.MinimumNArgs(1), 42 | Run: func(cmd *cobra.Command, args []string) { 43 | err := printDetectors(Println, cmd, args) 44 | if err != nil { 45 | DisplayError(err, getDetectorsCommandName) 46 | } 47 | }, 48 | } 49 | 50 | type Display func(*cobra.Command, *entity.DetectorOutput) error 51 | 52 | //printDetectors print detectors 53 | func printDetectors(display Display, cmd *cobra.Command, detectors []string) error { 54 | idStatus, _ := cmd.Flags().GetBool(getDetectorIDFlagName) 55 | commandHandler, err := GetADHandler() 56 | if err != nil { 57 | return err 58 | } 59 | // default is name 60 | action := ad.GetAnomalyDetectorsByNamePattern 61 | if idStatus { 62 | action = getDetectorsByID 63 | } 64 | results, err := getDetectors(commandHandler, detectors, action) 65 | if err != nil { 66 | return err 67 | } 68 | return fprint(cmd, display, results) 69 | } 70 | 71 | //getDetectors fetch detector from controller 72 | func getDetectors( 73 | commandHandler *ad.Handler, args []string, get func(*ad.Handler, string) ( 74 | []*entity.DetectorOutput, error)) ([]*entity.DetectorOutput, error) { 75 | var results []*entity.DetectorOutput 76 | for _, detector := range args { 77 | output, err := get(commandHandler, detector) 78 | if err != nil { 79 | return nil, err 80 | } 81 | results = append(results, output...) 82 | } 83 | return results, nil 84 | } 85 | 86 | //getDetectorsByID gets detector output based on ID as argument 87 | func getDetectorsByID(commandHandler *ad.Handler, ID string) ([]*entity.DetectorOutput, error) { 88 | 89 | output, err := ad.GetAnomalyDetectorByID(commandHandler, ID) 90 | if err != nil { 91 | return nil, err 92 | } 93 | return []*entity.DetectorOutput{output}, nil 94 | } 95 | 96 | //fprint displays the list of detectors. 97 | func fprint(cmd *cobra.Command, display Display, results []*entity.DetectorOutput) error { 98 | if results == nil { 99 | return nil 100 | } 101 | for _, d := range results { 102 | if err := display(cmd, d); err != nil { 103 | return err 104 | } 105 | } 106 | return nil 107 | } 108 | 109 | //FPrint prints detector configuration on writer 110 | //Since this is json format, use indent function to pretty print before printing on writer 111 | func FPrint(writer io.Writer, d *entity.DetectorOutput) error { 112 | formattedOutput, err := json.MarshalIndent(d, "", " ") 113 | if err != nil { 114 | return err 115 | } 116 | _, err = fmt.Fprintln(writer, string(formattedOutput)) 117 | return err 118 | } 119 | 120 | //Println prints detector configuration on stdout 121 | func Println(cmd *cobra.Command, d *entity.DetectorOutput) error { 122 | return FPrint(os.Stdout, d) 123 | } 124 | 125 | func init() { 126 | GetADCommand().AddCommand(getDetectorsCmd) 127 | getDetectorsCmd.Flags().BoolP(getDetectorIDFlagName, "", false, "Input is detector ID") 128 | getDetectorsCmd.Flags().BoolP("help", "h", false, "Help for "+getDetectorsCommandName) 129 | } 130 | -------------------------------------------------------------------------------- /commands/ad_start_stop.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | "fmt" 20 | "odfe-cli/handler/ad" 21 | 22 | "github.com/spf13/cobra" 23 | ) 24 | 25 | const ( 26 | startDetectorsCommandName = "start" 27 | stopDetectorsCommandName = "stop" 28 | idFlagName = "id" 29 | ) 30 | 31 | //startDetectorsCmd start detectors based on id, name or name regex pattern. 32 | //default input is name pattern, one can change this format to be id by passing --id flag 33 | var startDetectorsCmd = &cobra.Command{ 34 | Use: startDetectorsCommandName + " detector_name ..." + " [flags] ", 35 | Short: "Start detectors based on a list of IDs, names, or name regex patterns", 36 | Long: "Start detectors based on a list of IDs, names, or name regex patterns.\n" + 37 | "Wrap regex patterns in quotation marks to prevent the terminal from matching patterns against the files in the current directory.\n" + 38 | "The default input is detector name. Use the `--id` flag if input is detector ID instead of name", 39 | Args: cobra.MinimumNArgs(1), 40 | Run: func(cmd *cobra.Command, args []string) { 41 | idStatus, _ := cmd.Flags().GetBool(idFlagName) 42 | action := ad.StartAnomalyDetectorByNamePattern 43 | if idStatus { 44 | action = ad.StartAnomalyDetectorByID 45 | } 46 | err := execute(action, args) 47 | DisplayError(err, startDetectorsCommandName) 48 | }, 49 | } 50 | 51 | //stopDetectorsCmd stops detectors based on id and name pattern. 52 | //default input is name pattern, one can change this format to be id by passing --id flag 53 | var stopDetectorsCmd = &cobra.Command{ 54 | Use: stopDetectorsCommandName + " detector_name ..." + " [flags] ", 55 | Short: "Stop detectors based on a list of IDs, names, or name regex patterns", 56 | Long: "Stop detectors based on a list of IDs, names, or name regex patterns.\n" + 57 | "Wrap regex patterns in quotation marks to prevent the terminal from matching patterns against the files in the current directory.\n" + 58 | "The default input is detector name. Use the `--id` flag if input is detector ID instead of name", 59 | Run: func(cmd *cobra.Command, args []string) { 60 | //If no args, display usage 61 | if len(args) < 1 { 62 | fmt.Println(cmd.Usage()) 63 | return 64 | } 65 | idStatus, _ := cmd.Flags().GetBool(idFlagName) 66 | action := ad.StopAnomalyDetectorByNamePattern 67 | if idStatus { 68 | action = ad.StopAnomalyDetectorByID 69 | } 70 | err := execute(action, args) 71 | DisplayError(err, stopDetectorsCommandName) 72 | }, 73 | } 74 | 75 | func init() { 76 | startDetectorsCmd.Flags().BoolP(idFlagName, "", false, "Input is detector ID") 77 | startDetectorsCmd.Flags().BoolP("help", "h", false, "Help for "+startDetectorsCommandName) 78 | GetADCommand().AddCommand(startDetectorsCmd) 79 | stopDetectorsCmd.Flags().BoolP(idFlagName, "", false, "Input is detector ID") 80 | stopDetectorsCmd.Flags().BoolP("help", "h", false, "Help for "+stopDetectorsCommandName) 81 | GetADCommand().AddCommand(stopDetectorsCmd) 82 | } 83 | 84 | func execute(f func(*ad.Handler, string) error, detectors []string) error { 85 | // iterate over the arguments 86 | // the first return value is index of fileNames, we can omit it using _ 87 | commandHandler, err := GetADHandler() 88 | if err != nil { 89 | return err 90 | } 91 | for _, detector := range detectors { 92 | err := f(commandHandler, detector) 93 | if err != nil { 94 | return err 95 | } 96 | } 97 | return nil 98 | } 99 | -------------------------------------------------------------------------------- /commands/ad_update.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | handler "odfe-cli/handler/ad" 20 | 21 | "github.com/spf13/cobra" 22 | ) 23 | 24 | const ( 25 | updateDetectorsCommandName = "update" 26 | forceFlagName = "force" 27 | startFlagName = "start" 28 | ) 29 | 30 | //updateDetectorsCmd updates detectors with configuration from input file 31 | var updateDetectorsCmd = &cobra.Command{ 32 | Use: updateDetectorsCommandName + " json-file-path ... [flags]", 33 | Short: "Update detectors based on JSON files", 34 | Long: "Update detectors based on JSON files.\n" + 35 | "To begin, use `odfe-cli ad get detector-name > detector_to_be_updated.json` to download the detector. " + 36 | "Modify the file, and then use `odfe-cli ad update file-path` to update the detector.", 37 | Args: cobra.MinimumNArgs(1), 38 | Run: func(cmd *cobra.Command, args []string) { 39 | force, _ := cmd.Flags().GetBool(forceFlagName) 40 | start, _ := cmd.Flags().GetBool(startFlagName) 41 | err := updateDetectors(args, force, start) 42 | if err != nil { 43 | DisplayError(err, updateDetectorsCommandName) 44 | } 45 | }, 46 | } 47 | 48 | func init() { 49 | GetADCommand().AddCommand(updateDetectorsCmd) 50 | updateDetectorsCmd.Flags().BoolP(forceFlagName, "f", false, "Stop detector and update forcefully") 51 | updateDetectorsCmd.Flags().BoolP(startFlagName, "s", false, "Start detector if update is successful") 52 | updateDetectorsCmd.Flags().BoolP("help", "h", false, "Help for "+updateDetectorsCommandName) 53 | } 54 | 55 | func updateDetectors(fileNames []string, force bool, start bool) error { 56 | commandHandler, err := GetADHandler() 57 | if err != nil { 58 | return err 59 | } 60 | for _, name := range fileNames { 61 | err = handler.UpdateAnomalyDetector(commandHandler, name, force, start) 62 | if err != nil { 63 | return err 64 | } 65 | } 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /commands/completion.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package commands 16 | 17 | import ( 18 | "fmt" 19 | "os" 20 | 21 | "github.com/spf13/cobra" 22 | ) 23 | 24 | const ( 25 | CompletionCommandName = "completion" 26 | BashShell = "bash" 27 | ZshShell = "zsh" 28 | FishShell = "fish" 29 | PowerShell = "powershell" 30 | ) 31 | 32 | var longText = `To enable shell autocompletion: 33 | 34 | Bash: 35 | 36 | $ source <(odfe-cli completion bash) 37 | 38 | # To enable auto completion for commands for each session, execute once: 39 | Linux: 40 | $ odfe-cli completion bash > /etc/bash_completion.d/odfe-cli 41 | MacOS: 42 | $ odfe-cli completion bash > /usr/local/etc/bash_completion.d/odfe-cli 43 | 44 | Zsh: 45 | 46 | # If shell completion is not already enabled in your environment you will need 47 | # to enable it. You can execute the following once: 48 | 49 | $ echo "autoload -U compinit; compinit" >> ~/.zshrc 50 | 51 | # To enable auto completion for commands for each session, execute once: 52 | $ odfe-cli completion zsh > "${fpath[1]}/_odfe-cli" 53 | 54 | # You will need to start a new shell for this setup to take effect. 55 | 56 | Fish: 57 | 58 | $ odfe-cli completion fish | source 59 | 60 | # To enable auto completion for commands for each session, execute once: 61 | $ odfe-cli completion fish > ~/.config/fish/completions/odfe-cli.fish 62 | 63 | Powershell: 64 | 65 | PS> odfe-cli completion powershell | Out-String | Invoke-Expression 66 | 67 | # To enable auto completion for commands for each session, execute once: 68 | PS> odfe-cli completion powershell > odfe-cli.ps1 69 | # and source this file from your powershell profile. 70 | ` 71 | 72 | var completionCmd = &cobra.Command{ 73 | Use: fmt.Sprintf("%s [ %s | %s | %s | %s ]", CompletionCommandName, BashShell, ZshShell, FishShell, PowerShell), 74 | Short: "Generate completion script for your shell", 75 | Long: longText, 76 | DisableFlagsInUseLine: true, 77 | ValidArgs: []string{BashShell, ZshShell, FishShell, PowerShell}, 78 | Args: cobra.ExactValidArgs(1), 79 | Run: func(cmd *cobra.Command, args []string) { 80 | var err error 81 | switch args[0] { 82 | case BashShell: 83 | err = cmd.Root().GenBashCompletion(os.Stdout) 84 | case ZshShell: 85 | err = cmd.Root().GenZshCompletion(os.Stdout) 86 | case FishShell: 87 | err = cmd.Root().GenFishCompletion(os.Stdout, true) 88 | case PowerShell: 89 | err = cmd.Root().GenPowerShellCompletion(os.Stdout) 90 | } 91 | DisplayError(err, CompletionCommandName) 92 | }, 93 | } 94 | 95 | func init() { 96 | GetRoot().AddCommand(completionCmd) 97 | } 98 | -------------------------------------------------------------------------------- /commands/curl.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | "fmt" 20 | "odfe-cli/client" 21 | ctrl "odfe-cli/controller/es" 22 | entity "odfe-cli/entity/es" 23 | gateway "odfe-cli/gateway/es" 24 | handler "odfe-cli/handler/es" 25 | 26 | "github.com/spf13/cobra" 27 | ) 28 | 29 | const ( 30 | curlCommandName = "curl" 31 | curlPrettyFlagName = "pretty" 32 | curlPathFlagName = "path" 33 | curlQueryParamsFlagName = "query-params" 34 | curlDataFlagName = "data" 35 | curlHeadersFlagName = "headers" 36 | curlOutputFormatFlagName = "output-format" 37 | curlOutputFilterPathFlagName = "filter-path" 38 | ) 39 | 40 | //curlCommand is base command for Elasticsearch REST APIs. 41 | var curlCommand = &cobra.Command{ 42 | Use: curlCommandName, 43 | Short: "Manage Elasticsearch core features", 44 | Long: "Use the curl command to execute any REST API calls against the cluster.", 45 | } 46 | 47 | func init() { 48 | curlCommand.Flags().BoolP("help", "h", false, "Help for curl command") 49 | curlCommand.PersistentFlags().Bool(curlPrettyFlagName, false, "Response will be formatted") 50 | curlCommand.PersistentFlags().StringP(curlOutputFormatFlagName, "o", "", 51 | "Output format if supported by cluster, else, default format by Elasticsearch. Example json, yaml") 52 | curlCommand.PersistentFlags().StringP(curlOutputFilterPathFlagName, "f", "", 53 | "Filter output fields returned by Elasticsearch. Use comma ',' to separate list of filters") 54 | GetRoot().AddCommand(curlCommand) 55 | } 56 | 57 | //GetCurlCommand returns Curl base command, since this will be needed for subcommands 58 | //to add as parent later 59 | func GetCurlCommand() *cobra.Command { 60 | return curlCommand 61 | } 62 | 63 | //getCurlHandler returns handler by wiring the dependency manually 64 | func getCurlHandler() (*handler.Handler, error) { 65 | c, err := client.New(nil) 66 | if err != nil { 67 | return nil, err 68 | } 69 | profile, err := GetProfile() 70 | if err != nil { 71 | return nil, err 72 | } 73 | g := gateway.New(c, profile) 74 | facade := ctrl.New(g) 75 | return handler.New(facade), nil 76 | } 77 | 78 | //CurlActionExecute executes API based on user request 79 | func CurlActionExecute(input entity.CurlCommandRequest) error { 80 | 81 | commandHandler, err := getCurlHandler() 82 | if err != nil { 83 | return err 84 | } 85 | response, err := handler.Curl(commandHandler, input) 86 | if err == nil { 87 | fmt.Println(string(response)) 88 | return nil 89 | } 90 | if requestError, ok := err.(*entity.RequestError); ok { 91 | fmt.Println(requestError.GetResponse()) 92 | return nil 93 | } 94 | return err 95 | } 96 | 97 | func FormatOutput() bool { 98 | isPretty, _ := curlCommand.PersistentFlags().GetBool(curlPrettyFlagName) 99 | return isPretty 100 | } 101 | 102 | func GetUserInputAsStringForFlag(flagName string) string { 103 | format, _ := curlCommand.PersistentFlags().GetString(flagName) 104 | return format 105 | } 106 | 107 | func Run(cmd cobra.Command, cmdName string) { 108 | input := entity.CurlCommandRequest{ 109 | Action: cmdName, 110 | Pretty: FormatOutput(), 111 | OutputFormat: GetUserInputAsStringForFlag(curlOutputFormatFlagName), 112 | OutputFilterPath: GetUserInputAsStringForFlag(curlOutputFilterPathFlagName), 113 | } 114 | input.Path, _ = cmd.Flags().GetString(curlPathFlagName) 115 | input.QueryParams, _ = cmd.Flags().GetString(curlQueryParamsFlagName) 116 | input.Data, _ = cmd.Flags().GetString(curlDataFlagName) 117 | input.Headers, _ = cmd.Flags().GetString(curlHeadersFlagName) 118 | err := CurlActionExecute(input) 119 | DisplayError(err, cmdName) 120 | } 121 | -------------------------------------------------------------------------------- /commands/curl_delete.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | "github.com/spf13/cobra" 20 | ) 21 | 22 | const curlDeleteCommandName = "delete" 23 | 24 | var curlDeleteExample = ` 25 | # Delete a document from an index. 26 | odfe-cli curl delete --path "my-index/_doc/1" \ 27 | --query-params "routing=odfe-node1" 28 | ` 29 | 30 | var curlDeleteCmd = &cobra.Command{ 31 | Use: curlDeleteCommandName + " [flags] ", 32 | Short: "Delete command to execute requests against cluster", 33 | Long: "Delete command enables you to run any DELETE API against cluster", 34 | Example: curlDeleteExample, 35 | Run: func(cmd *cobra.Command, args []string) { 36 | Run(*cmd, curlDeleteCommandName) 37 | }, 38 | } 39 | 40 | func init() { 41 | GetCurlCommand().AddCommand(curlDeleteCmd) 42 | curlDeleteCmd.Flags().StringP(curlPathFlagName, "P", "", "URL path for the REST API") 43 | _ = curlDeleteCmd.MarkFlagRequired(curlPathFlagName) 44 | curlDeleteCmd.Flags().StringP(curlQueryParamsFlagName, "q", "", 45 | "URL query parameters (key & value) for the REST API. Use ‘&’ to separate multiple parameters. Ex: -q \"v=true&s=order:desc,index_patterns\"") 46 | curlDeleteCmd.Flags().StringP( 47 | curlHeadersFlagName, "H", "", 48 | "Headers for the REST API. Consists of case-insensitive name followed by a colon (`:`), then by its value. Use ';' to separate multiple parameters. Ex: -H \"content-type:json;accept-encoding:gzip\"") 49 | curlDeleteCmd.Flags().BoolP("help", "h", false, "Help for curl "+curlDeleteCommandName) 50 | } 51 | -------------------------------------------------------------------------------- /commands/curl_get.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | "github.com/spf13/cobra" 20 | ) 21 | 22 | const curlGetCommandName = "get" 23 | 24 | var curlGetExample = ` 25 | # get document count for an index 26 | odfe-cli curl get --path "_cat/count/my-index-01" --query-params "v=true" --pretty 27 | 28 | # get health status of a cluster. 29 | odfe-cli curl get --path "_cluster/health" --pretty --filter-path "status" 30 | 31 | # get explanation for cluster allocation for a given index and shard number 32 | odfe-cli curl get --path "_cluster/allocation/explain" \ 33 | --data '{ 34 | "index": "my-index-01", 35 | "shard": 0, 36 | "primary": false, 37 | "current_node": "nodeA" 38 | }' 39 | ` 40 | 41 | var curlGetCmd = &cobra.Command{ 42 | Use: curlGetCommandName + " [flags] ", 43 | Short: "Get command to execute requests against cluster", 44 | Long: "Get command enables you to run any GET API against cluster", 45 | Example: curlGetExample, 46 | Run: func(cmd *cobra.Command, args []string) { 47 | Run(*cmd, curlGetCommandName) 48 | }, 49 | } 50 | 51 | func init() { 52 | GetCurlCommand().AddCommand(curlGetCmd) 53 | curlGetCmd.Flags().StringP(curlPathFlagName, "P", "", "URL path for the REST API") 54 | _ = curlGetCmd.MarkFlagRequired(curlPathFlagName) 55 | curlGetCmd.Flags().StringP(curlQueryParamsFlagName, "q", "", 56 | "URL query parameters (key & value) for the REST API. Use ‘&’ to separate multiple parameters. Ex: -q \"v=true&s=order:desc,index_patterns\"") 57 | curlGetCmd.Flags().StringP( 58 | curlDataFlagName, "d", "", 59 | "Data for the REST API. If value starts with '@', the rest should be a file name to read the data from.") 60 | curlGetCmd.Flags().StringP( 61 | curlHeadersFlagName, "H", "", 62 | "Headers for the REST API. Consists of case-insensitive name followed by a colon (`:`), then by its value. Use ';' to separate multiple parameters. Ex: -H \"content-type:json;accept-encoding:gzip\"") 63 | curlGetCmd.Flags().BoolP("help", "h", false, "Help for curl "+curlGetCommandName) 64 | } 65 | -------------------------------------------------------------------------------- /commands/curl_post.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | "github.com/spf13/cobra" 20 | ) 21 | 22 | const curlPostCommandName = "post" 23 | 24 | var postExample = ` 25 | # change the allocation of shards in a cluster. 26 | odfe-cli curl post --path "_cluster/reroute" \ 27 | --data ' 28 | { 29 | "commands": [ 30 | { 31 | "move": { 32 | "index": "odfe-cli", "shard": 0, 33 | "from_node": "odfe-node1", "to_node": "odfe-node2" 34 | } 35 | }, 36 | { 37 | "allocate_replica": { 38 | "index": "test", "shard": 1, 39 | "node": "odfe-node3" 40 | } 41 | } 42 | ]}' \ 43 | --pretty 44 | 45 | # insert a document to an index 46 | odfe-cli curl post --path "my-index-01/_doc" \ 47 | --data ' 48 | { 49 | "message": "insert document", 50 | "ip": { 51 | "address": "127.0.0.1" 52 | } 53 | }' 54 | 55 | ` 56 | var curlPostCmd = &cobra.Command{ 57 | Use: curlPostCommandName + " [flags] ", 58 | Short: "Post command to execute requests against cluster", 59 | Long: "Post command enables you to run any POST API against cluster", 60 | Example: postExample, 61 | Run: func(cmd *cobra.Command, args []string) { 62 | Run(*cmd, curlPostCommandName) 63 | }, 64 | } 65 | 66 | func init() { 67 | GetCurlCommand().AddCommand(curlPostCmd) 68 | curlPostCmd.Flags().StringP(curlPathFlagName, "P", "", "URL path for the REST API") 69 | _ = curlPostCmd.MarkFlagRequired(curlPathFlagName) 70 | curlPostCmd.Flags().StringP(curlQueryParamsFlagName, "q", "", 71 | "URL query parameters (key & value) for the REST API. Use ‘&’ to separate multiple parameters. Ex: -q \"v=true&s=order:desc,index_patterns\"") 72 | curlPostCmd.Flags().StringP( 73 | curlDataFlagName, "d", "", 74 | "Data for the REST API. If value starts with '@', the rest should be a file name to read the data from.") 75 | curlPostCmd.Flags().StringP( 76 | curlHeadersFlagName, "H", "", 77 | "Headers for the REST API. Consists of case-insensitive name followed by a colon (`:`), then by its value. Use ';' to separate multiple parameters. Ex: -H \"content-type:json;accept-encoding:gzip\"") 78 | curlPostCmd.Flags().BoolP("help", "h", false, "Help for curl "+curlPostCommandName) 79 | } 80 | -------------------------------------------------------------------------------- /commands/curl_put.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | "github.com/spf13/cobra" 20 | ) 21 | 22 | const curlPutCommandName = "put" 23 | 24 | var curlPutExample = ` 25 | # Create a knn index from mapping setting saved in file "knn-mapping.json" 26 | odfe-cli curl put --path "my-knn-index" \ 27 | --data "@some-location/knn-mapping.json" \ 28 | --pretty 29 | 30 | # Update cluster settings transiently 31 | odfe-cli curl put --path "_cluster/settings" \ 32 | --query-params "flat_settings=true" \ 33 | --data ' 34 | { 35 | "transient" : { 36 | "indices.recovery.max_bytes_per_sec" : "20mb" 37 | } 38 | }' \ 39 | --pretty 40 | 41 | ` 42 | 43 | var curlPutCmd = &cobra.Command{ 44 | Use: curlPutCommandName + " [flags] ", 45 | Short: "Put command to execute requests against cluster", 46 | Long: "Put command enables you to run any PUT API against cluster", 47 | Example: curlPutExample, 48 | Run: func(cmd *cobra.Command, args []string) { 49 | Run(*cmd, curlPutCommandName) 50 | }, 51 | } 52 | 53 | func init() { 54 | GetCurlCommand().AddCommand(curlPutCmd) 55 | curlPutCmd.Flags().StringP(curlPathFlagName, "P", "", "URL path for the REST API") 56 | _ = curlPutCmd.MarkFlagRequired(curlPathFlagName) 57 | curlPutCmd.Flags().StringP(curlQueryParamsFlagName, "q", "", 58 | "URL query parameters (key & value) for the REST API. Use ‘&’ to separate multiple parameters. Ex: -q \"v=true&s=order:desc,index_patterns\"") 59 | curlPutCmd.Flags().StringP( 60 | curlDataFlagName, "d", "", 61 | "Data for the REST API. If value starts with '@', the rest should be a file name to read the data from.") 62 | curlPutCmd.Flags().StringP( 63 | curlHeadersFlagName, "H", "", 64 | "Headers for the REST API. Consists of case-insensitive name followed by a colon (`:`), then by its value. Use ';' to separate multiple parameters. Ex: -H \"content-type:json;accept-encoding:gzip\"") 65 | curlPutCmd.Flags().BoolP("help", "h", false, "Help for curl "+curlPutCommandName) 66 | } 67 | -------------------------------------------------------------------------------- /commands/file_operation_unix.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | /* 4 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"). 7 | * You may not use this file except in compliance with the License. 8 | * A copy of the License is located at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * or in the "license" file accompanying this file. This file is distributed 13 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 14 | * express or implied. See the License for the specific language governing 15 | * permissions and limitations under the License. 16 | */ 17 | 18 | package commands 19 | 20 | import ( 21 | "fmt" 22 | "os" 23 | "path/filepath" 24 | ) 25 | 26 | const ( 27 | FolderPermission = 0700 // only owner can read, write and execute 28 | FilePermission = 0600 // only owner can read and write 29 | ) 30 | 31 | // createDefaultConfigFolderIfNotExists creates default config file along with folder if 32 | // it doesn't exists 33 | func createDefaultConfigFileIfNotExists() error { 34 | defaultFilePath := GetDefaultConfigFilePath() 35 | if isExists(defaultFilePath) { 36 | return nil 37 | } 38 | folderPath := filepath.Dir(defaultFilePath) 39 | if !isExists(folderPath) { 40 | err := os.Mkdir(folderPath, FolderPermission) 41 | if err != nil { 42 | return err 43 | } 44 | } 45 | f, err := os.Create(defaultFilePath) 46 | if err != nil { 47 | return err 48 | } 49 | if err = f.Chmod(FilePermission); err != nil { 50 | return err 51 | } 52 | return f.Close() 53 | } 54 | 55 | func checkConfigFilePermission(configFilePath string) error { 56 | //check for config file permission 57 | info, err := os.Stat(configFilePath) 58 | if err != nil { 59 | return fmt.Errorf("failed to get config file info due to: %w", err) 60 | } 61 | mode := info.Mode().Perm() 62 | 63 | if mode != FilePermission { 64 | return fmt.Errorf("permissions %o for '%s' are too open. It is required that your config file is NOT accessible by others", mode, configFilePath) 65 | } 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /commands/file_operation_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | /* 4 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"). 7 | * You may not use this file except in compliance with the License. 8 | * A copy of the License is located at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * or in the "license" file accompanying this file. This file is distributed 13 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 14 | * express or implied. See the License for the specific language governing 15 | * permissions and limitations under the License. 16 | */ 17 | 18 | package commands 19 | 20 | import "fmt" 21 | 22 | func createDefaultConfigFileIfNotExists() error { 23 | return fmt.Errorf("creating default config file is not supported for windows. Please create manually") 24 | } 25 | 26 | func checkConfigFilePermission(configFilePath string) error { 27 | // since windows doesn't support create default config file, no validation is required 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /commands/knn.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | "fmt" 20 | "odfe-cli/client" 21 | ctrl "odfe-cli/controller/knn" 22 | gateway "odfe-cli/gateway/knn" 23 | handler "odfe-cli/handler/knn" 24 | 25 | "github.com/spf13/cobra" 26 | ) 27 | 28 | const ( 29 | knnCommandName = "knn" 30 | knnStatsCommandName = "stats" 31 | knnWarmupCommandName = "warmup" 32 | knnStatsNodesFlagName = "nodes" 33 | knnStatsNamesFlagName = "stat-names" 34 | ) 35 | 36 | //knnCommand is base command for k-NN plugin. 37 | var knnCommand = &cobra.Command{ 38 | Use: knnCommandName, 39 | Short: "Manage the k-NN plugin", 40 | Long: "Use the k-NN commands to perform operations like stats, warmup.", 41 | } 42 | 43 | //knnStatsCommandName provide stats command for k-NN plugin. 44 | var knnStatsCommand = &cobra.Command{ 45 | Use: knnStatsCommandName, 46 | Short: "Display current status of the k-NN Plugin", 47 | Long: "Display current status of the k-NN Plugin.", 48 | Run: func(cmd *cobra.Command, args []string) { 49 | h, err := GetKNNHandler() 50 | if err != nil { 51 | DisplayError(err, knnStatsCommandName) 52 | return 53 | } 54 | nodes, err := cmd.Flags().GetString(knnStatsNodesFlagName) 55 | if err != nil { 56 | DisplayError(err, knnStatsCommandName) 57 | return 58 | } 59 | names, err := cmd.Flags().GetString(knnStatsNamesFlagName) 60 | if err != nil { 61 | DisplayError(err, knnStatsCommandName) 62 | return 63 | } 64 | err = getStatistics(h, nodes, names) 65 | DisplayError(err, knnStatsCommandName) 66 | }, 67 | } 68 | 69 | //knnWarmupCommand warmups shards 70 | var knnWarmupCommand = &cobra.Command{ 71 | Use: knnWarmupCommandName + " index ..." + " [flags] ", 72 | Args: cobra.MinimumNArgs(1), 73 | Short: "Warmup shards for given indices", 74 | Long: "Warmup command loads all graphs for all of the shards (primaries and replicas) " + 75 | "for given indices into native memory.\nThis is an asynchronous operation. If the command times out, " + 76 | "the operation will still be going on in the cluster.\nTo monitor this, use the Elasticsearch _tasks API. " + 77 | "Use `odfe-cli knn stats` command to verify whether indices are successfully loaded into memory.", 78 | Run: func(cmd *cobra.Command, args []string) { 79 | h, err := GetKNNHandler() 80 | if err != nil { 81 | DisplayError(err, knnWarmupCommandName) 82 | return 83 | } 84 | err = warmupIndices(h, args) 85 | DisplayError(err, knnWarmupCommandName) 86 | }, 87 | } 88 | 89 | func GetKNNCommand() *cobra.Command { 90 | return knnCommand 91 | } 92 | 93 | func GetKNNStatsCommand() *cobra.Command { 94 | return knnStatsCommand 95 | } 96 | 97 | func GetKNNWarmupCommand() *cobra.Command { 98 | return knnWarmupCommand 99 | } 100 | 101 | func init() { 102 | //knn base command 103 | knnCommand.Flags().BoolP("help", "h", false, "Help for k-NN plugin") 104 | GetRoot().AddCommand(knnCommand) 105 | //knn stats command 106 | knnStatsCommand.Flags().BoolP("help", "h", false, "Help for k-NN plugin stats command") 107 | knnStatsCommand.Flags().StringP(knnStatsNodesFlagName, "n", "", "Input is list of node Ids, separated by ','") 108 | knnStatsCommand.Flags().StringP(knnStatsNamesFlagName, "s", "", "Input is list of stats names, separated by ','") 109 | knnCommand.AddCommand(knnStatsCommand) 110 | //knn warmup command 111 | knnWarmupCommand.Flags().BoolP("help", "h", false, "Help for k-NN plugin warmup command") 112 | knnCommand.AddCommand(knnWarmupCommand) 113 | } 114 | 115 | func getStatistics(h *handler.Handler, nodes string, names string) error { 116 | stats, err := handler.GetStatistics(h, nodes, names) 117 | if err != nil { 118 | return err 119 | } 120 | fmt.Println(string(stats)) 121 | return nil 122 | } 123 | 124 | func warmupIndices(h *handler.Handler, index []string) error { 125 | shards, err := handler.WarmupIndices(h, index) 126 | if err != nil { 127 | return err 128 | } 129 | if shards.Failed > 0 { 130 | return fmt.Errorf("%d/%d shards were failed to load into memory", shards.Failed, shards.Total) 131 | } 132 | fmt.Printf("successfully loaded %d shards into memory\n", shards.Total) 133 | return nil 134 | } 135 | 136 | //GetKNNHandler returns handler by wiring the dependency manually 137 | func GetKNNHandler() (*handler.Handler, error) { 138 | c, err := client.New(nil) 139 | if err != nil { 140 | return nil, err 141 | } 142 | profile, err := GetProfile() 143 | if err != nil { 144 | return nil, err 145 | } 146 | g := gateway.New(c, profile) 147 | ctr := ctrl.New(g) 148 | return handler.New(ctr), nil 149 | } 150 | -------------------------------------------------------------------------------- /commands/knn_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | "bytes" 20 | "testing" 21 | 22 | "github.com/spf13/cobra" 23 | "github.com/stretchr/testify/assert" 24 | ) 25 | 26 | func executeCommand(root *cobra.Command, args ...string) (output string, err error) { 27 | _, output, err = executeCommandC(root, args...) 28 | return output, err 29 | } 30 | 31 | func executeCommandC(root *cobra.Command, args ...string) (c *cobra.Command, output string, err error) { 32 | buf := new(bytes.Buffer) 33 | root.SetOut(buf) 34 | root.SetErr(buf) 35 | root.SetArgs(args) 36 | 37 | c, err = root.ExecuteC() 38 | 39 | return c, buf.String(), err 40 | } 41 | 42 | func TestGetStatistics(t *testing.T) { 43 | t.Run("test stats command arguments", func(t *testing.T) { 44 | rootCmd := GetRoot() 45 | knnCommand := GetKNNCommand() 46 | knnStatsCmd := GetKNNStatsCommand() 47 | knnCommand.AddCommand(knnStatsCmd) 48 | rootCmd.AddCommand(knnCommand) 49 | _, err := executeCommand(rootCmd, knnCommandName, knnStatsCommandName, "--nodes", "node1,node2", "--stat-names", "stat1") 50 | assert.NoError(t, err) 51 | statNames, err := knnStatsCmd.Flags().GetString(knnStatsNamesFlagName) 52 | assert.NoError(t, err) 53 | assert.EqualValues(t, "stat1", statNames) 54 | nodeNames, err := knnStatsCmd.Flags().GetString(knnStatsNodesFlagName) 55 | assert.NoError(t, err) 56 | assert.EqualValues(t, "node1,node2", nodeNames) 57 | }) 58 | } 59 | 60 | func TestWarmupIndices(t *testing.T) { 61 | t.Run("test warmup command failed", func(t *testing.T) { 62 | rootCmd := GetRoot() 63 | knnCommand := GetKNNCommand() 64 | knnWarmupCmd := GetKNNWarmupCommand() 65 | knnCommand.AddCommand(knnWarmupCmd) 66 | rootCmd.AddCommand(knnCommand) 67 | _, err := executeCommand(rootCmd, knnCommandName, knnWarmupCommandName) 68 | assert.Error(t, err) 69 | }) 70 | t.Run("test warmup command", func(t *testing.T) { 71 | rootCmd := GetRoot() 72 | knnCommand := GetKNNCommand() 73 | knnWarmupCmd := GetKNNWarmupCommand() 74 | knnCommand.AddCommand(knnWarmupCmd) 75 | rootCmd.AddCommand(knnCommand) 76 | result, err := executeCommand(rootCmd, knnCommandName, knnWarmupCommandName, "index1", "index2") 77 | assert.NoError(t, err) 78 | assert.Empty(t, result) 79 | }) 80 | } 81 | -------------------------------------------------------------------------------- /commands/root.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | "fmt" 20 | "odfe-cli/entity" 21 | "os" 22 | "path/filepath" 23 | "runtime" 24 | 25 | "github.com/spf13/cobra" 26 | ) 27 | 28 | const ( 29 | configFileType = "yaml" 30 | defaultConfigFileName = "config" 31 | flagConfig = "config" 32 | flagProfileName = "profile" 33 | odfeConfigEnvVarName = "ODFE_CLI_CONFIG" 34 | RootCommandName = "odfe-cli" 35 | version = "1.1.1" 36 | ) 37 | 38 | func buildVersionString() string { 39 | 40 | return fmt.Sprintf("%s %s/%s", version, runtime.GOOS, runtime.GOARCH) 41 | } 42 | 43 | var rootCommand = &cobra.Command{ 44 | Use: RootCommandName, 45 | Short: "odfe-cli is a unified command line interface for managing ODFE clusters", 46 | Version: buildVersionString(), 47 | } 48 | 49 | func GetRoot() *cobra.Command { 50 | return rootCommand 51 | } 52 | 53 | // Execute executes the root command. 54 | func Execute() error { 55 | err := rootCommand.Execute() 56 | return err 57 | } 58 | 59 | func GetDefaultConfigFilePath() string { 60 | return filepath.Join( 61 | getDefaultConfigFolderRootPath(), 62 | fmt.Sprintf(".%s", RootCommandName), 63 | fmt.Sprintf("%s.%s", defaultConfigFileName, configFileType), 64 | ) 65 | } 66 | 67 | func getDefaultConfigFolderRootPath() string { 68 | if homeDir, err := os.UserHomeDir(); err == nil { 69 | return homeDir 70 | } 71 | if cwd, err := os.Getwd(); err == nil { 72 | return cwd 73 | } 74 | return "" 75 | } 76 | 77 | func init() { 78 | cobra.OnInitialize() 79 | configFilePath := GetDefaultConfigFilePath() 80 | rootCommand.PersistentFlags().StringP(flagConfig, "c", "", fmt.Sprintf("Configuration file for odfe-cli, default is %s", configFilePath)) 81 | rootCommand.PersistentFlags().StringP(flagProfileName, "p", "", "Use a specific profile from your configuration file") 82 | rootCommand.Flags().BoolP("version", "v", false, "Version for odfe-cli") 83 | rootCommand.Flags().BoolP("help", "h", false, "Help for odfe-cli") 84 | } 85 | 86 | // GetConfigFilePath gets config file path for execution 87 | func GetConfigFilePath(configFlagValue string) (string, error) { 88 | 89 | if configFlagValue != "" { 90 | return configFlagValue, nil 91 | } 92 | if value, ok := os.LookupEnv(odfeConfigEnvVarName); ok { 93 | return value, nil 94 | } 95 | if err := createDefaultConfigFileIfNotExists(); err != nil { 96 | return "", err 97 | } 98 | return GetDefaultConfigFilePath(), nil 99 | } 100 | 101 | //isExists check if given path exists or not 102 | //if path is just a name, it will check in current directory 103 | func isExists(path string) bool { 104 | if _, err := os.Stat(path); os.IsNotExist(err) { 105 | return false 106 | } 107 | return true 108 | } 109 | 110 | // DisplayError prints command name and error on console and exists as well. 111 | func DisplayError(err error, cmdName string) { 112 | if err != nil { 113 | fmt.Println(cmdName, "Command failed.") 114 | fmt.Println("Reason:", err) 115 | } 116 | } 117 | 118 | // GetProfile gets profile details for current execution 119 | func GetProfile() (*entity.Profile, error) { 120 | p, err := GetProfileController() 121 | if err != nil { 122 | return nil, err 123 | } 124 | profileFlagValue, err := rootCommand.PersistentFlags().GetString(flagProfileName) 125 | if err != nil { 126 | return nil, err 127 | } 128 | profile, ok, err := p.GetProfileForExecution(profileFlagValue) 129 | if err != nil { 130 | return nil, err 131 | } 132 | if !ok { 133 | return nil, fmt.Errorf("no profile found for execution. Try %s %s --help for more information", RootCommandName, ProfileCommandName) 134 | } 135 | return &profile, nil 136 | } 137 | -------------------------------------------------------------------------------- /commands/root_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package commands 17 | 18 | import ( 19 | "fmt" 20 | "io/ioutil" 21 | "odfe-cli/entity" 22 | "os" 23 | "path/filepath" 24 | "runtime" 25 | "testing" 26 | 27 | "github.com/stretchr/testify/assert" 28 | ) 29 | 30 | func TestGetConfigFilePath(t *testing.T) { 31 | 32 | t.Run("config file path from os environment variable", func(t *testing.T) { 33 | err := os.Setenv(odfeConfigEnvVarName, "test/config.yml") 34 | assert.NoError(t, err) 35 | filePath, err := GetConfigFilePath("") 36 | assert.NoError(t, err) 37 | assert.EqualValues(t, "test/config.yml", filePath) 38 | }) 39 | t.Run("config file path from command line arguments", func(t *testing.T) { 40 | filePath, err := GetConfigFilePath("test/config.yml") 41 | assert.NoError(t, err) 42 | assert.EqualValues(t, "test/config.yml", filePath) 43 | }) 44 | } 45 | 46 | func TestGetRoot(t *testing.T) { 47 | t.Run("test root command", func(t *testing.T) { 48 | root := GetRoot() 49 | assert.NotNil(t, root) 50 | root.SetArgs([]string{"--config", "test/config.yml"}) 51 | cmd, err := root.ExecuteC() 52 | assert.NoError(t, err) 53 | actual, err := cmd.Flags().GetString(flagConfig) 54 | assert.NoError(t, err) 55 | assert.EqualValues(t, "test/config.yml", actual) 56 | }) 57 | } 58 | 59 | func TestVersionString(t *testing.T) { 60 | t.Run("test version flag", func(t *testing.T) { 61 | root := GetRoot() 62 | assert.NotNil(t, root) 63 | root.SetArgs([]string{"--version"}) 64 | cmd, err := root.ExecuteC() 65 | assert.NoError(t, err) 66 | expected := "1.1.1 " + runtime.GOOS + "/" + runtime.GOARCH 67 | assert.EqualValues(t, expected, cmd.Version) 68 | }) 69 | } 70 | func createTempConfigFile(testFilePath string) (*os.File, error) { 71 | content, err := ioutil.ReadFile(testFilePath) 72 | if err != nil { 73 | return nil, err 74 | } 75 | tmpfile, err := ioutil.TempFile(os.TempDir(), "test-file") 76 | if err != nil { 77 | return nil, err 78 | } 79 | if _, err := tmpfile.Write(content); err != nil { 80 | os.Remove(tmpfile.Name()) // clean up 81 | return nil, err 82 | } 83 | if runtime.GOOS == "windows" { 84 | return tmpfile, nil 85 | } 86 | if err := tmpfile.Chmod(0600); err != nil { 87 | os.Remove(tmpfile.Name()) // clean up 88 | return nil, err 89 | } 90 | return tmpfile, nil 91 | } 92 | 93 | func TestGetProfile(t *testing.T) { 94 | t.Run("get default profile", func(t *testing.T) { 95 | root := GetRoot() 96 | assert.NotNil(t, root) 97 | profileFile, err := createTempConfigFile("testdata/config.yaml") 98 | assert.NoError(t, err) 99 | filePath, err := filepath.Abs(profileFile.Name()) 100 | assert.NoError(t, err) 101 | root.SetArgs([]string{"--config", filePath}) 102 | _, err = root.ExecuteC() 103 | assert.NoError(t, err) 104 | actual, err := GetProfile() 105 | assert.NoError(t, err) 106 | expectedProfile := entity.Profile{Name: "default", Endpoint: "http://localhost:9200", UserName: "default", Password: "admin"} 107 | assert.EqualValues(t, expectedProfile, *actual) 108 | os.Remove(profileFile.Name()) 109 | }) 110 | t.Run("test get profile", func(t *testing.T) { 111 | root := GetRoot() 112 | assert.NotNil(t, root) 113 | profileFile, err := createTempConfigFile("testdata/config.yaml") 114 | assert.NoError(t, err) 115 | filePath, err := filepath.Abs(profileFile.Name()) 116 | assert.NoError(t, err) 117 | root.SetArgs([]string{"--config", filePath, "--profile", "test"}) 118 | _, err = root.ExecuteC() 119 | assert.NoError(t, err) 120 | actual, err := GetProfile() 121 | assert.NoError(t, err) 122 | expectedProfile := entity.Profile{Name: "test", Endpoint: "https://localhost:9200", UserName: "admin", Password: "admin"} 123 | assert.EqualValues(t, expectedProfile, *actual) 124 | }) 125 | t.Run("Profile mismatch", func(t *testing.T) { 126 | root := GetRoot() 127 | assert.NotNil(t, root) 128 | profileFile, err := createTempConfigFile("testdata/config.yaml") 129 | assert.NoError(t, err) 130 | filePath, err := filepath.Abs(profileFile.Name()) 131 | assert.NoError(t, err) 132 | root.SetArgs([]string{"--config", filePath, "--profile", "test1"}) 133 | _, err = root.ExecuteC() 134 | assert.NoError(t, err) 135 | a, err := GetProfile() 136 | assert.EqualErrorf(t, err, "profile 'test1' does not exist", "unexpected error") 137 | fmt.Print(a) 138 | }) 139 | t.Run("no config file found", func(t *testing.T) { 140 | root := GetRoot() 141 | assert.NotNil(t, root) 142 | root.SetArgs([]string{"--config", "testdata/config1.yaml", "--profile", "test1"}) 143 | _, err := root.ExecuteC() 144 | assert.NoError(t, err) 145 | _, err = GetProfile() 146 | assert.EqualError(t, err, "failed to get config file info due to: stat testdata/config1.yaml: no such file or directory", "unexpected error") 147 | }) 148 | t.Run("invalid config file permission", func(t *testing.T) { 149 | if runtime.GOOS == "windows" { 150 | t.Skipf("test case does not work on %s", runtime.GOOS) 151 | } 152 | 153 | root := GetRoot() 154 | assert.NotNil(t, root) 155 | profileFile, err := createTempConfigFile("testdata/config.yaml") 156 | assert.NoError(t, err) 157 | assert.NoError(t, profileFile.Chmod(0750)) 158 | filePath, err := filepath.Abs(profileFile.Name()) 159 | assert.NoError(t, err) 160 | root.SetArgs([]string{"--config", filePath, "--profile", "test"}) 161 | _, err = root.ExecuteC() 162 | assert.NoError(t, err) 163 | _, err = GetProfile() 164 | assert.EqualError(t, err, fmt.Sprintf("permissions 750 for '%s' are too open. It is required that your config file is NOT accessible by others", filePath), "unexpected error") 165 | }) 166 | } 167 | -------------------------------------------------------------------------------- /commands/testdata/config.yaml: -------------------------------------------------------------------------------- 1 | profiles: 2 | - name: test 3 | endpoint: https://localhost:9200 4 | user: admin 5 | password: admin 6 | - name: default 7 | endpoint: http://localhost:9200 8 | user: default 9 | password: admin 10 | -------------------------------------------------------------------------------- /controller/ad/testdata/create_failed_response.json: -------------------------------------------------------------------------------- 1 | { 2 | "error": { 3 | "root_cause": [ 4 | { 5 | "type": "illegal_argument_exception", 6 | "reason": "Cannot create anomaly detector with name [testdata-detector] as it's already used by detector [wR_1XXMBs3q1IVz33Sk-]" 7 | } 8 | ], 9 | "type": "illegal_argument_exception", 10 | "reason": "Cannot create anomaly detector with name [testdata-detector] as it's already used by detector [wR_1XXMBs3q1IVz33Sk-]" 11 | }, 12 | "status": 400 13 | } 14 | -------------------------------------------------------------------------------- /controller/ad/testdata/create_request.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test-detector", 3 | "description": "Test detector", 4 | "time_field": "timestamp", 5 | "indices": [ 6 | "order*" 7 | ], 8 | "feature_attributes": [ 9 | { 10 | "feature_name": "total_order", 11 | "feature_enabled": true, 12 | "aggregation_query": { 13 | "total_order": { 14 | "sum": { 15 | "field": "value" 16 | } 17 | } 18 | } 19 | } 20 | ], 21 | "filter_query": { 22 | "bool": { 23 | "filter": [ 24 | { 25 | "exists": { 26 | "field": "value", 27 | "boost": 1 28 | } 29 | } 30 | ] 31 | } 32 | }, 33 | "detection_interval": { 34 | "period": { 35 | "interval": 1, 36 | "unit": "Minutes" 37 | } 38 | }, 39 | "window_delay": { 40 | "period": { 41 | "interval": 1, 42 | "unit": "Minutes" 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /controller/ad/testdata/create_response.json: -------------------------------------------------------------------------------- 1 | { 2 | "_id": "m4ccEnIBTXsGi3mvMt9p", 3 | "_version": 1, 4 | "_seq_no": 3, 5 | "_primary_term": 1, 6 | "anomaly_detector": { 7 | "name": "test-detector", 8 | "description": "Test detector", 9 | "time_field": "timestamp", 10 | "indices": [ 11 | "order*" 12 | ], 13 | "filter_query": { 14 | "bool": { 15 | "filter": [ 16 | { 17 | "exists": { 18 | "field": "value", 19 | "boost": 1.0 20 | } 21 | } 22 | ], 23 | "adjust_pure_negative": true, 24 | "boost": 1.0 25 | } 26 | }, 27 | "detection_interval": { 28 | "period": { 29 | "interval": 1, 30 | "unit": "Minutes" 31 | } 32 | }, 33 | "window_delay": { 34 | "period": { 35 | "interval": 1, 36 | "unit": "Minutes" 37 | } 38 | }, 39 | "schema_version": 0, 40 | "feature_attributes": [ 41 | { 42 | "feature_id": "mYccEnIBTXsGi3mvMd8_", 43 | "feature_name": "total_order", 44 | "feature_enabled": true, 45 | "aggregation_query": { 46 | "total_order": { 47 | "sum": { 48 | "field": "value" 49 | } 50 | } 51 | } 52 | } 53 | ] 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /controller/ad/testdata/get_response.json: -------------------------------------------------------------------------------- 1 | { 2 | "_id" : "detectorID", 3 | "_version" : 1, 4 | "_primary_term" : 1, 5 | "_seq_no" : 3, 6 | "anomaly_detector" : { 7 | "name" : "detector", 8 | "description" : "Test detector", 9 | "time_field" : "timestamp", 10 | "indices" : [ 11 | "order*" 12 | ], 13 | "filter_query" : {"bool" : {"filter" : [{"exists" : {"field" : "value","boost" : 1.0}}],"adjust_pure_negative" : true,"boost" : 1.0}}, 14 | "detection_interval" : { 15 | "period" : { 16 | "interval" : 5, 17 | "unit" : "Minutes" 18 | } 19 | }, 20 | "window_delay" : { 21 | "period" : { 22 | "interval" : 1, 23 | "unit" : "Minutes" 24 | } 25 | }, 26 | "schema_version" : 0, 27 | "feature_attributes" : [ 28 | { 29 | "feature_id" : "mYccEnIBTXsGi3mvMd8_", 30 | "feature_name" : "total_order", 31 | "feature_enabled" : true, 32 | "aggregation_query" : {"total_order":{"sum":{"field":"value"}}} 33 | } 34 | ], 35 | "last_update_time" : 1589441737319 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /controller/ad/testdata/search_response.json: -------------------------------------------------------------------------------- 1 | { 2 | "took" : 6, 3 | "timed_out" : false, 4 | "_shards" : { 5 | "total" : 1, 6 | "successful" : 1, 7 | "skipped" : 0, 8 | "failed" : 0 9 | }, 10 | "hits" : { 11 | "total" : { 12 | "value" : 1, 13 | "relation" : "eq" 14 | }, 15 | "max_score" : 0.06453852, 16 | "hits" : [ 17 | { 18 | "_index" : ".opendistro-anomaly-detectors", 19 | "_type" : "_doc", 20 | "_id" : "detectorID", 21 | "_version" : 1, 22 | "_seq_no" : 494, 23 | "_primary_term" : 4, 24 | "_score" : 0.06453852, 25 | "_source" : { 26 | "name" : "detector", 27 | "description" : "Test detector", 28 | "time_field" : "utc_time", 29 | "indices" : [ 30 | "kibana_sample_data_ecommerce*" 31 | ], 32 | "filter_query" : { 33 | "bool" : { 34 | "must" : [ 35 | { 36 | "bool" : { 37 | "filter" : [ 38 | { 39 | "term" : { 40 | "day_of_week" : { 41 | "value" : "Thursday", 42 | "boost" : 1.0 43 | } 44 | } 45 | } 46 | ], 47 | "adjust_pure_negative" : true, 48 | "boost" : 1.0 49 | } 50 | }, 51 | { 52 | "bool" : { 53 | "filter" : [ 54 | { 55 | "term" : { 56 | "currency" : { 57 | "value" : "EUR", 58 | "boost" : 1.0 59 | } 60 | } 61 | } 62 | ], 63 | "adjust_pure_negative" : true, 64 | "boost" : 1.0 65 | } 66 | } 67 | ], 68 | "adjust_pure_negative" : true, 69 | "boost" : 1.0 70 | } 71 | }, 72 | "detection_interval" : { 73 | "period" : { 74 | "interval" : 1, 75 | "unit" : "Minutes" 76 | } 77 | }, 78 | "window_delay" : { 79 | "period" : { 80 | "interval" : 1, 81 | "unit" : "Minutes" 82 | } 83 | }, 84 | "schema_version" : 0, 85 | "feature_attributes" : [ 86 | { 87 | "feature_id" : "xVh0bnMBLlLTlH7nzohm", 88 | "feature_name" : "sum_total_quantity", 89 | "feature_enabled" : true, 90 | "aggregation_query" : { 91 | "sum_total_quantity" : { 92 | "sum" : { 93 | "field" : "total_quantity" 94 | } 95 | } 96 | } 97 | }, 98 | { 99 | "feature_id" : "xlh0bnMBLlLTlH7nzohm", 100 | "feature_name" : "average_total_quantity", 101 | "feature_enabled" : true, 102 | "aggregation_query" : { 103 | "average_total_quantity" : { 104 | "avg" : { 105 | "field" : "total_quantity" 106 | } 107 | } 108 | } 109 | } 110 | ], 111 | "last_update_time" : 1595286015594 112 | } 113 | } 114 | ] 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /controller/config/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package config 17 | 18 | import ( 19 | "io/ioutil" 20 | "odfe-cli/entity" 21 | "os" 22 | 23 | "gopkg.in/yaml.v3" 24 | ) 25 | 26 | //go:generate go run -mod=mod github.com/golang/mock/mockgen -destination=mocks/mock_config.go -package=mocks . Controller 27 | type Controller interface { 28 | Read() (entity.Config, error) 29 | Write(config entity.Config) error 30 | } 31 | 32 | type controller struct { 33 | path string 34 | } 35 | 36 | //Read deserialize config file into entity.Config 37 | func (c controller) Read() (result entity.Config, err error) { 38 | contents, err := ioutil.ReadFile(c.path) 39 | if err != nil { 40 | return 41 | } 42 | err = yaml.Unmarshal(contents, &result) 43 | return 44 | } 45 | 46 | //Write serialize entity.Config into file path 47 | func (c controller) Write(config entity.Config) (err error) { 48 | file, err := os.Create(c.path) //overwrite if file exists 49 | if err != nil { 50 | return err 51 | } 52 | defer func() { 53 | err = file.Close() 54 | }() 55 | contents, err := yaml.Marshal(config) 56 | if err != nil { 57 | return err 58 | } 59 | _, err = file.Write(contents) 60 | if err != nil { 61 | return err 62 | } 63 | return file.Sync() 64 | } 65 | 66 | //New returns config controller instance 67 | func New(path string) Controller { 68 | return controller{ 69 | path: path, 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /controller/config/config_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package config 17 | 18 | import ( 19 | "fmt" 20 | "io/ioutil" 21 | "odfe-cli/entity" 22 | "os" 23 | "path/filepath" 24 | "testing" 25 | 26 | "gopkg.in/yaml.v3" 27 | 28 | "github.com/stretchr/testify/assert" 29 | ) 30 | 31 | const testFileName = "config.yaml" 32 | const testFolderName = "testdata" 33 | 34 | func getSampleConfig() entity.Config { 35 | return entity.Config{ 36 | Profiles: []entity.Profile{ 37 | { 38 | Name: "local", 39 | Endpoint: "https://localhost:9200", 40 | UserName: "admin", Password: "admin", 41 | }, 42 | { 43 | Name: "default", 44 | Endpoint: "https://127.0.0.1:9200", 45 | UserName: "dadmin", Password: "dadmin", 46 | }, 47 | }} 48 | } 49 | 50 | func TestControllerRead(t *testing.T) { 51 | t.Run("success", func(t *testing.T) { 52 | ctrl := New(filepath.Join(testFolderName, testFileName)) 53 | cfg, err := ctrl.Read() 54 | assert.NoError(t, err) 55 | expected := getSampleConfig() 56 | assert.EqualValues(t, expected, cfg) 57 | }) 58 | t.Run("fail", func(t *testing.T) { 59 | fileName := filepath.Join(testFolderName, "invalid", testFileName) 60 | ctrl := New(fileName) 61 | _, err := ctrl.Read() 62 | assert.EqualError(t, err, fmt.Sprintf("open %s: no such file or directory", fileName)) 63 | }) 64 | } 65 | func TestControllerWrite(t *testing.T) { 66 | t.Run("success", func(t *testing.T) { 67 | f, err := ioutil.TempFile("", "config") 68 | assert.NoError(t, err) 69 | defer func() { 70 | err = os.Remove(f.Name()) 71 | assert.NoError(t, err) 72 | }() 73 | ctrl := New(f.Name()) 74 | err = ctrl.Write(getSampleConfig()) 75 | assert.NoError(t, err) 76 | contents, err := ioutil.ReadFile(f.Name()) 77 | assert.NoError(t, err) 78 | var config entity.Config 79 | err = yaml.Unmarshal(contents, &config) 80 | assert.NoError(t, err) 81 | assert.EqualValues(t, getSampleConfig(), config) 82 | }) 83 | } 84 | -------------------------------------------------------------------------------- /controller/config/mocks/mock_config.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: odfe-cli/controller/config (interfaces: Controller) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | entity "odfe-cli/entity" 9 | reflect "reflect" 10 | 11 | gomock "github.com/golang/mock/gomock" 12 | ) 13 | 14 | // MockController is a mock of Controller interface 15 | type MockController struct { 16 | ctrl *gomock.Controller 17 | recorder *MockControllerMockRecorder 18 | } 19 | 20 | // MockControllerMockRecorder is the mock recorder for MockController 21 | type MockControllerMockRecorder struct { 22 | mock *MockController 23 | } 24 | 25 | // NewMockController creates a new mock instance 26 | func NewMockController(ctrl *gomock.Controller) *MockController { 27 | mock := &MockController{ctrl: ctrl} 28 | mock.recorder = &MockControllerMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use 33 | func (m *MockController) EXPECT() *MockControllerMockRecorder { 34 | return m.recorder 35 | } 36 | 37 | // Read mocks base method 38 | func (m *MockController) Read() (entity.Config, error) { 39 | m.ctrl.T.Helper() 40 | ret := m.ctrl.Call(m, "Read") 41 | ret0, _ := ret[0].(entity.Config) 42 | ret1, _ := ret[1].(error) 43 | return ret0, ret1 44 | } 45 | 46 | // Read indicates an expected call of Read 47 | func (mr *MockControllerMockRecorder) Read() *gomock.Call { 48 | mr.mock.ctrl.T.Helper() 49 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Read", reflect.TypeOf((*MockController)(nil).Read)) 50 | } 51 | 52 | // Write mocks base method 53 | func (m *MockController) Write(arg0 entity.Config) error { 54 | m.ctrl.T.Helper() 55 | ret := m.ctrl.Call(m, "Write", arg0) 56 | ret0, _ := ret[0].(error) 57 | return ret0 58 | } 59 | 60 | // Write indicates an expected call of Write 61 | func (mr *MockControllerMockRecorder) Write(arg0 interface{}) *gomock.Call { 62 | mr.mock.ctrl.T.Helper() 63 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockController)(nil).Write), arg0) 64 | } 65 | -------------------------------------------------------------------------------- /controller/config/testdata/config.yaml: -------------------------------------------------------------------------------- 1 | profiles: 2 | - endpoint: https://localhost:9200 3 | user: admin 4 | password: admin 5 | name: local 6 | - endpoint: https://127.0.0.1:9200 7 | user: dadmin 8 | password: dadmin 9 | name: default 10 | -------------------------------------------------------------------------------- /controller/es/es.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | package es 16 | 17 | import ( 18 | "context" 19 | "encoding/json" 20 | "odfe-cli/entity/es" 21 | esg "odfe-cli/gateway/es" 22 | mapper "odfe-cli/mapper/es" 23 | 24 | "fmt" 25 | ) 26 | 27 | //go:generate go run -mod=mod github.com/golang/mock/mockgen -destination=mocks/mock_es.go -package=mocks . Controller 28 | 29 | //Controller is an interface for Elasticsearch 30 | type Controller interface { 31 | GetDistinctValues(ctx context.Context, index string, field string) ([]interface{}, error) 32 | Curl(ctx context.Context, param es.CurlCommandRequest) ([]byte, error) 33 | } 34 | 35 | type controller struct { 36 | gateway esg.Gateway 37 | } 38 | 39 | //New returns new instance of Controller 40 | func New(gateway esg.Gateway) Controller { 41 | return &controller{ 42 | gateway, 43 | } 44 | } 45 | 46 | //GetDistinctValues get only unique values for given index, given field name 47 | func (c controller) GetDistinctValues(ctx context.Context, index string, field string) ([]interface{}, error) { 48 | if len(index) == 0 || len(field) == 0 { 49 | return nil, fmt.Errorf("index and field cannot be empty") 50 | } 51 | response, err := c.gateway.SearchDistinctValues(ctx, index, field) 52 | if err != nil { 53 | return nil, err 54 | } 55 | var data es.Response 56 | err = json.Unmarshal(response, &data) 57 | if err != nil { 58 | return nil, err 59 | } 60 | 61 | var values []interface{} 62 | for _, bucket := range data.Aggregations.Items.Buckets { 63 | values = append(values, bucket.Key) 64 | } 65 | return values, nil 66 | } 67 | 68 | //Curl accept user request and convert to format which Elasticsearch can understand 69 | func (c controller) Curl(ctx context.Context, param es.CurlCommandRequest) ([]byte, error) { 70 | curlRequest, err := mapper.CommandToCurlRequestParameter(param) 71 | if err != nil { 72 | return nil, err 73 | } 74 | return c.gateway.Curl(ctx, curlRequest) 75 | } 76 | -------------------------------------------------------------------------------- /controller/es/es_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package es 17 | 18 | import ( 19 | "context" 20 | "errors" 21 | "io/ioutil" 22 | "net/http" 23 | "odfe-cli/entity/es" 24 | "odfe-cli/gateway/es/mocks" 25 | "path/filepath" 26 | "testing" 27 | 28 | "github.com/golang/mock/gomock" 29 | "github.com/stretchr/testify/assert" 30 | ) 31 | 32 | func helperLoadBytes(t *testing.T, name string) []byte { 33 | path := filepath.Join("testdata", name) // relative path 34 | bytes, err := ioutil.ReadFile(path) 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | return bytes 39 | } 40 | 41 | func helperConvertToInterface(input []string) []interface{} { 42 | s := make([]interface{}, len(input)) 43 | for i, v := range input { 44 | s[i] = v 45 | } 46 | return s 47 | } 48 | 49 | func TestController_GetDistinctValues(t *testing.T) { 50 | t.Run("empty index name", func(t *testing.T) { 51 | mockCtrl := gomock.NewController(t) 52 | defer mockCtrl.Finish() 53 | 54 | mockGateway := mocks.NewMockGateway(mockCtrl) 55 | ctx := context.Background() 56 | ctrl := New(mockGateway) 57 | _, err := ctrl.GetDistinctValues(ctx, "", "f1") 58 | assert.Error(t, err) 59 | }) 60 | t.Run("empty field name", func(t *testing.T) { 61 | mockCtrl := gomock.NewController(t) 62 | defer mockCtrl.Finish() 63 | 64 | mockGateway := mocks.NewMockGateway(mockCtrl) 65 | ctx := context.Background() 66 | ctrl := New(mockGateway) 67 | _, err := ctrl.GetDistinctValues(ctx, "", "") 68 | assert.Error(t, err) 69 | }) 70 | t.Run("gateway failed", func(t *testing.T) { 71 | mockCtrl := gomock.NewController(t) 72 | defer mockCtrl.Finish() 73 | 74 | mockGateway := mocks.NewMockGateway(mockCtrl) 75 | ctx := context.Background() 76 | mockGateway.EXPECT().SearchDistinctValues(ctx, "example", "f1").Return(nil, errors.New("search failed")) 77 | ctrl := New(mockGateway) 78 | _, err := ctrl.GetDistinctValues(ctx, "example", "f1") 79 | assert.Error(t, err) 80 | }) 81 | t.Run("gateway response failed", func(t *testing.T) { 82 | mockCtrl := gomock.NewController(t) 83 | defer mockCtrl.Finish() 84 | 85 | mockGateway := mocks.NewMockGateway(mockCtrl) 86 | ctx := context.Background() 87 | mockGateway.EXPECT().SearchDistinctValues(ctx, "example", "f1").Return([]byte("No response"), nil) 88 | ctrl := New(mockGateway) 89 | _, err := ctrl.GetDistinctValues(ctx, "example", "f1") 90 | assert.Error(t, err) 91 | }) 92 | t.Run("get distinct success", func(t *testing.T) { 93 | mockCtrl := gomock.NewController(t) 94 | defer mockCtrl.Finish() 95 | mockGateway := mocks.NewMockGateway(mockCtrl) 96 | ctx := context.Background() 97 | expectedResult := helperConvertToInterface([]string{"Packaged Foods", "Dairy", "Meat and Seafood"}) 98 | mockGateway.EXPECT().SearchDistinctValues(ctx, "example", "f1").Return(helperLoadBytes(t, "search_result.json"), nil) 99 | ctrl := New(mockGateway) 100 | result, err := ctrl.GetDistinctValues(ctx, "example", "f1") 101 | assert.NoError(t, err) 102 | assert.EqualValues(t, expectedResult, result) 103 | 104 | }) 105 | } 106 | 107 | func TestController_Curl(t *testing.T) { 108 | commandRequest := es.CurlCommandRequest{ 109 | Action: "post", 110 | Path: "", 111 | QueryParams: "", 112 | Headers: "", 113 | Data: "", 114 | Pretty: false, 115 | } 116 | 117 | request := es.CurlRequest{ 118 | Action: http.MethodPost, 119 | Path: "", 120 | QueryParams: "", 121 | Headers: nil, 122 | Data: nil, 123 | } 124 | t.Run("gateway success", func(t *testing.T) { 125 | mockCtrl := gomock.NewController(t) 126 | defer mockCtrl.Finish() 127 | mockGateway := mocks.NewMockGateway(mockCtrl) 128 | ctx := context.Background() 129 | mockGateway.EXPECT().Curl(ctx, request).Return([]byte("response"), nil) 130 | ctrl := New(mockGateway) 131 | data, err := ctrl.Curl(ctx, commandRequest) 132 | assert.NoError(t, err, "received error") 133 | assert.EqualValues(t, []byte("response"), data) 134 | }) 135 | t.Run("gateway response failed", func(t *testing.T) { 136 | mockCtrl := gomock.NewController(t) 137 | defer mockCtrl.Finish() 138 | mockGateway := mocks.NewMockGateway(mockCtrl) 139 | ctx := context.Background() 140 | mockGateway.EXPECT().Curl(ctx, request).Return(nil, errors.New("gateway failed")) 141 | ctrl := New(mockGateway) 142 | _, err := ctrl.Curl(ctx, commandRequest) 143 | assert.Error(t, err) 144 | }) 145 | t.Run("mapper failed", func(t *testing.T) { 146 | mockCtrl := gomock.NewController(t) 147 | defer mockCtrl.Finish() 148 | mockGateway := mocks.NewMockGateway(mockCtrl) 149 | ctx := context.Background() 150 | //mockGateway.EXPECT().Curl(ctx, request).Return(nil, errors.New("gateway failed")) 151 | ctrl := New(mockGateway) 152 | _, err := ctrl.Curl(ctx, es.CurlCommandRequest{}) 153 | assert.EqualErrorf(t, err, "action cannot be empty", "wrong error message") 154 | }) 155 | } 156 | -------------------------------------------------------------------------------- /controller/es/mocks/mock_es.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: odfe-cli/controller/es (interfaces: Controller) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | context "context" 9 | es "odfe-cli/entity/es" 10 | reflect "reflect" 11 | 12 | gomock "github.com/golang/mock/gomock" 13 | ) 14 | 15 | // MockController is a mock of Controller interface 16 | type MockController struct { 17 | ctrl *gomock.Controller 18 | recorder *MockControllerMockRecorder 19 | } 20 | 21 | // MockControllerMockRecorder is the mock recorder for MockController 22 | type MockControllerMockRecorder struct { 23 | mock *MockController 24 | } 25 | 26 | // NewMockController creates a new mock instance 27 | func NewMockController(ctrl *gomock.Controller) *MockController { 28 | mock := &MockController{ctrl: ctrl} 29 | mock.recorder = &MockControllerMockRecorder{mock} 30 | return mock 31 | } 32 | 33 | // EXPECT returns an object that allows the caller to indicate expected use 34 | func (m *MockController) EXPECT() *MockControllerMockRecorder { 35 | return m.recorder 36 | } 37 | 38 | // Curl mocks base method 39 | func (m *MockController) Curl(arg0 context.Context, arg1 es.CurlCommandRequest) ([]byte, error) { 40 | m.ctrl.T.Helper() 41 | ret := m.ctrl.Call(m, "Curl", arg0, arg1) 42 | ret0, _ := ret[0].([]byte) 43 | ret1, _ := ret[1].(error) 44 | return ret0, ret1 45 | } 46 | 47 | // Curl indicates an expected call of Curl 48 | func (mr *MockControllerMockRecorder) Curl(arg0, arg1 interface{}) *gomock.Call { 49 | mr.mock.ctrl.T.Helper() 50 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Curl", reflect.TypeOf((*MockController)(nil).Curl), arg0, arg1) 51 | } 52 | 53 | // GetDistinctValues mocks base method 54 | func (m *MockController) GetDistinctValues(arg0 context.Context, arg1, arg2 string) ([]interface{}, error) { 55 | m.ctrl.T.Helper() 56 | ret := m.ctrl.Call(m, "GetDistinctValues", arg0, arg1, arg2) 57 | ret0, _ := ret[0].([]interface{}) 58 | ret1, _ := ret[1].(error) 59 | return ret0, ret1 60 | } 61 | 62 | // GetDistinctValues indicates an expected call of GetDistinctValues 63 | func (mr *MockControllerMockRecorder) GetDistinctValues(arg0, arg1, arg2 interface{}) *gomock.Call { 64 | mr.mock.ctrl.T.Helper() 65 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDistinctValues", reflect.TypeOf((*MockController)(nil).GetDistinctValues), arg0, arg1, arg2) 66 | } 67 | -------------------------------------------------------------------------------- /controller/es/testdata/search_result.json: -------------------------------------------------------------------------------- 1 | { 2 | "took": 80, 3 | "timed_out": false, 4 | "_shards": { 5 | "total": 5, 6 | "successful": 5, 7 | "skipped": 0, 8 | "failed": 0 9 | }, 10 | "hits": { 11 | "total": 14, 12 | "max_score": 0, 13 | "hits": [] 14 | }, 15 | "aggregations": { 16 | "items": { 17 | "doc_count_error_upper_bound": 0, 18 | "sum_other_doc_count": 0, 19 | "buckets": [ 20 | { 21 | "key": "Packaged Foods", 22 | "doc_count": 4 23 | }, 24 | { 25 | "key": "Dairy", 26 | "doc_count": 3 27 | }, 28 | { 29 | "key": "Meat and Seafood", 30 | "doc_count": 2 31 | } 32 | ] 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /controller/knn/knn.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package knn 17 | 18 | import ( 19 | "context" 20 | "encoding/json" 21 | entity "odfe-cli/entity/knn" 22 | "odfe-cli/gateway/knn" 23 | "strings" 24 | ) 25 | 26 | //go:generate go run -mod=mod github.com/golang/mock/mockgen -destination=mocks/mock_knn.go -package=mocks . Controller 27 | 28 | //Controller is an interface for the k-NN plugin controllers 29 | type Controller interface { 30 | GetStatistics(context.Context, string, string) ([]byte, error) 31 | WarmupIndices(context.Context, []string) (*entity.Shards, error) 32 | } 33 | 34 | type controller struct { 35 | gateway knn.Gateway 36 | } 37 | 38 | //GetStatistics gets stats data based on nodes and stat names 39 | func (c controller) GetStatistics(ctx context.Context, nodes string, names string) ([]byte, error) { 40 | return c.gateway.GetStatistics(ctx, nodes, names) 41 | } 42 | 43 | //New returns new Controller instance 44 | func New(gateway knn.Gateway) Controller { 45 | return &controller{ 46 | gateway, 47 | } 48 | } 49 | 50 | //WarmupIndices will load all the graphs for all of the shards (primaries and replicas) 51 | //of all the indices specified in the request into native memory 52 | func (c controller) WarmupIndices(ctx context.Context, index []string) (*entity.Shards, error) { 53 | indices := strings.Join(index, ",") 54 | response, err := c.gateway.WarmupIndices(ctx, indices) 55 | if err != nil { 56 | return nil, err 57 | } 58 | var warmupAPI entity.WarmupAPIResponse 59 | err = json.Unmarshal(response, &warmupAPI) 60 | if err != nil { 61 | return nil, err 62 | } 63 | return &warmupAPI.Shards, nil 64 | } 65 | -------------------------------------------------------------------------------- /controller/knn/knn_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package knn 17 | 18 | import ( 19 | "context" 20 | "encoding/json" 21 | "errors" 22 | "odfe-cli/entity/knn" 23 | "odfe-cli/gateway/knn/mocks" 24 | "testing" 25 | 26 | "github.com/golang/mock/gomock" 27 | "github.com/stretchr/testify/assert" 28 | ) 29 | 30 | func TestControllerGetStatistics(t *testing.T) { 31 | t.Run("gateway failed", func(t *testing.T) { 32 | mockCtrl := gomock.NewController(t) 33 | defer mockCtrl.Finish() 34 | 35 | mockGateway := mocks.NewMockGateway(mockCtrl) 36 | ctx := context.Background() 37 | mockGateway.EXPECT().GetStatistics(ctx, "", "").Return(nil, errors.New("gateway failed")) 38 | ctrl := New(mockGateway) 39 | _, err := ctrl.GetStatistics(ctx, "", "") 40 | assert.Error(t, err) 41 | }) 42 | t.Run("get stats success", func(t *testing.T) { 43 | mockCtrl := gomock.NewController(t) 44 | defer mockCtrl.Finish() 45 | mockGateway := mocks.NewMockGateway(mockCtrl) 46 | ctx := context.Background() 47 | mockGateway.EXPECT().GetStatistics(ctx, "node1", "stats").Return([]byte(`response succeeded`), nil) 48 | ctrl := New(mockGateway) 49 | result, err := ctrl.GetStatistics(ctx, "node1", "stats") 50 | assert.NoError(t, err) 51 | assert.EqualValues(t, []byte(`response succeeded`), result) 52 | }) 53 | } 54 | 55 | func TestControllerWarmupIndices(t *testing.T) { 56 | t.Run("gateway failed", func(t *testing.T) { 57 | mockCtrl := gomock.NewController(t) 58 | defer mockCtrl.Finish() 59 | 60 | mockGateway := mocks.NewMockGateway(mockCtrl) 61 | ctx := context.Background() 62 | mockGateway.EXPECT().WarmupIndices(ctx, "index1").Return(nil, errors.New("gateway failed")) 63 | ctrl := New(mockGateway) 64 | _, err := ctrl.WarmupIndices(ctx, []string{"index1"}) 65 | assert.Error(t, err) 66 | }) 67 | t.Run("warmup success", func(t *testing.T) { 68 | mockCtrl := gomock.NewController(t) 69 | defer mockCtrl.Finish() 70 | mockGateway := mocks.NewMockGateway(mockCtrl) 71 | ctx := context.Background() 72 | expectedResponse := knn.WarmupAPIResponse{ 73 | Shards: knn.Shards{ 74 | Total: 10, 75 | Successful: 8, 76 | Failed: 2, 77 | }, 78 | } 79 | rawMessage, err := json.Marshal(expectedResponse) 80 | assert.NoError(t, err) 81 | mockGateway.EXPECT().WarmupIndices(ctx, "index1").Return(rawMessage, nil) 82 | ctrl := New(mockGateway) 83 | result, err := ctrl.WarmupIndices(ctx, []string{"index1"}) 84 | assert.NoError(t, err) 85 | assert.EqualValues(t, expectedResponse.Shards, *result) 86 | }) 87 | } 88 | -------------------------------------------------------------------------------- /controller/knn/mocks/mock_knn.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: odfe-cli/controller/knn (interfaces: Controller) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | context "context" 9 | knn "odfe-cli/entity/knn" 10 | reflect "reflect" 11 | 12 | gomock "github.com/golang/mock/gomock" 13 | ) 14 | 15 | // MockController is a mock of Controller interface 16 | type MockController struct { 17 | ctrl *gomock.Controller 18 | recorder *MockControllerMockRecorder 19 | } 20 | 21 | // MockControllerMockRecorder is the mock recorder for MockController 22 | type MockControllerMockRecorder struct { 23 | mock *MockController 24 | } 25 | 26 | // NewMockController creates a new mock instance 27 | func NewMockController(ctrl *gomock.Controller) *MockController { 28 | mock := &MockController{ctrl: ctrl} 29 | mock.recorder = &MockControllerMockRecorder{mock} 30 | return mock 31 | } 32 | 33 | // EXPECT returns an object that allows the caller to indicate expected use 34 | func (m *MockController) EXPECT() *MockControllerMockRecorder { 35 | return m.recorder 36 | } 37 | 38 | // GetStatistics mocks base method 39 | func (m *MockController) GetStatistics(arg0 context.Context, arg1, arg2 string) ([]byte, error) { 40 | m.ctrl.T.Helper() 41 | ret := m.ctrl.Call(m, "GetStatistics", arg0, arg1, arg2) 42 | ret0, _ := ret[0].([]byte) 43 | ret1, _ := ret[1].(error) 44 | return ret0, ret1 45 | } 46 | 47 | // GetStatistics indicates an expected call of GetStatistics 48 | func (mr *MockControllerMockRecorder) GetStatistics(arg0, arg1, arg2 interface{}) *gomock.Call { 49 | mr.mock.ctrl.T.Helper() 50 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatistics", reflect.TypeOf((*MockController)(nil).GetStatistics), arg0, arg1, arg2) 51 | } 52 | 53 | // WarmupIndices mocks base method 54 | func (m *MockController) WarmupIndices(arg0 context.Context, arg1 []string) (*knn.Shards, error) { 55 | m.ctrl.T.Helper() 56 | ret := m.ctrl.Call(m, "WarmupIndices", arg0, arg1) 57 | ret0, _ := ret[0].(*knn.Shards) 58 | ret1, _ := ret[1].(error) 59 | return ret0, ret1 60 | } 61 | 62 | // WarmupIndices indicates an expected call of WarmupIndices 63 | func (mr *MockControllerMockRecorder) WarmupIndices(arg0, arg1 interface{}) *gomock.Call { 64 | mr.mock.ctrl.T.Helper() 65 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WarmupIndices", reflect.TypeOf((*MockController)(nil).WarmupIndices), arg0, arg1) 66 | } 67 | -------------------------------------------------------------------------------- /controller/profile/mocks/mock_profile.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: odfe-cli/controller/profile (interfaces: Controller) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | entity "odfe-cli/entity" 9 | reflect "reflect" 10 | 11 | gomock "github.com/golang/mock/gomock" 12 | ) 13 | 14 | // MockController is a mock of Controller interface 15 | type MockController struct { 16 | ctrl *gomock.Controller 17 | recorder *MockControllerMockRecorder 18 | } 19 | 20 | // MockControllerMockRecorder is the mock recorder for MockController 21 | type MockControllerMockRecorder struct { 22 | mock *MockController 23 | } 24 | 25 | // NewMockController creates a new mock instance 26 | func NewMockController(ctrl *gomock.Controller) *MockController { 27 | mock := &MockController{ctrl: ctrl} 28 | mock.recorder = &MockControllerMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use 33 | func (m *MockController) EXPECT() *MockControllerMockRecorder { 34 | return m.recorder 35 | } 36 | 37 | // CreateProfile mocks base method 38 | func (m *MockController) CreateProfile(arg0 entity.Profile) error { 39 | m.ctrl.T.Helper() 40 | ret := m.ctrl.Call(m, "CreateProfile", arg0) 41 | ret0, _ := ret[0].(error) 42 | return ret0 43 | } 44 | 45 | // CreateProfile indicates an expected call of CreateProfile 46 | func (mr *MockControllerMockRecorder) CreateProfile(arg0 interface{}) *gomock.Call { 47 | mr.mock.ctrl.T.Helper() 48 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateProfile", reflect.TypeOf((*MockController)(nil).CreateProfile), arg0) 49 | } 50 | 51 | // DeleteProfiles mocks base method 52 | func (m *MockController) DeleteProfiles(arg0 []string) error { 53 | m.ctrl.T.Helper() 54 | ret := m.ctrl.Call(m, "DeleteProfiles", arg0) 55 | ret0, _ := ret[0].(error) 56 | return ret0 57 | } 58 | 59 | // DeleteProfiles indicates an expected call of DeleteProfiles 60 | func (mr *MockControllerMockRecorder) DeleteProfiles(arg0 interface{}) *gomock.Call { 61 | mr.mock.ctrl.T.Helper() 62 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteProfiles", reflect.TypeOf((*MockController)(nil).DeleteProfiles), arg0) 63 | } 64 | 65 | // GetProfileForExecution mocks base method 66 | func (m *MockController) GetProfileForExecution(arg0 string) (entity.Profile, bool, error) { 67 | m.ctrl.T.Helper() 68 | ret := m.ctrl.Call(m, "GetProfileForExecution", arg0) 69 | ret0, _ := ret[0].(entity.Profile) 70 | ret1, _ := ret[1].(bool) 71 | ret2, _ := ret[2].(error) 72 | return ret0, ret1, ret2 73 | } 74 | 75 | // GetProfileForExecution indicates an expected call of GetProfileForExecution 76 | func (mr *MockControllerMockRecorder) GetProfileForExecution(arg0 interface{}) *gomock.Call { 77 | mr.mock.ctrl.T.Helper() 78 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProfileForExecution", reflect.TypeOf((*MockController)(nil).GetProfileForExecution), arg0) 79 | } 80 | 81 | // GetProfileNames mocks base method 82 | func (m *MockController) GetProfileNames() ([]string, error) { 83 | m.ctrl.T.Helper() 84 | ret := m.ctrl.Call(m, "GetProfileNames") 85 | ret0, _ := ret[0].([]string) 86 | ret1, _ := ret[1].(error) 87 | return ret0, ret1 88 | } 89 | 90 | // GetProfileNames indicates an expected call of GetProfileNames 91 | func (mr *MockControllerMockRecorder) GetProfileNames() *gomock.Call { 92 | mr.mock.ctrl.T.Helper() 93 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProfileNames", reflect.TypeOf((*MockController)(nil).GetProfileNames)) 94 | } 95 | 96 | // GetProfiles mocks base method 97 | func (m *MockController) GetProfiles() ([]entity.Profile, error) { 98 | m.ctrl.T.Helper() 99 | ret := m.ctrl.Call(m, "GetProfiles") 100 | ret0, _ := ret[0].([]entity.Profile) 101 | ret1, _ := ret[1].(error) 102 | return ret0, ret1 103 | } 104 | 105 | // GetProfiles indicates an expected call of GetProfiles 106 | func (mr *MockControllerMockRecorder) GetProfiles() *gomock.Call { 107 | mr.mock.ctrl.T.Helper() 108 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProfiles", reflect.TypeOf((*MockController)(nil).GetProfiles)) 109 | } 110 | 111 | // GetProfilesMap mocks base method 112 | func (m *MockController) GetProfilesMap() (map[string]entity.Profile, error) { 113 | m.ctrl.T.Helper() 114 | ret := m.ctrl.Call(m, "GetProfilesMap") 115 | ret0, _ := ret[0].(map[string]entity.Profile) 116 | ret1, _ := ret[1].(error) 117 | return ret0, ret1 118 | } 119 | 120 | // GetProfilesMap indicates an expected call of GetProfilesMap 121 | func (mr *MockControllerMockRecorder) GetProfilesMap() *gomock.Call { 122 | mr.mock.ctrl.T.Helper() 123 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProfilesMap", reflect.TypeOf((*MockController)(nil).GetProfilesMap)) 124 | } 125 | -------------------------------------------------------------------------------- /controller/profile/profile.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package profile 17 | 18 | import ( 19 | "fmt" 20 | "odfe-cli/controller/config" 21 | "odfe-cli/entity" 22 | "os" 23 | "strings" 24 | ) 25 | 26 | const ( 27 | odfeProfileEnvVarName = "ODFE_PROFILE" 28 | odfeDefaultProfileName = "default" 29 | ) 30 | 31 | //go:generate go run -mod=mod github.com/golang/mock/mockgen -destination=mocks/mock_profile.go -package=mocks . Controller 32 | type Controller interface { 33 | CreateProfile(profile entity.Profile) error 34 | DeleteProfiles(names []string) error 35 | GetProfiles() ([]entity.Profile, error) 36 | GetProfileNames() ([]string, error) 37 | GetProfilesMap() (map[string]entity.Profile, error) 38 | GetProfileForExecution(name string) (entity.Profile, bool, error) 39 | } 40 | 41 | type controller struct { 42 | configCtrl config.Controller 43 | } 44 | 45 | //New returns new config controller instance 46 | func New(c config.Controller) Controller { 47 | return &controller{ 48 | configCtrl: c, 49 | } 50 | } 51 | 52 | //GetProfiles gets list of profiles fom config file 53 | func (c controller) GetProfiles() ([]entity.Profile, error) { 54 | data, err := c.configCtrl.Read() 55 | if err != nil { 56 | return nil, err 57 | } 58 | return data.Profiles, nil 59 | } 60 | 61 | //GetProfileNames gets list of profile names 62 | func (c controller) GetProfileNames() ([]string, error) { 63 | profiles, err := c.GetProfiles() 64 | if err != nil { 65 | return nil, err 66 | } 67 | var names []string 68 | for _, profile := range profiles { 69 | names = append(names, profile.Name) 70 | } 71 | return names, nil 72 | } 73 | 74 | //GetProfilesMap returns a map view of the profiles contained in config 75 | func (c controller) GetProfilesMap() (map[string]entity.Profile, error) { 76 | profiles, err := c.GetProfiles() 77 | if err != nil { 78 | return nil, err 79 | } 80 | result := make(map[string]entity.Profile) 81 | for _, p := range profiles { 82 | result[p.Name] = p 83 | } 84 | return result, nil 85 | } 86 | 87 | //CreateProfile creates profile by gets list of existing profiles, append new profile to list 88 | //and saves it in config file 89 | func (c controller) CreateProfile(p entity.Profile) error { 90 | data, err := c.configCtrl.Read() 91 | if err != nil { 92 | return err 93 | } 94 | data.Profiles = append(data.Profiles, p) 95 | return c.configCtrl.Write(data) 96 | } 97 | 98 | //DeleteProfiles loads all profile, deletes selected profiles, and saves rest in config file 99 | func (c controller) DeleteProfiles(names []string) error { 100 | profilesMap, err := c.GetProfilesMap() 101 | if err != nil { 102 | return err 103 | } 104 | var invalidProfileNames []string 105 | for _, name := range names { 106 | if _, ok := profilesMap[name]; !ok { 107 | invalidProfileNames = append(invalidProfileNames, name) 108 | continue 109 | } 110 | delete(profilesMap, name) 111 | } 112 | 113 | //load config 114 | data, err := c.configCtrl.Read() 115 | if err != nil { 116 | return err 117 | } 118 | 119 | //empty existing profile 120 | data.Profiles = nil 121 | for _, p := range profilesMap { 122 | // add existing profiles to the list 123 | data.Profiles = append(data.Profiles, p) 124 | } 125 | 126 | //save config 127 | err = c.configCtrl.Write(data) 128 | if err != nil { 129 | return err 130 | } 131 | 132 | // if found any invalid profiles 133 | if len(invalidProfileNames) > 0 { 134 | return fmt.Errorf("no profiles found for: %s", strings.Join(invalidProfileNames, ", ")) 135 | } 136 | return nil 137 | } 138 | 139 | // GetProfileForExecution returns profile information for current command execution 140 | // if profile name is provided as an argument, will return the profile, 141 | // if profile name is not provided as argument, we will check for environment variable 142 | // in session, then will check for profile named `default` 143 | // bool determines whether profile is valid or not 144 | func (c controller) GetProfileForExecution(name string) (value entity.Profile, ok bool, err error) { 145 | profiles, err := c.GetProfilesMap() 146 | if err != nil { 147 | return 148 | } 149 | if name != "" { 150 | if value, ok = profiles[name]; ok { 151 | return 152 | } 153 | return value, ok, fmt.Errorf("profile '%s' does not exist", name) 154 | } 155 | if envProfileName, exists := os.LookupEnv(odfeProfileEnvVarName); exists { 156 | if value, ok = profiles[envProfileName]; ok { 157 | return 158 | } 159 | return value, ok, fmt.Errorf("profile '%s' does not exist", envProfileName) 160 | } 161 | value, ok = profiles[odfeDefaultProfileName] 162 | return 163 | } 164 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Copied from https://opendistro.github.io/for-elasticsearch-docs/docs/install/docker/ 2 | # removed node-2 since we don't need to create two node cluster for integration test 3 | version: '3' 4 | services: 5 | odfe-test-node1: 6 | image: amazon/opendistro-for-elasticsearch:${ODFE_VERSION:-latest} 7 | container_name: odfe-test-node1 8 | environment: 9 | - cluster.name=odfe-test-cluster 10 | - node.name=odfe-test-node1 11 | - discovery.seed_hosts=odfe-test-node1 12 | - cluster.initial_master_nodes=odfe-test-node1 13 | - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping 14 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM 15 | ulimits: 16 | memlock: 17 | soft: -1 18 | hard: -1 19 | nofile: 20 | soft: 65536 # maximum number of open files for the Elasticsearch user, set to at least 65536 on modern systems 21 | hard: 65536 22 | volumes: 23 | - odfe-test-data1:/usr/share/elasticsearch/data 24 | ports: 25 | - 9200:9200 26 | - 9600:9600 # required for Performance Analyzer 27 | networks: 28 | - odfe-test-net 29 | 30 | volumes: 31 | odfe-test-data1: 32 | 33 | networks: 34 | odfe-test-net: 35 | -------------------------------------------------------------------------------- /docs/dev/images/design.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/odfe-cli/0d877a4ea99818da64fca8aa7104bc6a1d29e080/docs/dev/images/design.png -------------------------------------------------------------------------------- /docs/guide/release.md: -------------------------------------------------------------------------------- 1 | # Release guidelines 2 | 3 | 4 | odfe-cli versions are expressed as **x.y.z**, where **x** is the major version, **y** is the minor version, and **z** is the patch version, following [Semantic Versioning](https://semver.org/) terminology. 5 | 6 | **The major, minor & patch version is related to odfe-cli compatibility, instead of odfe’s compatibility.** 7 | 8 | 9 | **Q: What goes into new patch version release train?** 10 | 11 | * Known bugs that prevent users from executing tasks. 12 | * eg: Failed to delete non-running detectors by delete command, failed to handle error. 13 | * Security vulnerabilities. 14 | * eg: leaking user credentials, insecure interaction with components. 15 | * cli’s dependency is updated with new patch version which fixes bugs and security issues that affect your feature. (Treat your dependent library as your library) 16 | * eg: cobra released a new version which fixes some security vulnerabilities like [#1259](https://github.com/spf13/cobra/pull/1259) 17 | 18 | Any committer, who fixed issues, **related to above use case**, should notify admin to initiate new patch release. There is no SLA for patch releases, since this is triggered based on severity of the issue. 19 | 20 | 21 | 22 | **Q: What goes into new minor version release train?** 23 | 24 | * Any new commands ( could be a new plugin, a new sub command for any plugin ). A new command could be a new API that was released long ago but being included in new odfe-cli release or new API that will be released in next odfe release. 25 | * eg: on-board [k-nn](https://github.com/opendistro-for-elasticsearch/odfe-cli/pull/20/) plugin, add auto-complete feature for cli commands. 26 | * New parameter/flags for any command. 27 | * eg: odfe-cli 1.0.0 only displays detector configuration using get command for given name, if user would also like to see [detector job](https://opendistro.github.io/for-elasticsearch-docs/docs/ad/api/#get-detector), they can add new flag (job) to get command to enable this feature. 28 | * Any incompatible changes that were introduced with respect to API changes. 29 | * eg: if API is added in odfe 1.13.0 and updated in a backward incompatible way in later releases, this will be addressed in CLI as minor version 30 | 31 | admin will trigger new minor release train whenever new plugin is on-boarded or every 45 days if any commits related to above use case is added. After every minor release, admin will create a new table where contributors can include candidates for next release. 32 | 33 | 34 | 35 | **Q**: **What goes into new major version release train?** 36 | 37 | * Any changes related to framework or addition of new component with respect to odfe-cli. 38 | * eg: Use new rest library (networking layer), framework re-design to ease new plugin on-boarding, user experience changes like update command formats, rename command names, etc. 39 | 40 | admin will trigger new major release train every 120 days if any commits related to above use case is added. After every major release, admin will create a new table related to next release where contributors can include candidates for next release. 41 | 42 | 43 | -------------------------------------------------------------------------------- /entity/ad/ad.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package ad 17 | 18 | import ( 19 | "encoding/json" 20 | "odfe-cli/entity" 21 | ) 22 | 23 | //Feature structure for detector features 24 | type Feature struct { 25 | Name string `json:"feature_name"` 26 | Enabled bool `json:"feature_enabled"` 27 | AggregationQuery json.RawMessage `json:"aggregation_query"` 28 | } 29 | 30 | //Period represents time interval 31 | type Period struct { 32 | Duration int32 `json:"interval"` 33 | Unit string `json:"unit"` 34 | } 35 | 36 | //Interval represent unit of time 37 | type Interval struct { 38 | Period Period `json:"period"` 39 | } 40 | 41 | //CreateDetector represents Detector creation request 42 | type CreateDetector struct { 43 | Name string `json:"name"` 44 | Description string `json:"description,omitempty"` 45 | TimeField string `json:"time_field"` 46 | Index []string `json:"indices"` 47 | Features []Feature `json:"feature_attributes"` 48 | Filter json.RawMessage `json:"filter_query,omitempty"` 49 | Interval Interval `json:"detection_interval"` 50 | Delay Interval `json:"window_delay"` 51 | } 52 | 53 | //FeatureRequest represents feature request 54 | type FeatureRequest struct { 55 | AggregationType []string `json:"aggregation_type"` 56 | Enabled bool `json:"enabled"` 57 | Field []string `json:"field"` 58 | } 59 | 60 | //CreateDetectorRequest represents request for AD 61 | type CreateDetectorRequest struct { 62 | Name string `json:"name"` 63 | Description string `json:"description"` 64 | TimeField string `json:"time_field"` 65 | Index []string `json:"index"` 66 | Features []FeatureRequest `json:"features"` 67 | Filter json.RawMessage `json:"filter,omitempty"` 68 | Interval string `json:"interval"` 69 | Delay string `json:"window_delay"` 70 | Start bool `json:"start"` 71 | PartitionField *string `json:"partition_field"` 72 | } 73 | 74 | //Bool type for must query 75 | type Bool struct { 76 | Must []json.RawMessage `json:"must"` 77 | } 78 | 79 | //Query type to represent query 80 | type Query struct { 81 | Bool Bool `json:"bool"` 82 | } 83 | 84 | //Detector type to map name to ID 85 | type Detector struct { 86 | Name string 87 | ID string 88 | } 89 | 90 | //CreateFailedError structure if create failed 91 | type CreateFailedError struct { 92 | Type string `json:"type"` 93 | Reason string `json:"reason"` 94 | } 95 | 96 | //CreateError Error type in Create Response 97 | type CreateError struct { 98 | Error CreateFailedError `json:"error"` 99 | Status int32 `json:"status"` 100 | } 101 | 102 | //Configuration represents configuration in config file 103 | type Configuration struct { 104 | Profiles []entity.Profile `mapstructure:"profiles"` 105 | } 106 | 107 | //Match specifies name 108 | type Match struct { 109 | Name string `json:"name"` 110 | } 111 | 112 | //SearchQuery contains match names 113 | type SearchQuery struct { 114 | Match Match `json:"match"` 115 | } 116 | 117 | //SearchRequest represents structure for search detectors 118 | type SearchRequest struct { 119 | Query SearchQuery `json:"query"` 120 | } 121 | 122 | //Source contains detectors metadata 123 | type Source struct { 124 | Name string `json:"name"` 125 | } 126 | 127 | //Hit contains search results 128 | type Hit struct { 129 | ID string `json:"_id"` 130 | Source Source `json:"_source"` 131 | } 132 | 133 | //Container represents structure for search response 134 | type Container struct { 135 | Hits []Hit `json:"hits"` 136 | } 137 | 138 | //SearchResponse represents structure for search response 139 | type SearchResponse struct { 140 | Hits Container `json:"hits"` 141 | } 142 | 143 | type Metadata CreateDetector 144 | 145 | type AnomalyDetector struct { 146 | Metadata 147 | SchemaVersion int32 `json:"schema_version"` 148 | LastUpdateTime uint64 `json:"last_update_time"` 149 | } 150 | 151 | //DetectorResponse represents detector's setting 152 | type DetectorResponse struct { 153 | ID string `json:"_id"` 154 | AnomalyDetector AnomalyDetector `json:"anomaly_detector"` 155 | } 156 | 157 | //DetectorOutput represents detector's setting displayed to user 158 | type DetectorOutput struct { 159 | ID string 160 | Name string `json:"name"` 161 | Description string `json:"description"` 162 | TimeField string `json:"time_field"` 163 | Index []string `json:"indices"` 164 | Features []Feature `json:"features"` 165 | Filter json.RawMessage `json:"filter_query"` 166 | Interval string `json:"detection_interval"` 167 | Delay string `json:"window_delay"` 168 | LastUpdatedAt uint64 `json:"last_update_time"` 169 | SchemaVersion int32 `json:"schema_version"` 170 | } 171 | 172 | //UpdateDetectorUserInput represents user's detector input for update 173 | type UpdateDetectorUserInput DetectorOutput 174 | 175 | // UpdateDetector represents detector's settings updated by api 176 | type UpdateDetector CreateDetector 177 | -------------------------------------------------------------------------------- /entity/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package entity 17 | 18 | //Config represents config file structure 19 | type Config struct { 20 | Profiles []Profile `yaml:"profiles"` 21 | } 22 | -------------------------------------------------------------------------------- /entity/es/es.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package es 17 | 18 | //Terms contains fields 19 | type Terms struct { 20 | Field string `json:"field"` 21 | } 22 | 23 | //DistinctGroups contains terms 24 | type DistinctGroups struct { 25 | Term Terms `json:"terms"` 26 | } 27 | 28 | //Aggregate contains list of items 29 | type Aggregate struct { 30 | Group DistinctGroups `json:"items"` 31 | } 32 | 33 | //SearchRequest structure for request 34 | type SearchRequest struct { 35 | Agg Aggregate `json:"aggs"` 36 | Size int32 `json:"size"` 37 | } 38 | 39 | //Bucket represents bucket used by ES for aggregations 40 | type Bucket struct { 41 | Key interface{} `json:"key"` 42 | DocCount int64 `json:"doc_count"` 43 | } 44 | 45 | //Items contains buckets defined by es response 46 | type Items struct { 47 | Buckets []Bucket `json:"buckets"` 48 | } 49 | 50 | //Aggregations contains items defined by es response 51 | type Aggregations struct { 52 | Items Items `json:"items"` 53 | } 54 | 55 | //Response response defined by es response 56 | type Response struct { 57 | Aggregations Aggregations `json:"aggregations"` 58 | } 59 | 60 | //CurlRequest contains parameter to execute REST Action 61 | type CurlRequest struct { 62 | Action string 63 | Path string 64 | QueryParams string 65 | Headers map[string]string 66 | Data []byte 67 | } 68 | 69 | //CurlCommandRequest contains parameter from command 70 | type CurlCommandRequest struct { 71 | Action string 72 | Path string 73 | QueryParams string 74 | Headers string 75 | Data string 76 | Pretty bool 77 | OutputFormat string 78 | OutputFilterPath string 79 | } 80 | -------------------------------------------------------------------------------- /entity/es/request_error.go: -------------------------------------------------------------------------------- 1 | package es 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | ) 9 | 10 | //RequestError contains more information that can be used by client to provide 11 | //better error message 12 | type RequestError struct { 13 | statusCode int 14 | err error 15 | response []byte 16 | } 17 | 18 | //NewRequestError builds RequestError 19 | func NewRequestError(statusCode int, body io.ReadCloser, err error) *RequestError { 20 | return &RequestError{ 21 | statusCode: statusCode, 22 | err: err, 23 | response: getResponseBody(body), 24 | } 25 | } 26 | 27 | //Error inherits error interface to pass as error 28 | func (r *RequestError) Error() string { 29 | return r.err.Error() 30 | } 31 | 32 | //StatusCode to get response's status code 33 | func (r *RequestError) StatusCode() int { 34 | return r.statusCode 35 | } 36 | 37 | //GetResponse to get error response from Elasticsearch 38 | func (r *RequestError) GetResponse() string { 39 | var data map[string]interface{} 40 | if err := json.Unmarshal(r.response, &data); err != nil { 41 | return string(r.response) 42 | } 43 | formattedResponse, _ := json.MarshalIndent(data, "", " ") 44 | return string(formattedResponse) 45 | } 46 | 47 | //getResponseBody to extract response body from elasticsearch server 48 | func getResponseBody(b io.Reader) []byte { 49 | resBytes, err := ioutil.ReadAll(b) 50 | if err != nil { 51 | fmt.Println("failed to read response") 52 | } 53 | return resBytes 54 | } 55 | -------------------------------------------------------------------------------- /entity/knn/knn.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package knn 17 | 18 | //Shards represents number of shards succeeded or failed to warmup 19 | type Shards struct { 20 | Total int `json:"total"` 21 | Successful int `json:"successful"` 22 | Failed int `json:"failed"` 23 | } 24 | 25 | //WarmupAPIResponse warmup api response structure 26 | type WarmupAPIResponse struct { 27 | Shards Shards `json:"_shards"` 28 | } 29 | 30 | //RootCause gives information about type and reason 31 | type RootCause struct { 32 | Type string `json:"type"` 33 | Reason string `json:"reason"` 34 | } 35 | 36 | //Error contains root cause 37 | type Error struct { 38 | RootCause []RootCause `json:"root_cause"` 39 | } 40 | 41 | //ErrorResponse knn request failure error response 42 | type ErrorResponse struct { 43 | KNNError Error `json:"error"` 44 | Status int `json:"status"` 45 | } 46 | -------------------------------------------------------------------------------- /entity/profile.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package entity 17 | 18 | type AWSIAM struct { 19 | ProfileName string `yaml:"profile"` 20 | ServiceName string `yaml:"service"` 21 | } 22 | type Profile struct { 23 | Name string `yaml:"name"` 24 | Endpoint string `yaml:"endpoint"` 25 | UserName string `yaml:"user,omitempty"` 26 | Password string `yaml:"password,omitempty"` 27 | AWS *AWSIAM `yaml:"aws_iam,omitempty"` 28 | MaxRetry *int `yaml:"max_retry,omitempty"` 29 | Timeout *int64 `yaml:"timeout,omitempty"` 30 | } 31 | -------------------------------------------------------------------------------- /gateway/ad/mocks/mock_ad.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: odfe-cli/gateway/ad (interfaces: Gateway) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | context "context" 9 | reflect "reflect" 10 | 11 | gomock "github.com/golang/mock/gomock" 12 | ) 13 | 14 | // MockGateway is a mock of Gateway interface 15 | type MockGateway struct { 16 | ctrl *gomock.Controller 17 | recorder *MockGatewayMockRecorder 18 | } 19 | 20 | // MockGatewayMockRecorder is the mock recorder for MockGateway 21 | type MockGatewayMockRecorder struct { 22 | mock *MockGateway 23 | } 24 | 25 | // NewMockGateway creates a new mock instance 26 | func NewMockGateway(ctrl *gomock.Controller) *MockGateway { 27 | mock := &MockGateway{ctrl: ctrl} 28 | mock.recorder = &MockGatewayMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use 33 | func (m *MockGateway) EXPECT() *MockGatewayMockRecorder { 34 | return m.recorder 35 | } 36 | 37 | // CreateDetector mocks base method 38 | func (m *MockGateway) CreateDetector(arg0 context.Context, arg1 interface{}) ([]byte, error) { 39 | m.ctrl.T.Helper() 40 | ret := m.ctrl.Call(m, "CreateDetector", arg0, arg1) 41 | ret0, _ := ret[0].([]byte) 42 | ret1, _ := ret[1].(error) 43 | return ret0, ret1 44 | } 45 | 46 | // CreateDetector indicates an expected call of CreateDetector 47 | func (mr *MockGatewayMockRecorder) CreateDetector(arg0, arg1 interface{}) *gomock.Call { 48 | mr.mock.ctrl.T.Helper() 49 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateDetector", reflect.TypeOf((*MockGateway)(nil).CreateDetector), arg0, arg1) 50 | } 51 | 52 | // DeleteDetector mocks base method 53 | func (m *MockGateway) DeleteDetector(arg0 context.Context, arg1 string) error { 54 | m.ctrl.T.Helper() 55 | ret := m.ctrl.Call(m, "DeleteDetector", arg0, arg1) 56 | ret0, _ := ret[0].(error) 57 | return ret0 58 | } 59 | 60 | // DeleteDetector indicates an expected call of DeleteDetector 61 | func (mr *MockGatewayMockRecorder) DeleteDetector(arg0, arg1 interface{}) *gomock.Call { 62 | mr.mock.ctrl.T.Helper() 63 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteDetector", reflect.TypeOf((*MockGateway)(nil).DeleteDetector), arg0, arg1) 64 | } 65 | 66 | // GetDetector mocks base method 67 | func (m *MockGateway) GetDetector(arg0 context.Context, arg1 string) ([]byte, error) { 68 | m.ctrl.T.Helper() 69 | ret := m.ctrl.Call(m, "GetDetector", arg0, arg1) 70 | ret0, _ := ret[0].([]byte) 71 | ret1, _ := ret[1].(error) 72 | return ret0, ret1 73 | } 74 | 75 | // GetDetector indicates an expected call of GetDetector 76 | func (mr *MockGatewayMockRecorder) GetDetector(arg0, arg1 interface{}) *gomock.Call { 77 | mr.mock.ctrl.T.Helper() 78 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDetector", reflect.TypeOf((*MockGateway)(nil).GetDetector), arg0, arg1) 79 | } 80 | 81 | // SearchDetector mocks base method 82 | func (m *MockGateway) SearchDetector(arg0 context.Context, arg1 interface{}) ([]byte, error) { 83 | m.ctrl.T.Helper() 84 | ret := m.ctrl.Call(m, "SearchDetector", arg0, arg1) 85 | ret0, _ := ret[0].([]byte) 86 | ret1, _ := ret[1].(error) 87 | return ret0, ret1 88 | } 89 | 90 | // SearchDetector indicates an expected call of SearchDetector 91 | func (mr *MockGatewayMockRecorder) SearchDetector(arg0, arg1 interface{}) *gomock.Call { 92 | mr.mock.ctrl.T.Helper() 93 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchDetector", reflect.TypeOf((*MockGateway)(nil).SearchDetector), arg0, arg1) 94 | } 95 | 96 | // StartDetector mocks base method 97 | func (m *MockGateway) StartDetector(arg0 context.Context, arg1 string) error { 98 | m.ctrl.T.Helper() 99 | ret := m.ctrl.Call(m, "StartDetector", arg0, arg1) 100 | ret0, _ := ret[0].(error) 101 | return ret0 102 | } 103 | 104 | // StartDetector indicates an expected call of StartDetector 105 | func (mr *MockGatewayMockRecorder) StartDetector(arg0, arg1 interface{}) *gomock.Call { 106 | mr.mock.ctrl.T.Helper() 107 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartDetector", reflect.TypeOf((*MockGateway)(nil).StartDetector), arg0, arg1) 108 | } 109 | 110 | // StopDetector mocks base method 111 | func (m *MockGateway) StopDetector(arg0 context.Context, arg1 string) (*string, error) { 112 | m.ctrl.T.Helper() 113 | ret := m.ctrl.Call(m, "StopDetector", arg0, arg1) 114 | ret0, _ := ret[0].(*string) 115 | ret1, _ := ret[1].(error) 116 | return ret0, ret1 117 | } 118 | 119 | // StopDetector indicates an expected call of StopDetector 120 | func (mr *MockGatewayMockRecorder) StopDetector(arg0, arg1 interface{}) *gomock.Call { 121 | mr.mock.ctrl.T.Helper() 122 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StopDetector", reflect.TypeOf((*MockGateway)(nil).StopDetector), arg0, arg1) 123 | } 124 | 125 | // UpdateDetector mocks base method 126 | func (m *MockGateway) UpdateDetector(arg0 context.Context, arg1 string, arg2 interface{}) error { 127 | m.ctrl.T.Helper() 128 | ret := m.ctrl.Call(m, "UpdateDetector", arg0, arg1, arg2) 129 | ret0, _ := ret[0].(error) 130 | return ret0 131 | } 132 | 133 | // UpdateDetector indicates an expected call of UpdateDetector 134 | func (mr *MockGatewayMockRecorder) UpdateDetector(arg0, arg1, arg2 interface{}) *gomock.Call { 135 | mr.mock.ctrl.T.Helper() 136 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDetector", reflect.TypeOf((*MockGateway)(nil).UpdateDetector), arg0, arg1, arg2) 137 | } 138 | -------------------------------------------------------------------------------- /gateway/ad/testdata/create_result.json: -------------------------------------------------------------------------------- 1 | { 2 | "_id" : "m4ccEnIBTXsGi3mvMt9p", 3 | "_version" : 1, 4 | "_seq_no" : 3, 5 | "_primary_term" : 1, 6 | "anomaly_detector" : { 7 | "name" : "test-detector", 8 | "description" : "Test detector", 9 | "time_field" : "timestamp", 10 | "indices" : [ 11 | "order*" 12 | ], 13 | "filter_query" : { 14 | "bool" : { 15 | "filter" : [ 16 | { 17 | "exists" : { 18 | "field" : "value", 19 | "boost" : 1.0 20 | } 21 | } 22 | ], 23 | "adjust_pure_negative" : true, 24 | "boost" : 1.0 25 | } 26 | }, 27 | "detection_interval" : { 28 | "period" : { 29 | "interval" : 1, 30 | "unit" : "Minutes" 31 | } 32 | }, 33 | "window_delay" : { 34 | "period" : { 35 | "interval" : 1, 36 | "unit" : "Minutes" 37 | } 38 | }, 39 | "schema_version" : 0, 40 | "feature_attributes" : [ 41 | { 42 | "feature_id" : "mYccEnIBTXsGi3mvMd8_", 43 | "feature_name" : "total_order", 44 | "feature_enabled" : true, 45 | "aggregation_query" : { 46 | "total_order" : { 47 | "sum" : { 48 | "field" : "value" 49 | } 50 | } 51 | } 52 | } 53 | ] 54 | } 55 | } -------------------------------------------------------------------------------- /gateway/ad/testdata/get_result.json: -------------------------------------------------------------------------------- 1 | { 2 | "_id" : "m4ccEnIBTXsGi3mvMt9p", 3 | "_version" : 1, 4 | "_primary_term" : 1, 5 | "_seq_no" : 3, 6 | "anomaly_detector" : { 7 | "name" : "test-detector", 8 | "description" : "Test detector", 9 | "time_field" : "timestamp", 10 | "indices" : [ 11 | "order*" 12 | ], 13 | "filter_query" : { 14 | "bool" : { 15 | "filter" : [ 16 | { 17 | "exists" : { 18 | "field" : "value", 19 | "boost" : 1.0 20 | } 21 | } 22 | ], 23 | "adjust_pure_negative" : true, 24 | "boost" : 1.0 25 | } 26 | }, 27 | "detection_interval" : { 28 | "period" : { 29 | "interval" : 1, 30 | "unit" : "Minutes" 31 | } 32 | }, 33 | "window_delay" : { 34 | "period" : { 35 | "interval" : 1, 36 | "unit" : "Minutes" 37 | } 38 | }, 39 | "schema_version" : 0, 40 | "feature_attributes" : [ 41 | { 42 | "feature_id" : "mYccEnIBTXsGi3mvMd8_", 43 | "feature_name" : "total_order", 44 | "feature_enabled" : true, 45 | "aggregation_query" : { 46 | "total_order" : { 47 | "sum" : { 48 | "field" : "value" 49 | } 50 | } 51 | } 52 | } 53 | ], 54 | "last_update_time" : 1589441737319 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /gateway/ad/testdata/search_result.json: -------------------------------------------------------------------------------- 1 | { 2 | "took": 13, 3 | "timed_out": false, 4 | "_shards": { 5 | "total": 5, 6 | "successful": 5, 7 | "skipped": 0, 8 | "failed": 0 9 | }, 10 | "hits": { 11 | "total": { 12 | "value": 994, 13 | "relation": "eq" 14 | }, 15 | "max_score": 3.5410638, 16 | "hits": [ 17 | { 18 | "_index": ".opendistro-anomaly-detectors", 19 | "_type": "_doc", 20 | "_id": "m4ccEnIBTXsGi3mvMt9p", 21 | "_version": 2, 22 | "_seq_no": 221, 23 | "_primary_term": 1, 24 | "_score": 3.5410638, 25 | "_source": { 26 | "name": "test-detector", 27 | "description": "Test detector", 28 | "time_field": "timestamp", 29 | "indices": [ 30 | "order*" 31 | ], 32 | "filter_query": { 33 | "bool": { 34 | "filter": [ 35 | { 36 | "exists": { 37 | "field": "value", 38 | "boost": 1 39 | } 40 | } 41 | ], 42 | "adjust_pure_negative": true, 43 | "boost": 1 44 | } 45 | }, 46 | "detection_interval": { 47 | "period": { 48 | "interval": 10, 49 | "unit": "Minutes" 50 | } 51 | }, 52 | "window_delay": { 53 | "period": { 54 | "interval": 1, 55 | "unit": "Minutes" 56 | } 57 | }, 58 | "schema_version": 0, 59 | "feature_attributes": [ 60 | { 61 | "feature_id": "xxokEnIBcpeWMD987A1X", 62 | "feature_name": "total_order", 63 | "feature_enabled": true, 64 | "aggregation_query": { 65 | "total_order": { 66 | "sum": { 67 | "field": "value" 68 | } 69 | } 70 | } 71 | } 72 | ], 73 | "last_update_time": 1589442309241 74 | } 75 | } 76 | ] 77 | } 78 | } -------------------------------------------------------------------------------- /gateway/aws/signer/aws.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package signer 17 | 18 | import ( 19 | "bytes" 20 | "errors" 21 | "odfe-cli/entity" 22 | "time" 23 | 24 | "github.com/aws/aws-sdk-go/aws/credentials" 25 | "github.com/aws/aws-sdk-go/aws/session" 26 | v4 "github.com/aws/aws-sdk-go/aws/signer/v4" 27 | "github.com/hashicorp/go-retryablehttp" 28 | ) 29 | 30 | func GetV4Signer(credentials *credentials.Credentials) *v4.Signer { 31 | return v4.NewSigner(credentials) 32 | } 33 | func sign(req *retryablehttp.Request, region *string, serviceName string, signer *v4.Signer) error { 34 | bodyBytes, err := req.BodyBytes() 35 | if err != nil { 36 | return err 37 | } 38 | if region == nil || len(*region) == 0 { 39 | return errors.New("aws region is not found. Either set 'AWS_REGION' or add this information during aws profile creation step") 40 | } 41 | // Sign the request 42 | _, err = signer.Sign(req.Request, bytes.NewReader(bodyBytes), serviceName, *region, time.Now()) 43 | return err 44 | } 45 | 46 | //SignRequest signs the request using SigV4 47 | func SignRequest(req *retryablehttp.Request, awsProfile entity.AWSIAM, getSigner func(*credentials.Credentials) *v4.Signer) error { 48 | awsSession, err := session.NewSessionWithOptions(session.Options{ 49 | Profile: awsProfile.ProfileName, 50 | SharedConfigState: session.SharedConfigEnable, 51 | }) 52 | 53 | if err != nil { 54 | return err 55 | } 56 | signer := getSigner(awsSession.Config.Credentials) 57 | return sign(req, awsSession.Config.Region, awsProfile.ServiceName, signer) 58 | } 59 | -------------------------------------------------------------------------------- /gateway/aws/signer/aws_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package signer 17 | 18 | import ( 19 | "net/http" 20 | "odfe-cli/entity" 21 | "os" 22 | "testing" 23 | 24 | "github.com/aws/aws-sdk-go/aws/credentials" 25 | v4 "github.com/aws/aws-sdk-go/aws/signer/v4" 26 | "github.com/hashicorp/go-retryablehttp" 27 | "github.com/stretchr/testify/assert" 28 | ) 29 | 30 | func buildSigner() *v4.Signer { 31 | return &v4.Signer{ 32 | Credentials: credentials.NewStaticCredentials("AKID", "SECRET", "SESSION"), 33 | } 34 | } 35 | 36 | func TestV4Signer(t *testing.T) { 37 | t.Run("sign request success", func(t *testing.T) { 38 | req, _ := retryablehttp.NewRequest(http.MethodGet, "https://localhost:9200", nil) 39 | region := os.Getenv("AWS_REGION") 40 | os.Setenv("AWS_REGION", "us-west-2") 41 | defer func() { 42 | os.Setenv("AWS_REGION", region) 43 | }() 44 | err := SignRequest(req, entity.AWSIAM{ 45 | ProfileName: "test1", 46 | ServiceName: "es", 47 | }, func(c *credentials.Credentials) *v4.Signer { 48 | return buildSigner() 49 | }) 50 | assert.NoError(t, err) 51 | q := req.Header 52 | assert.NotEmpty(t, q.Get("Authorization")) 53 | assert.NotEmpty(t, q.Get("X-Amz-Date")) 54 | }) 55 | t.Run("sign request failed due to no region found", func(t *testing.T) { 56 | req, _ := retryablehttp.NewRequest(http.MethodGet, "https://localhost:9200", nil) 57 | region := os.Getenv("AWS_REGION") 58 | os.Setenv("AWS_REGION", "") 59 | defer func() { 60 | os.Setenv("AWS_REGION", region) 61 | }() 62 | err := SignRequest(req, entity.AWSIAM{ 63 | ProfileName: "test1", 64 | ServiceName: "es", 65 | }, func(c *credentials.Credentials) *v4.Signer { 66 | return buildSigner() 67 | }) 68 | assert.EqualErrorf( 69 | t, err, "aws region is not found. Either set 'AWS_REGION' or add this information during aws profile creation step", "unexpected error") 70 | }) 71 | } 72 | -------------------------------------------------------------------------------- /gateway/es/es.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package es 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "net/http" 22 | "net/url" 23 | "odfe-cli/client" 24 | "odfe-cli/entity" 25 | "odfe-cli/entity/es" 26 | gw "odfe-cli/gateway" 27 | ) 28 | 29 | const search = "_search" 30 | 31 | //go:generate go run -mod=mod github.com/golang/mock/mockgen -destination=mocks/mock_es.go -package=mocks . Gateway 32 | 33 | //Gateway interface to call ES 34 | type Gateway interface { 35 | SearchDistinctValues(ctx context.Context, index string, field string) ([]byte, error) 36 | Curl(ctx context.Context, request es.CurlRequest) ([]byte, error) 37 | } 38 | 39 | type gateway struct { 40 | gw.HTTPGateway 41 | } 42 | 43 | // New returns new Gateway instance 44 | func New(c *client.Client, p *entity.Profile) Gateway { 45 | return &gateway{ 46 | *gw.NewHTTPGateway(c, p), 47 | } 48 | } 49 | func buildPayload(field string) *es.SearchRequest { 50 | return &es.SearchRequest{ 51 | Size: 0, // This will skip data in the response 52 | Agg: es.Aggregate{ 53 | Group: es.DistinctGroups{ 54 | Term: es.Terms{ 55 | Field: field, 56 | }, 57 | }, 58 | }, 59 | } 60 | } 61 | 62 | func (g *gateway) buildSearchURL(index string) (*url.URL, error) { 63 | endpoint, err := gw.GetValidEndpoint(g.Profile) 64 | if err != nil { 65 | return nil, err 66 | } 67 | endpoint.Path = fmt.Sprintf("%s/%s", index, search) 68 | return endpoint, nil 69 | } 70 | 71 | //SearchDistinctValues gets distinct values on index for given field 72 | func (g *gateway) SearchDistinctValues(ctx context.Context, index string, field string) ([]byte, error) { 73 | searchURL, err := g.buildSearchURL(index) 74 | if err != nil { 75 | return nil, err 76 | } 77 | searchRequest, err := g.BuildRequest(ctx, http.MethodGet, buildPayload(field), searchURL.String(), gw.GetDefaultHeaders()) 78 | if err != nil { 79 | return nil, err 80 | } 81 | response, err := g.Call(searchRequest, http.StatusOK) 82 | if err != nil { 83 | return nil, err 84 | } 85 | return response, nil 86 | } 87 | 88 | //Curl executes REST request based on request parameters 89 | func (g *gateway) Curl(ctx context.Context, request es.CurlRequest) ([]byte, error) { 90 | 91 | requestURL, err := g.buildURL(request) 92 | if err != nil { 93 | return nil, err 94 | } 95 | //append request headers with gateway default headers 96 | headers := gw.GetDefaultHeaders() 97 | for k, v := range request.Headers { 98 | headers[k] = v 99 | } 100 | curlRequest, err := g.BuildCurlRequest(ctx, request.Action, request.Data, requestURL.String(), headers) 101 | if err != nil { 102 | return nil, err 103 | } 104 | response, err := g.Execute(curlRequest) 105 | if err != nil { 106 | return nil, err 107 | } 108 | return response, nil 109 | } 110 | 111 | func (g *gateway) buildURL(request es.CurlRequest) (*url.URL, error) { 112 | endpoint, err := gw.GetValidEndpoint(g.Profile) 113 | if err != nil { 114 | return nil, err 115 | } 116 | endpoint.Path = request.Path 117 | endpoint.RawQuery = request.QueryParams 118 | return endpoint, nil 119 | } 120 | -------------------------------------------------------------------------------- /gateway/es/mocks/mock_es.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: odfe-cli/gateway/es (interfaces: Gateway) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | context "context" 9 | es "odfe-cli/entity/es" 10 | reflect "reflect" 11 | 12 | gomock "github.com/golang/mock/gomock" 13 | ) 14 | 15 | // MockGateway is a mock of Gateway interface 16 | type MockGateway struct { 17 | ctrl *gomock.Controller 18 | recorder *MockGatewayMockRecorder 19 | } 20 | 21 | // MockGatewayMockRecorder is the mock recorder for MockGateway 22 | type MockGatewayMockRecorder struct { 23 | mock *MockGateway 24 | } 25 | 26 | // NewMockGateway creates a new mock instance 27 | func NewMockGateway(ctrl *gomock.Controller) *MockGateway { 28 | mock := &MockGateway{ctrl: ctrl} 29 | mock.recorder = &MockGatewayMockRecorder{mock} 30 | return mock 31 | } 32 | 33 | // EXPECT returns an object that allows the caller to indicate expected use 34 | func (m *MockGateway) EXPECT() *MockGatewayMockRecorder { 35 | return m.recorder 36 | } 37 | 38 | // Curl mocks base method 39 | func (m *MockGateway) Curl(arg0 context.Context, arg1 es.CurlRequest) ([]byte, error) { 40 | m.ctrl.T.Helper() 41 | ret := m.ctrl.Call(m, "Curl", arg0, arg1) 42 | ret0, _ := ret[0].([]byte) 43 | ret1, _ := ret[1].(error) 44 | return ret0, ret1 45 | } 46 | 47 | // Curl indicates an expected call of Curl 48 | func (mr *MockGatewayMockRecorder) Curl(arg0, arg1 interface{}) *gomock.Call { 49 | mr.mock.ctrl.T.Helper() 50 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Curl", reflect.TypeOf((*MockGateway)(nil).Curl), arg0, arg1) 51 | } 52 | 53 | // SearchDistinctValues mocks base method 54 | func (m *MockGateway) SearchDistinctValues(arg0 context.Context, arg1, arg2 string) ([]byte, error) { 55 | m.ctrl.T.Helper() 56 | ret := m.ctrl.Call(m, "SearchDistinctValues", arg0, arg1, arg2) 57 | ret0, _ := ret[0].([]byte) 58 | ret1, _ := ret[1].(error) 59 | return ret0, ret1 60 | } 61 | 62 | // SearchDistinctValues indicates an expected call of SearchDistinctValues 63 | func (mr *MockGatewayMockRecorder) SearchDistinctValues(arg0, arg1, arg2 interface{}) *gomock.Call { 64 | mr.mock.ctrl.T.Helper() 65 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SearchDistinctValues", reflect.TypeOf((*MockGateway)(nil).SearchDistinctValues), arg0, arg1, arg2) 66 | } 67 | -------------------------------------------------------------------------------- /gateway/es/testdata/search_result.json: -------------------------------------------------------------------------------- 1 | { 2 | "took": 80, 3 | "timed_out": false, 4 | "_shards": { 5 | "total": 5, 6 | "successful": 5, 7 | "skipped": 0, 8 | "failed": 0 9 | }, 10 | "hits": { 11 | "total": 14, 12 | "max_score": 0, 13 | "hits": [] 14 | }, 15 | "aggregations": { 16 | "items": { 17 | "doc_count_error_upper_bound": 0, 18 | "sum_other_doc_count": 0, 19 | "buckets": [ 20 | { 21 | "key": "Packaged Foods", 22 | "doc_count": 4 23 | }, 24 | { 25 | "key": "Dairy", 26 | "doc_count": 3 27 | }, 28 | { 29 | "key": "Meat and Seafood", 30 | "doc_count": 2 31 | } 32 | ] 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /gateway/gateway.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package gateway 17 | 18 | import ( 19 | "context" 20 | "encoding/json" 21 | "errors" 22 | "fmt" 23 | "io/ioutil" 24 | "net/http" 25 | "net/url" 26 | "odfe-cli/client" 27 | "odfe-cli/entity" 28 | "odfe-cli/entity/es" 29 | "odfe-cli/gateway/aws/signer" 30 | "os" 31 | "strconv" 32 | "time" 33 | 34 | "github.com/hashicorp/go-retryablehttp" 35 | ) 36 | 37 | //HTTPGateway type for gateway client 38 | type HTTPGateway struct { 39 | Client *client.Client 40 | Profile *entity.Profile 41 | } 42 | 43 | //GetDefaultHeaders returns common headers 44 | func GetDefaultHeaders() map[string]string { 45 | return map[string]string{ 46 | "content-type": "application/json", 47 | } 48 | } 49 | 50 | //NewHTTPGateway creates new HTTPGateway instance 51 | func NewHTTPGateway(c *client.Client, p *entity.Profile) *HTTPGateway { 52 | // set max retry if provided by command 53 | if p.MaxRetry != nil { 54 | c.HTTPClient.RetryMax = *p.MaxRetry 55 | } 56 | //override with environment variable if exists 57 | if val, ok := overrideValue(p, "ODFE_MAX_RETRY"); ok { 58 | c.HTTPClient.RetryMax = *val 59 | } 60 | 61 | // set connection timeout if provided by command 62 | if p.Timeout != nil { 63 | c.HTTPClient.HTTPClient.Timeout = time.Duration(*p.Timeout) * time.Second 64 | } 65 | //override with environment variable if exists 66 | if duration, ok := overrideValue(p, "ODFE_TIMEOUT"); ok { 67 | c.HTTPClient.HTTPClient.Timeout = time.Duration(*duration) * time.Second 68 | } 69 | return &HTTPGateway{ 70 | Client: c, 71 | Profile: p, 72 | } 73 | } 74 | 75 | func overrideValue(p *entity.Profile, envVariable string) (*int, bool) { 76 | if val, ok := os.LookupEnv(envVariable); ok { 77 | //ignore error from non positive number 78 | if attempt, err := strconv.Atoi(val); err == nil { 79 | return &attempt, true 80 | } 81 | } 82 | return nil, false 83 | } 84 | 85 | //isValidResponse checks whether the response is valid or not by checking the status code 86 | func (g *HTTPGateway) isValidResponse(response *http.Response) error { 87 | if response == nil { 88 | return errors.New("response is nil") 89 | } 90 | // client error if 400 <= status code < 500 91 | if response.StatusCode >= http.StatusBadRequest && response.StatusCode < http.StatusInternalServerError { 92 | 93 | return es.NewRequestError( 94 | response.StatusCode, 95 | response.Body, 96 | fmt.Errorf("%d Client Error: %s for url: %s", response.StatusCode, response.Status, response.Request.URL)) 97 | } 98 | // server error if status code >= 500 99 | if response.StatusCode >= http.StatusInternalServerError { 100 | 101 | return es.NewRequestError( 102 | response.StatusCode, 103 | response.Body, 104 | fmt.Errorf("%d Server Error: %s for url: %s", response.StatusCode, response.Status, response.Request.URL)) 105 | } 106 | return nil 107 | } 108 | 109 | //Execute calls request using http and check if status code is ok or not 110 | func (g *HTTPGateway) Execute(req *retryablehttp.Request) ([]byte, error) { 111 | if g.Profile.AWS != nil { 112 | //sign request 113 | if err := signer.SignRequest(req, *g.Profile.AWS, signer.GetV4Signer); err != nil { 114 | return nil, err 115 | } 116 | } 117 | response, err := g.Client.HTTPClient.Do(req) 118 | if err != nil { 119 | return nil, err 120 | } 121 | defer func() { 122 | err := response.Body.Close() 123 | if err != nil { 124 | return 125 | } 126 | }() 127 | if err = g.isValidResponse(response); err != nil { 128 | return nil, err 129 | } 130 | return ioutil.ReadAll(response.Body) 131 | } 132 | 133 | //Call calls request using http and return error if status code is not expected 134 | func (g *HTTPGateway) Call(req *retryablehttp.Request, statusCode int) ([]byte, error) { 135 | resBytes, err := g.Execute(req) 136 | if err == nil { 137 | return resBytes, nil 138 | } 139 | r, ok := err.(*es.RequestError) 140 | if !ok { 141 | return nil, err 142 | } 143 | if r.StatusCode() != statusCode { 144 | return nil, fmt.Errorf(r.GetResponse()) 145 | } 146 | return nil, err 147 | 148 | } 149 | 150 | //BuildRequest builds request based on method and appends payload for given url with headers 151 | // TODO: Deprecate this method by replace this with BuildCurlRequest 152 | func (g *HTTPGateway) BuildRequest(ctx context.Context, method string, payload interface{}, url string, headers map[string]string) (*retryablehttp.Request, error) { 153 | reqBytes, err := json.Marshal(payload) 154 | if err != nil { 155 | return nil, err 156 | } 157 | return g.BuildCurlRequest(ctx, method, reqBytes, url, headers) 158 | } 159 | 160 | //BuildCurlRequest builds request based on method and add payload (in byte) 161 | func (g *HTTPGateway) BuildCurlRequest(ctx context.Context, method string, payload []byte, url string, headers map[string]string) (*retryablehttp.Request, error) { 162 | r, err := retryablehttp.NewRequest(method, url, payload) 163 | if err != nil { 164 | return nil, err 165 | } 166 | req := r.WithContext(ctx) 167 | if len(g.Profile.UserName) != 0 { 168 | req.SetBasicAuth(g.Profile.UserName, g.Profile.Password) 169 | } 170 | if len(headers) == 0 { 171 | return req, nil 172 | } 173 | for key, value := range headers { 174 | req.Header.Set(key, value) 175 | } 176 | return req, nil 177 | } 178 | 179 | //GetValidEndpoint get url based on user config 180 | func GetValidEndpoint(profile *entity.Profile) (*url.URL, error) { 181 | u, err := url.ParseRequestURI(profile.Endpoint) 182 | if err != nil { 183 | return nil, fmt.Errorf("invalid endpoint: %v due to %v", profile.Endpoint, err) 184 | } 185 | return u, nil 186 | } 187 | -------------------------------------------------------------------------------- /gateway/gateway_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package gateway 17 | 18 | import ( 19 | "odfe-cli/client/mocks" 20 | "odfe-cli/entity" 21 | "os" 22 | "testing" 23 | "time" 24 | 25 | "github.com/stretchr/testify/assert" 26 | ) 27 | 28 | func TestGetValidEndpoint(t *testing.T) { 29 | t.Run("valid endpoint", func(t *testing.T) { 30 | 31 | profile := entity.Profile{ 32 | Name: "test1", 33 | Endpoint: "https://localhost:9200", 34 | UserName: "foo", 35 | Password: "bar", 36 | } 37 | url, err := GetValidEndpoint(&profile) 38 | assert.NoError(t, err) 39 | assert.EqualValues(t, "https://localhost:9200", url.String()) 40 | }) 41 | t.Run("empty endpoint", func(t *testing.T) { 42 | profile := entity.Profile{ 43 | Name: "test1", 44 | Endpoint: "", 45 | UserName: "foo", 46 | Password: "bar", 47 | } 48 | _, err := GetValidEndpoint(&profile) 49 | assert.EqualErrorf(t, err, "invalid endpoint: due to parse \"\": empty url", "failed to get expected error") 50 | }) 51 | } 52 | 53 | func TestGatewayRetryVal(t *testing.T) { 54 | t.Run("default retry max value", func(t *testing.T) { 55 | profile := entity.Profile{ 56 | Name: "test1", 57 | Endpoint: "https://localhost:9200", 58 | } 59 | testClient := mocks.NewTestClient(nil) 60 | NewHTTPGateway(testClient, &profile) 61 | assert.EqualValues(t, 4, testClient.HTTPClient.RetryMax) 62 | }) 63 | t.Run("profile retry max value", func(t *testing.T) { 64 | valAttempt := 2 65 | profile := entity.Profile{ 66 | Name: "test1", 67 | Endpoint: "https://localhost:9200", 68 | MaxRetry: &valAttempt, 69 | } 70 | testClient := mocks.NewTestClient(nil) 71 | NewHTTPGateway(testClient, &profile) 72 | assert.EqualValues(t, valAttempt, testClient.HTTPClient.RetryMax) 73 | }) 74 | 75 | t.Run("override from os variable", func(t *testing.T) { 76 | val := os.Getenv("ODFE_MAX_RETRY") 77 | defer func() { 78 | assert.NoError(t, os.Setenv("ODFE_MAX_RETRY", val)) 79 | }() 80 | os.Setenv("ODFE_MAX_RETRY", "10") 81 | valAttempt := 2 82 | profile := entity.Profile{ 83 | Name: "test1", 84 | Endpoint: "https://localhost:9200", 85 | MaxRetry: &valAttempt, 86 | } 87 | testClient := mocks.NewTestClient(nil) 88 | NewHTTPGateway(testClient, &profile) 89 | assert.EqualValues(t, 10, testClient.HTTPClient.RetryMax) 90 | }) 91 | } 92 | 93 | func TestGatewayConnectionTimeout(t *testing.T) { 94 | t.Run("default timeout", func(t *testing.T) { 95 | profile := entity.Profile{ 96 | Name: "test1", 97 | Endpoint: "https://localhost:9200", 98 | } 99 | testClient := mocks.NewTestClient(nil) 100 | NewHTTPGateway(testClient, &profile) 101 | assert.EqualValues(t, 10*time.Second, testClient.HTTPClient.HTTPClient.Timeout) 102 | }) 103 | t.Run("configure profile timeout", func(t *testing.T) { 104 | timeout := int64(60) 105 | profile := entity.Profile{ 106 | Name: "test1", 107 | Endpoint: "https://localhost:9200", 108 | Timeout: &timeout, 109 | } 110 | testClient := mocks.NewTestClient(nil) 111 | NewHTTPGateway(testClient, &profile) 112 | assert.EqualValues(t, time.Duration(timeout)*time.Second, testClient.HTTPClient.HTTPClient.Timeout) 113 | }) 114 | 115 | t.Run("override from os variable", func(t *testing.T) { 116 | val := os.Getenv("ODFE_TIMEOUT") 117 | defer func() { 118 | assert.NoError(t, os.Setenv("ODFE_TIMEOUT", val)) 119 | }() 120 | os.Setenv("ODFE_TIMEOUT", "5") 121 | timeout := int64(60) 122 | profile := entity.Profile{ 123 | Name: "test1", 124 | Endpoint: "https://localhost:9200", 125 | Timeout: &timeout, 126 | } 127 | testClient := mocks.NewTestClient(nil) 128 | NewHTTPGateway(testClient, &profile) 129 | assert.EqualValues(t, 5*time.Second, testClient.HTTPClient.HTTPClient.Timeout) 130 | }) 131 | } 132 | -------------------------------------------------------------------------------- /gateway/knn/knn.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package knn 17 | 18 | import ( 19 | "context" 20 | "encoding/json" 21 | "errors" 22 | "fmt" 23 | "net/http" 24 | "net/url" 25 | "odfe-cli/client" 26 | "odfe-cli/entity" 27 | "odfe-cli/entity/knn" 28 | gw "odfe-cli/gateway" 29 | ) 30 | 31 | const ( 32 | baseURL = "_opendistro/_knn" 33 | statsURL = baseURL + "/stats" 34 | nodeStatsURLTemplate = baseURL + "/%s/stats/%s" 35 | warmupIndicesURLTemplate = baseURL + "/warmup/%s" 36 | ) 37 | 38 | //go:generate go run -mod=mod github.com/golang/mock/mockgen -destination=mocks/mock_knn.go -package=mocks . Gateway 39 | 40 | // Gateway interface to k-NN Plugin 41 | type Gateway interface { 42 | GetStatistics(ctx context.Context, nodes string, names string) ([]byte, error) 43 | WarmupIndices(ctx context.Context, indices string) ([]byte, error) 44 | } 45 | 46 | type gateway struct { 47 | gw.HTTPGateway 48 | } 49 | 50 | // New creates new Gateway instance 51 | func New(c *client.Client, p *entity.Profile) Gateway { 52 | return &gateway{*gw.NewHTTPGateway(c, p)} 53 | } 54 | 55 | //buildStatsURL to construct url for stats 56 | func (g *gateway) buildStatsURL(nodes string, names string) (*url.URL, error) { 57 | endpoint, err := gw.GetValidEndpoint(g.Profile) 58 | if err != nil { 59 | return nil, err 60 | } 61 | path := statsURL 62 | // if either of filter parameters are non-empty, use filter template 63 | if nodes != "" || names != "" { 64 | path = fmt.Sprintf(nodeStatsURLTemplate, nodes, names) 65 | } 66 | endpoint.Path = path 67 | return endpoint, nil 68 | } 69 | 70 | //buildWarmupURL to construct url for warming up indices 71 | func (g *gateway) buildWarmupURL(indices string) (*url.URL, error) { 72 | endpoint, err := gw.GetValidEndpoint(g.Profile) 73 | if err != nil { 74 | return nil, err 75 | } 76 | endpoint.Path = fmt.Sprintf(warmupIndicesURLTemplate, indices) 77 | return endpoint, nil 78 | } 79 | 80 | /*GetStatistics provides information about the current status of the KNN Plugin. 81 | GET /_opendistro/_knn/stats 82 | { 83 | "_nodes" : { 84 | "total" : 1, 85 | "successful" : 1, 86 | "failed" : 0 87 | }, 88 | "cluster_name" : "_run", 89 | "circuit_breaker_triggered" : false, 90 | "nodes" : { 91 | "HYMrXXsBSamUkcAjhjeN0w" : { 92 | "eviction_count" : 0, 93 | "miss_count" : 1, 94 | "graph_memory_usage" : 1, 95 | "graph_memory_usage_percentage" : 3.68, 96 | "graph_index_requests" : 7, 97 | "graph_index_errors" : 1, 98 | "knn_query_requests" : 4, 99 | "graph_query_requests" : 30, 100 | "graph_query_errors" : 15, 101 | "indices_in_cache" : { 102 | "myindex" : { 103 | "graph_memory_usage" : 2, 104 | "graph_memory_usage_percentage" : 3.68, 105 | "graph_count" : 2 106 | } 107 | }, 108 | "cache_capacity_reached" : false, 109 | "load_exception_count" : 0, 110 | "hit_count" : 0, 111 | "load_success_count" : 1, 112 | "total_load_time" : 2878745, 113 | "script_compilations" : 1, 114 | "script_compilation_errors" : 0, 115 | "script_query_requests" : 534, 116 | "script_query_errors" : 0 117 | } 118 | } 119 | } 120 | To filter stats query by nodeID and statName: 121 | GET /_opendistro/_knn/nodeId1,nodeId2/stats/statName1,statName2 122 | */ 123 | func (g gateway) GetStatistics(ctx context.Context, nodes string, names string) ([]byte, error) { 124 | statsURL, err := g.buildStatsURL(nodes, names) 125 | if err != nil { 126 | return nil, err 127 | } 128 | request, err := g.BuildRequest(ctx, http.MethodGet, "", statsURL.String(), gw.GetDefaultHeaders()) 129 | if err != nil { 130 | return nil, err 131 | } 132 | response, err := g.Call(request, http.StatusOK) 133 | if err != nil { 134 | return nil, processKNNError(err) 135 | } 136 | return response, nil 137 | } 138 | 139 | func processKNNError(err error) error { 140 | var k knn.ErrorResponse 141 | data := fmt.Sprintf("%v", err) 142 | responseErr := json.Unmarshal([]byte(data), &k) 143 | if responseErr != nil { 144 | return err 145 | } 146 | if len(k.KNNError.RootCause) > 0 { 147 | return errors.New(k.KNNError.RootCause[0].Reason) 148 | } 149 | return err 150 | } 151 | 152 | /* WarmupIndices will perform warmup on given indices 153 | GET /_opendistro/_knn/warmup/index1,index2,index3?pretty 154 | { 155 | "_shards" : { 156 | "total" : 6, 157 | "successful" : 6, 158 | "failed" : 0 159 | } 160 | } 161 | */ 162 | func (g gateway) WarmupIndices(ctx context.Context, indices string) ([]byte, error) { 163 | warmupURL, err := g.buildWarmupURL(indices) 164 | if err != nil { 165 | return nil, err 166 | } 167 | request, err := g.BuildRequest(ctx, http.MethodGet, "", warmupURL.String(), gw.GetDefaultHeaders()) 168 | if err != nil { 169 | return nil, err 170 | } 171 | response, err := g.Call(request, http.StatusOK) 172 | if err != nil { 173 | return nil, processKNNError(err) 174 | } 175 | return response, nil 176 | } 177 | -------------------------------------------------------------------------------- /gateway/knn/knn_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package knn 17 | 18 | import ( 19 | "bytes" 20 | "context" 21 | "encoding/json" 22 | "io/ioutil" 23 | "net/http" 24 | "odfe-cli/client" 25 | "odfe-cli/client/mocks" 26 | "odfe-cli/entity" 27 | "odfe-cli/entity/knn" 28 | "testing" 29 | 30 | "github.com/stretchr/testify/assert" 31 | ) 32 | 33 | func getTestClient(t *testing.T, url string, code int, response []byte) *client.Client { 34 | return mocks.NewTestClient(func(req *http.Request) *http.Response { 35 | // Test request parameters 36 | assert.Equal(t, req.URL.String(), url) 37 | assert.EqualValues(t, len(req.Header), 2) 38 | return &http.Response{ 39 | StatusCode: code, 40 | // Send response to be tested 41 | Body: ioutil.NopCloser(bytes.NewBuffer(response)), 42 | // Must be set to non-nil value or it panics 43 | Header: make(http.Header), 44 | Status: "SOME OUTPUT", 45 | Request: req, 46 | } 47 | }) 48 | } 49 | 50 | func TestGatewayGetStatistics(t *testing.T) { 51 | ctx := context.Background() 52 | t.Run("full stats succeeded", func(t *testing.T) { 53 | 54 | testClient := getTestClient(t, "http://localhost:9200/_opendistro/_knn/stats", 200, []byte("success")) 55 | testGateway := New(testClient, &entity.Profile{ 56 | Endpoint: "http://localhost:9200", 57 | UserName: "admin", 58 | Password: "admin", 59 | }) 60 | actual, err := testGateway.GetStatistics(ctx, "", "") 61 | assert.NoError(t, err) 62 | assert.EqualValues(t, string(actual), "success") 63 | }) 64 | t.Run("filtered node and stats succeeded", func(t *testing.T) { 65 | 66 | testClient := getTestClient(t, "http://localhost:9200/_opendistro/_knn/node1,node2/stats/stat1", 200, []byte("success")) 67 | testGateway := New(testClient, &entity.Profile{ 68 | Endpoint: "http://localhost:9200", 69 | UserName: "admin", 70 | Password: "admin", 71 | }) 72 | actual, err := testGateway.GetStatistics(ctx, "node1,node2", "stat1") 73 | assert.NoError(t, err) 74 | assert.EqualValues(t, string(actual), "success") 75 | }) 76 | t.Run("filtered node succeeded", func(t *testing.T) { 77 | 78 | testClient := getTestClient(t, "http://localhost:9200/_opendistro/_knn/node1,node2/stats/", 200, []byte("success")) 79 | testGateway := New(testClient, &entity.Profile{ 80 | Endpoint: "http://localhost:9200", 81 | UserName: "admin", 82 | Password: "admin", 83 | }) 84 | actual, err := testGateway.GetStatistics(ctx, "node1,node2", "") 85 | assert.NoError(t, err) 86 | assert.EqualValues(t, string(actual), "success") 87 | }) 88 | t.Run("filtered stats succeeded", func(t *testing.T) { 89 | 90 | testClient := getTestClient(t, "http://localhost:9200/_opendistro/_knn//stats/stat1,stat2", 200, []byte("success")) 91 | testGateway := New(testClient, &entity.Profile{ 92 | Endpoint: "http://localhost:9200", 93 | UserName: "admin", 94 | Password: "admin", 95 | }) 96 | actual, err := testGateway.GetStatistics(ctx, "", "stat1,stat2") 97 | assert.NoError(t, err) 98 | assert.EqualValues(t, string(actual), "success") 99 | }) 100 | t.Run("gateway failed due to gateway user config", func(t *testing.T) { 101 | 102 | testClient := getTestClient(t, "http://localhost:9200/_opendistro/_knn/stats", 400, []byte("failed")) 103 | testGateway := New(testClient, &entity.Profile{ 104 | Endpoint: "http://localhost:9200", 105 | UserName: "admin", 106 | Password: "admin", 107 | }) 108 | _, err := testGateway.GetStatistics(ctx, "", "") 109 | assert.Error(t, err) 110 | }) 111 | t.Run("failed due to invalid stat names", func(t *testing.T) { 112 | reason := "request [/_opendistro/_knn//stats/graph_count] contains unrecognized stat: [stat1]" 113 | response, _ := json.Marshal(knn.ErrorResponse{ 114 | KNNError: knn.Error{ 115 | RootCause: []knn.RootCause{ 116 | { 117 | Type: "stat_not_found_exception", 118 | Reason: reason, 119 | }, 120 | }, 121 | }, 122 | Status: 404, 123 | }) 124 | testClient := getTestClient(t, "http://localhost:9200/_opendistro/_knn/index1/stats/invalid-stats", 404, response) 125 | testGateway := New(testClient, &entity.Profile{ 126 | Endpoint: "http://localhost:9200", 127 | UserName: "admin", 128 | Password: "admin", 129 | }) 130 | _, err := testGateway.GetStatistics(ctx, "index1", "invalid-stats") 131 | assert.EqualErrorf(t, err, reason, "failed to parse error") 132 | }) 133 | } 134 | 135 | func TestGatewayWarmupIndices(t *testing.T) { 136 | ctx := context.Background() 137 | t.Run("warmup indices", func(t *testing.T) { 138 | 139 | testClient := getTestClient(t, "http://localhost:9200/_opendistro/_knn/warmup/index1,index2", 200, []byte("success")) 140 | testGateway := New(testClient, &entity.Profile{ 141 | Endpoint: "http://localhost:9200", 142 | UserName: "admin", 143 | Password: "admin", 144 | }) 145 | actual, err := testGateway.WarmupIndices(ctx, "index1,index2") 146 | assert.NoError(t, err) 147 | assert.EqualValues(t, string(actual), "success") 148 | }) 149 | t.Run("failed due to invalid index", func(t *testing.T) { 150 | 151 | response, _ := json.Marshal(knn.ErrorResponse{ 152 | KNNError: knn.Error{ 153 | RootCause: []knn.RootCause{ 154 | { 155 | Type: "index_not_found_exception", 156 | Reason: "no such index", 157 | }, 158 | }, 159 | }, 160 | Status: 404, 161 | }) 162 | testClient := getTestClient(t, "http://localhost:9200/_opendistro/_knn/warmup/index1", 404, response) 163 | testGateway := New(testClient, &entity.Profile{ 164 | Endpoint: "http://localhost:9200", 165 | UserName: "admin", 166 | Password: "admin", 167 | }) 168 | _, err := testGateway.WarmupIndices(ctx, "index1") 169 | assert.EqualErrorf(t, err, "no such index", "failed to parse error") 170 | }) 171 | } 172 | -------------------------------------------------------------------------------- /gateway/knn/mocks/mock_knn.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: odfe-cli/gateway/knn (interfaces: Gateway) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | context "context" 9 | reflect "reflect" 10 | 11 | gomock "github.com/golang/mock/gomock" 12 | ) 13 | 14 | // MockGateway is a mock of Gateway interface 15 | type MockGateway struct { 16 | ctrl *gomock.Controller 17 | recorder *MockGatewayMockRecorder 18 | } 19 | 20 | // MockGatewayMockRecorder is the mock recorder for MockGateway 21 | type MockGatewayMockRecorder struct { 22 | mock *MockGateway 23 | } 24 | 25 | // NewMockGateway creates a new mock instance 26 | func NewMockGateway(ctrl *gomock.Controller) *MockGateway { 27 | mock := &MockGateway{ctrl: ctrl} 28 | mock.recorder = &MockGatewayMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use 33 | func (m *MockGateway) EXPECT() *MockGatewayMockRecorder { 34 | return m.recorder 35 | } 36 | 37 | // GetStatistics mocks base method 38 | func (m *MockGateway) GetStatistics(arg0 context.Context, arg1, arg2 string) ([]byte, error) { 39 | m.ctrl.T.Helper() 40 | ret := m.ctrl.Call(m, "GetStatistics", arg0, arg1, arg2) 41 | ret0, _ := ret[0].([]byte) 42 | ret1, _ := ret[1].(error) 43 | return ret0, ret1 44 | } 45 | 46 | // GetStatistics indicates an expected call of GetStatistics 47 | func (mr *MockGatewayMockRecorder) GetStatistics(arg0, arg1, arg2 interface{}) *gomock.Call { 48 | mr.mock.ctrl.T.Helper() 49 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStatistics", reflect.TypeOf((*MockGateway)(nil).GetStatistics), arg0, arg1, arg2) 50 | } 51 | 52 | // WarmupIndices mocks base method 53 | func (m *MockGateway) WarmupIndices(arg0 context.Context, arg1 string) ([]byte, error) { 54 | m.ctrl.T.Helper() 55 | ret := m.ctrl.Call(m, "WarmupIndices", arg0, arg1) 56 | ret0, _ := ret[0].([]byte) 57 | ret1, _ := ret[1].(error) 58 | return ret0, ret1 59 | } 60 | 61 | // WarmupIndices indicates an expected call of WarmupIndices 62 | func (mr *MockGatewayMockRecorder) WarmupIndices(arg0, arg1 interface{}) *gomock.Call { 63 | mr.mock.ctrl.T.Helper() 64 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WarmupIndices", reflect.TypeOf((*MockGateway)(nil).WarmupIndices), arg0, arg1) 65 | } 66 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module odfe-cli 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go v1.37.25 7 | github.com/cheggaaa/pb/v3 v3.0.5 8 | github.com/golang/mock v1.4.4 9 | github.com/hashicorp/go-retryablehttp v0.6.7 10 | github.com/spf13/cobra v1.1.1 11 | github.com/stretchr/testify v1.6.1 12 | golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf 13 | gopkg.in/yaml.v2 v2.2.8 14 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 15 | 16 | ) 17 | -------------------------------------------------------------------------------- /handler/ad/testdata/create.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test-detector-ecommerce0", 3 | "description": "Test detector", 4 | "time_field": "utc_time", 5 | "index": ["kibana_sample_data_ecommerce*"], 6 | "features": [{ 7 | "aggregation_type": ["sum", "average"], 8 | "enabled": true, 9 | "field":["total_quantity"] 10 | }], 11 | "filter": { 12 | "bool": { 13 | "filter": { 14 | "term": { 15 | "currency": "EUR" 16 | } 17 | }} 18 | }, 19 | "interval": "1m", 20 | "window_delay": "1m", 21 | "start": true, 22 | "partition_field": "day_of_week" 23 | } -------------------------------------------------------------------------------- /handler/ad/testdata/invalid.txt: -------------------------------------------------------------------------------- 1 | invalid content -------------------------------------------------------------------------------- /handler/ad/testdata/update.json: -------------------------------------------------------------------------------- 1 | { 2 | "ID": "m4ccEnIBTXsGi3mvMt9p", 3 | "name": "test-detector", 4 | "description": "Test detector", 5 | "time_field": "timestamp", 6 | "indices": [ 7 | "order*" 8 | ], 9 | "features": [ 10 | { 11 | "feature_name": "total_order", 12 | "feature_enabled": true, 13 | "aggregation_query":{"total_order":{"sum":{"field":"value"}}} 14 | } 15 | ], 16 | "filter_query": {"bool" : {"filter" : [{"exists" : {"field" : "value","boost" : 1.0}}],"adjust_pure_negative" : true,"boost" : 1.0}}, 17 | "detection_interval": "5m", 18 | "window_delay": "1m", 19 | "last_update_time": 1589441737319, 20 | "schema_version": 0 21 | } 22 | 23 | -------------------------------------------------------------------------------- /handler/es/es.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package es 17 | 18 | import ( 19 | "context" 20 | "odfe-cli/controller/es" 21 | entity "odfe-cli/entity/es" 22 | ) 23 | 24 | //Handler is facade for controller 25 | type Handler struct { 26 | es.Controller 27 | } 28 | 29 | // New returns new Handler instance 30 | func New(controller es.Controller) *Handler { 31 | return &Handler{ 32 | controller, 33 | } 34 | } 35 | 36 | //Curl executes REST API as defined by curl command 37 | func Curl(h *Handler, request entity.CurlCommandRequest) ([]byte, error) { 38 | return h.Curl(request) 39 | } 40 | 41 | //Curl executes REST API as defined by curl command 42 | func (h *Handler) Curl(request entity.CurlCommandRequest) ([]byte, error) { 43 | ctx := context.Background() 44 | return h.Controller.Curl(ctx, request) 45 | } 46 | -------------------------------------------------------------------------------- /handler/es/es_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package es 17 | 18 | import ( 19 | "context" 20 | "errors" 21 | "odfe-cli/controller/es/mocks" 22 | entity "odfe-cli/entity/es" 23 | "testing" 24 | 25 | "github.com/golang/mock/gomock" 26 | "github.com/stretchr/testify/assert" 27 | ) 28 | 29 | func TestHandlerCurl(t *testing.T) { 30 | ctx := context.Background() 31 | mockCtrl := gomock.NewController(t) 32 | defer mockCtrl.Finish() 33 | arg := entity.CurlCommandRequest{} 34 | t.Run("success", func(t *testing.T) { 35 | mockedController := mocks.NewMockController(mockCtrl) 36 | mockedController.EXPECT().Curl(ctx, arg).Return([]byte(`{"result" : "success"}`), nil) 37 | instance := New(mockedController) 38 | response, err := Curl(instance, arg) 39 | assert.NoError(t, err) 40 | assert.EqualValues(t, "{\"result\" : \"success\"}", string(response)) 41 | }) 42 | t.Run("failed to execute", func(t *testing.T) { 43 | mockedController := mocks.NewMockController(mockCtrl) 44 | mockedController.EXPECT().Curl(ctx, arg).Return(nil, errors.New("failed to execute")) 45 | instance := New(mockedController) 46 | _, err := instance.Curl(arg) 47 | assert.EqualError(t, err, "failed to execute") 48 | }) 49 | } 50 | -------------------------------------------------------------------------------- /handler/knn/knn.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package knn 17 | 18 | import ( 19 | "context" 20 | "encoding/json" 21 | "odfe-cli/controller/knn" 22 | entity "odfe-cli/entity/knn" 23 | ) 24 | 25 | //Handler is facade for controller 26 | type Handler struct { 27 | knn.Controller 28 | } 29 | 30 | // New returns new Handler instance 31 | func New(controller knn.Controller) *Handler { 32 | return &Handler{ 33 | controller, 34 | } 35 | } 36 | 37 | //GetStatistics gets stats data based on nodes and stat names 38 | func GetStatistics(h *Handler, nodes string, names string) ([]byte, error) { 39 | return h.GetStatistics(nodes, names) 40 | } 41 | 42 | //GetStatistics gets stats data based on nodes and stat names 43 | func (h *Handler) GetStatistics(nodes string, names string) ([]byte, error) { 44 | ctx := context.Background() 45 | response, err := h.Controller.GetStatistics(ctx, nodes, names) 46 | if err != nil { 47 | return nil, err 48 | } 49 | var data map[string]interface{} 50 | 51 | if err := json.Unmarshal(response, &data); err != nil { 52 | return nil, err 53 | } 54 | return json.MarshalIndent(data, "", " ") 55 | } 56 | 57 | //WarmupIndices warmups knn index 58 | func WarmupIndices(h *Handler, index []string) (*entity.Shards, error) { 59 | return h.WarmupIndices(index) 60 | } 61 | 62 | //WarmupIndices warmups shard based on knn index and returns status of shards 63 | func (h *Handler) WarmupIndices(index []string) (*entity.Shards, error) { 64 | ctx := context.Background() 65 | return h.Controller.WarmupIndices(ctx, index) 66 | } 67 | -------------------------------------------------------------------------------- /handler/knn/knn_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package knn 17 | 18 | import ( 19 | "context" 20 | "errors" 21 | "odfe-cli/controller/knn/mocks" 22 | entity "odfe-cli/entity/knn" 23 | "testing" 24 | 25 | "github.com/golang/mock/gomock" 26 | "github.com/stretchr/testify/assert" 27 | ) 28 | 29 | func TestHandlerGetStatistics(t *testing.T) { 30 | ctx := context.Background() 31 | mockCtrl := gomock.NewController(t) 32 | defer mockCtrl.Finish() 33 | t.Run("get stats success", func(t *testing.T) { 34 | mockedController := mocks.NewMockController(mockCtrl) 35 | mockedController.EXPECT().GetStatistics(ctx, "node1", "stats-name").Return([]byte("{}"), nil) 36 | instance := New(mockedController) 37 | response, err := GetStatistics(instance, "node1", "stats-name") 38 | assert.NoError(t, err) 39 | assert.EqualValues(t, "{}", string(response)) 40 | }) 41 | t.Run("get stats failure", func(t *testing.T) { 42 | mockedController := mocks.NewMockController(mockCtrl) 43 | mockedController.EXPECT().GetStatistics(ctx, "node1", "stats-name").Return(nil, errors.New("failed to fetch data")) 44 | instance := New(mockedController) 45 | _, err := instance.GetStatistics("node1", "stats-name") 46 | assert.EqualError(t, err, "failed to fetch data") 47 | }) 48 | } 49 | 50 | func TestHandlerWarmupIndices(t *testing.T) { 51 | ctx := context.Background() 52 | mockCtrl := gomock.NewController(t) 53 | defer mockCtrl.Finish() 54 | t.Run("warmup success", func(t *testing.T) { 55 | mockedController := mocks.NewMockController(mockCtrl) 56 | result := &entity.Shards{ 57 | Total: 10, 58 | Successful: 5, 59 | Failed: 5, 60 | } 61 | mockedController.EXPECT().WarmupIndices(ctx, []string{"index1"}).Return(result, nil) 62 | instance := New(mockedController) 63 | response, err := WarmupIndices(instance, []string{"index1"}) 64 | assert.NoError(t, err) 65 | assert.EqualValues(t, *result, *response) 66 | }) 67 | t.Run("warmup failure", func(t *testing.T) { 68 | mockedController := mocks.NewMockController(mockCtrl) 69 | mockedController.EXPECT().WarmupIndices(ctx, []string{"index1"}).Return(nil, errors.New("failed")) 70 | instance := New(mockedController) 71 | _, err := instance.WarmupIndices([]string{"index1"}) 72 | assert.EqualError(t, err, "failed") 73 | }) 74 | } 75 | -------------------------------------------------------------------------------- /it/es/testdata/bulk-user-request: -------------------------------------------------------------------------------- 1 | { "index" : { "_index" : "bulk-user-request", "_id" : "1" } } 2 | {"price": 100, "color" : "RED"} 3 | { "index" : { "_index" : "bulk-user-request", "_id" : "2" } } 4 | {"price": 101,"color" : "RED"} 5 | { "index" : { "_index" : "bulk-user-request", "_id" : "3" } } 6 | {"price": 102,"color" : "RED"} 7 | { "index" : { "_index" : "bulk-user-request", "_id" : "4" } } 8 | {"price": 103,"color" : "BLUE"} 9 | { "index" : { "_index" : "bulk-user-request", "_id" : "5" } } 10 | {"price": 104,"color" : "BLUE"} 11 | -------------------------------------------------------------------------------- /it/es/testdata/sample-index: -------------------------------------------------------------------------------- 1 | { "index": { "_id": 1 }} 2 | { "title": "Elasticsearch: The Definitive Guide", "num_reviews": 20, "publisher": "oreilly" } 3 | { "index": { "_id": 2 }} 4 | { "title": "Taming Text: How to Find, Organize, and Manipulate It", "num_reviews": 12, "publisher": "manning" } 5 | -------------------------------------------------------------------------------- /it/es/testdata/sample-index-compressed.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendistro-for-elasticsearch/odfe-cli/0d877a4ea99818da64fca8aa7104bc6a1d29e080/it/es/testdata/sample-index-compressed.gz -------------------------------------------------------------------------------- /it/helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package it 17 | 18 | import ( 19 | "bytes" 20 | "context" 21 | "fmt" 22 | "io/ioutil" 23 | "net/http" 24 | "odfe-cli/client" 25 | "odfe-cli/entity" 26 | "os" 27 | "path/filepath" 28 | 29 | "github.com/hashicorp/go-retryablehttp" 30 | "github.com/stretchr/testify/suite" 31 | ) 32 | 33 | type ODFECLISuite struct { 34 | suite.Suite 35 | Client *client.Client 36 | Profile *entity.Profile 37 | } 38 | 39 | //HelperLoadBytes loads file from testdata and stream contents 40 | func HelperLoadBytes(name string) []byte { 41 | path := filepath.Join("testdata", name) // relative path 42 | contents, err := ioutil.ReadFile(path) 43 | if err != nil { 44 | fmt.Println(err) 45 | os.Exit(1) 46 | } 47 | return contents 48 | } 49 | 50 | // DeleteIndex deletes index by name 51 | func (a *ODFECLISuite) DeleteIndex(indexName string) { 52 | _, err := a.callRequest(http.MethodDelete, []byte(""), fmt.Sprintf("%s/%s", a.Profile.Endpoint, indexName)) 53 | 54 | if err != nil { 55 | fmt.Println(err) 56 | os.Exit(1) 57 | } 58 | } 59 | 60 | func (a *ODFECLISuite) ValidateProfile() error { 61 | if a.Profile.Endpoint == "" { 62 | return fmt.Errorf("odfe endpoint cannot be empty. set env ODFE_ENDPOINT") 63 | } 64 | if a.Profile.UserName == "" { 65 | return fmt.Errorf("odfe user name cannot be empty. set env ODFE_USER") 66 | } 67 | if a.Profile.Password == "" { 68 | return fmt.Errorf("odfe endpoint cannot be empty. set env ODFE_PASSWORD") 69 | } 70 | return nil 71 | } 72 | 73 | //CreateIndex creates test data for plugin processing 74 | func (a *ODFECLISuite) CreateIndex(indexFileName string, mappingFileName string) { 75 | if mappingFileName != "" { 76 | mapping, err := a.callRequest( 77 | http.MethodPut, HelperLoadBytes(mappingFileName), fmt.Sprintf("%s/%s", a.Profile.Endpoint, indexFileName)) 78 | if err != nil { 79 | fmt.Println(err) 80 | os.Exit(1) 81 | } 82 | fmt.Println(string(mapping)) 83 | } 84 | res, err := a.callRequest( 85 | http.MethodPost, HelperLoadBytes(indexFileName), fmt.Sprintf("%s/_bulk?refresh", a.Profile.Endpoint)) 86 | if err != nil { 87 | fmt.Println(err) 88 | os.Exit(1) 89 | } 90 | fmt.Println(string(res)) 91 | } 92 | 93 | func (a *ODFECLISuite) callRequest(method string, reqBytes []byte, url string) ([]byte, error) { 94 | var reqReader *bytes.Reader 95 | if reqBytes != nil { 96 | reqReader = bytes.NewReader(reqBytes) 97 | } 98 | r, err := retryablehttp.NewRequest(method, url, reqReader) 99 | if err != nil { 100 | return nil, err 101 | } 102 | req := r.WithContext(context.Background()) 103 | req.SetBasicAuth(a.Profile.UserName, a.Profile.Password) 104 | req.Header.Set("Content-Type", "application/x-ndjson") 105 | response, err := a.Client.HTTPClient.Do(req) 106 | if err != nil { 107 | return nil, err 108 | } 109 | defer func() { 110 | err := response.Body.Close() 111 | if err != nil { 112 | return 113 | } 114 | }() 115 | return ioutil.ReadAll(response.Body) 116 | } 117 | -------------------------------------------------------------------------------- /it/testdata/ecommerce: -------------------------------------------------------------------------------- 1 | { "index" : { "_index" : "ecommerce", "_id" : "1" } } 2 | {"currency" : "EUR","customer_id" : 38,"day_of_week" : "Monday","day_of_week_i" : 0,"total_quantity" : 2} 3 | -------------------------------------------------------------------------------- /it/testdata/knn-sample-index: -------------------------------------------------------------------------------- 1 | { "index" : { "_index" : "knn-sample-index", "_id" : "1" } } 2 | {"my_dense_vector": [1, 1],"color" : "RED"} 3 | { "index" : { "_index" : "knn-sample-index", "_id" : "2" } } 4 | {"my_dense_vector": [2, 2],"color" : "RED"} 5 | { "index" : { "_index" : "knn-sample-index", "_id" : "3" } } 6 | {"my_dense_vector": [3, 3],"color" : "RED"} 7 | { "index" : { "_index" : "knn-sample-index", "_id" : "4" } } 8 | {"my_dense_vector": [10, 10],"color" : "BLUE"} 9 | { "index" : { "_index" : "knn-sample-index", "_id" : "5" } } 10 | {"my_dense_vector": [30, 30],"color" : "BLUE"} 11 | -------------------------------------------------------------------------------- /it/testdata/knn-sample-index-mapping: -------------------------------------------------------------------------------- 1 | { 2 | "settings" : { 3 | "number_of_shards" : 1, 4 | "number_of_replicas" : 0, 5 | "index.knn" : true 6 | }, 7 | "mappings": { 8 | "properties": { 9 | "my_dense_vector": { 10 | "type": "knn_vector", 11 | "dimension": 2 12 | }, 13 | "color" : { 14 | "type" : "keyword" 15 | } 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | // odfe-cli is an unified command line tool for odfe clusters 17 | package main 18 | 19 | import ( 20 | "odfe-cli/commands" 21 | "os" 22 | ) 23 | 24 | func main() { 25 | if err := commands.Execute(); err != nil { 26 | // By default every command should handle their error message 27 | os.Exit(1) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /mapper/es/es.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package es 17 | 18 | import ( 19 | "encoding/json" 20 | "errors" 21 | "fmt" 22 | "io/ioutil" 23 | "net/http" 24 | "odfe-cli/entity/es" 25 | "strings" 26 | ) 27 | 28 | const ( 29 | HeaderSeparator = ":" 30 | MultipleHeaderSeparator = ";" 31 | QueryParamSeparator = "&" 32 | FileNameIdentifier = "@" 33 | PrettyPrintQueryParameter = "pretty=true" 34 | FormatQueryParameterTemplate = "format=%s" 35 | FilterPathQueryParameterTemplate = "filter_path=%s" 36 | ) 37 | 38 | //CommandToCurlRequestParameter map user input to Elasticsearch request 39 | func CommandToCurlRequestParameter(request es.CurlCommandRequest) (result es.CurlRequest, err error) { 40 | 41 | if result.Action, err = toHTTPAction(request.Action); err != nil { 42 | return es.CurlRequest{}, err 43 | } 44 | if result.Headers, err = toHTTPHeaders(request.Headers); err != nil { 45 | return es.CurlRequest{}, err 46 | } 47 | if result.Data, err = toCurlPayload(request.Data); err != nil { 48 | return es.CurlRequest{}, err 49 | } 50 | if !isEmpty(request.Path) { 51 | result.Path = request.Path 52 | } 53 | result.QueryParams = request.QueryParams 54 | var additionalQueryParams []string 55 | if request.Pretty { 56 | additionalQueryParams = append(additionalQueryParams, PrettyPrintQueryParameter) 57 | } 58 | if !isEmpty(request.OutputFormat) { 59 | additionalQueryParams = append(additionalQueryParams, fmt.Sprintf(FormatQueryParameterTemplate, strings.TrimSpace(request.OutputFormat))) 60 | 61 | } 62 | if !isEmpty(request.OutputFilterPath) { 63 | additionalQueryParams = append(additionalQueryParams, fmt.Sprintf(FilterPathQueryParameterTemplate, strings.TrimSpace(request.OutputFilterPath))) 64 | } 65 | if len(additionalQueryParams) > 0 { 66 | result.QueryParams = appendQueryParameter(result.QueryParams, additionalQueryParams) 67 | } 68 | return 69 | } 70 | 71 | func appendQueryParameter(path string, param []string) string { 72 | splitValues := strings.Split(path, QueryParamSeparator) 73 | splitValues = append(splitValues, param...) 74 | return strings.Join(splitValues, QueryParamSeparator) 75 | } 76 | 77 | func getSupportedHTTPAction() []string { 78 | return []string{ 79 | http.MethodGet, 80 | http.MethodPut, 81 | http.MethodPost, 82 | http.MethodDelete, 83 | } 84 | } 85 | 86 | func isEmpty(input string) bool { 87 | if len(input) == 0 { 88 | return true 89 | } 90 | trimSpaceAction := strings.TrimSpace(input) 91 | return trimSpaceAction == "" 92 | } 93 | 94 | func toHTTPAction(action string) (string, error) { 95 | if isEmpty(action) { 96 | return "", errors.New("action cannot be empty") 97 | } 98 | upperAction := strings.ToUpper(strings.TrimSpace(action)) 99 | for _, verb := range getSupportedHTTPAction() { 100 | if verb == upperAction { 101 | return verb, nil 102 | } 103 | } 104 | return "", fmt.Errorf("action: %s is not supported. Supported values are: %v", action, getSupportedHTTPAction()) 105 | } 106 | 107 | func processHeader(header string) (name string, value string, err error) { 108 | if isEmpty(header) { // ignore any empty header 109 | return 110 | } 111 | values := strings.Split(header, HeaderSeparator) 112 | if len(values) != 2 { 113 | return name, value, fmt.Errorf("invalid header format, received %s but expected is 'name: value'", header) 114 | } 115 | name = strings.ToLower(strings.TrimSpace(values[0])) 116 | value = strings.ToLower(strings.TrimSpace(values[1])) 117 | return 118 | } 119 | 120 | func toHTTPHeaders(headers string) (map[string]string, error) { 121 | if isEmpty(headers) { 122 | return nil, nil 123 | } 124 | splitHeaders := strings.Split(strings.TrimSpace(headers), MultipleHeaderSeparator) 125 | httpHeaders := map[string]string{} 126 | for _, header := range splitHeaders { 127 | name, value, err := processHeader(header) 128 | if err != nil { 129 | return nil, err 130 | } 131 | if len(name) > 0 && len(value) > 0 { // will ignore empty header 132 | httpHeaders[name] = value 133 | } 134 | } 135 | return httpHeaders, nil 136 | } 137 | 138 | func toCurlPayload(data string) (payload []byte, err error) { 139 | if isEmpty(data) { 140 | return 141 | } 142 | // if data is file name, read file contents 143 | if strings.HasPrefix(data, FileNameIdentifier) && !isEmpty(strings.TrimPrefix(data, FileNameIdentifier)) { 144 | return ioutil.ReadFile(data[1:]) 145 | } 146 | // if data is invalid json string 147 | if !json.Valid([]byte(data)) { 148 | return nil, fmt.Errorf("invalid data: %s, data can be either valid json or filename with prefix '@'", data) 149 | } 150 | return []byte(data), nil 151 | } 152 | -------------------------------------------------------------------------------- /mapper/es/es_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | package es 17 | 18 | import ( 19 | "io/ioutil" 20 | "net/http" 21 | "odfe-cli/entity/es" 22 | "path/filepath" 23 | "reflect" 24 | "testing" 25 | ) 26 | 27 | func helperLoadBytes(t *testing.T, name string) []byte { 28 | path := filepath.Join("testdata", name) // relative path 29 | contents, err := ioutil.ReadFile(path) 30 | if err != nil { 31 | t.Fatal(err) 32 | } 33 | return contents 34 | } 35 | 36 | func TestCommandToCurlRequestParameter(t *testing.T) { 37 | type args struct { 38 | request es.CurlCommandRequest 39 | } 40 | tests := []struct { 41 | name string 42 | args args 43 | wantResult es.CurlRequest 44 | wantErr bool 45 | }{ 46 | { 47 | "success: with data from file", 48 | args{ 49 | request: es.CurlCommandRequest{ 50 | Action: "post", 51 | Path: "sample-path/two", 52 | QueryParams: "a=b&c=d", 53 | Headers: "ct:value;h:23", 54 | Data: "@testdata/index.json", 55 | Pretty: false, 56 | }, 57 | }, 58 | es.CurlRequest{ 59 | Action: http.MethodPost, 60 | Path: "sample-path/two", 61 | QueryParams: "a=b&c=d", 62 | Headers: map[string]string{ 63 | "ct": "value", 64 | "h": "23", 65 | }, 66 | Data: helperLoadBytes(t, "index.json"), 67 | }, 68 | false, 69 | }, 70 | { 71 | "success: with data from stdin", 72 | args{ 73 | request: es.CurlCommandRequest{ 74 | Action: "post", 75 | Path: "sample-path/two", 76 | QueryParams: "a=b&c=d", 77 | Headers: "ct:value;h:23", 78 | Data: string(helperLoadBytes(t, "index.json")), 79 | Pretty: true, 80 | }, 81 | }, 82 | es.CurlRequest{ 83 | Action: http.MethodPost, 84 | Path: "sample-path/two", 85 | QueryParams: "a=b&c=d&pretty=true", 86 | Headers: map[string]string{ 87 | "ct": "value", 88 | "h": "23", 89 | }, 90 | Data: helperLoadBytes(t, "index.json"), 91 | }, 92 | false, 93 | }, 94 | { 95 | "success: with basic data", 96 | args{ 97 | request: es.CurlCommandRequest{ 98 | Action: "post", 99 | Path: "", 100 | QueryParams: "", 101 | Headers: "", 102 | Data: "", 103 | Pretty: true, 104 | OutputFormat: "yaml", 105 | }, 106 | }, 107 | es.CurlRequest{ 108 | Action: http.MethodPost, 109 | Path: "", 110 | QueryParams: "&pretty=true&format=yaml", 111 | Headers: nil, 112 | Data: nil, 113 | }, 114 | false, 115 | }, 116 | { 117 | "fail: invalid action", 118 | args{ 119 | request: es.CurlCommandRequest{ 120 | Action: "test", 121 | Path: "sample-path/two", 122 | QueryParams: "a=b&c=d", 123 | Headers: "ct:value;h:23", 124 | Data: "@testdata/index.json", 125 | Pretty: false, 126 | }, 127 | }, 128 | es.CurlRequest{}, 129 | true, 130 | }, 131 | { 132 | "fail: empty action", 133 | args{ 134 | request: es.CurlCommandRequest{ 135 | Action: "", 136 | Path: "sample-path/two", 137 | QueryParams: "a=b&c=d", 138 | Headers: "ct:value;h:23", 139 | Data: "@testdata/index.json", 140 | Pretty: false, 141 | }, 142 | }, 143 | es.CurlRequest{}, 144 | true, 145 | }, 146 | { 147 | "fail: invalid header", 148 | args{ 149 | request: es.CurlCommandRequest{ 150 | Action: "post", 151 | Path: "sample-path/two", 152 | QueryParams: "a=b&c=d", 153 | Headers: "ct:value:invalid;h:23", 154 | Data: "@testdata/index.json", 155 | Pretty: false, 156 | }, 157 | }, 158 | es.CurlRequest{}, 159 | true, 160 | }, 161 | { 162 | "success: empty header", 163 | args{ 164 | request: es.CurlCommandRequest{ 165 | Action: "Get", 166 | Path: " ", 167 | QueryParams: "", 168 | Headers: " ; ", 169 | Data: "{}", 170 | Pretty: true, 171 | }, 172 | }, 173 | es.CurlRequest{ 174 | Action: http.MethodGet, 175 | QueryParams: "&pretty=true", 176 | Headers: map[string]string{}, 177 | Data: []byte(`{}`), 178 | }, 179 | false, 180 | }, 181 | { 182 | "fail: invalid data", 183 | args{ 184 | request: es.CurlCommandRequest{ 185 | Action: "post", 186 | Path: "", 187 | QueryParams: "", 188 | Headers: "", 189 | Data: "this is not a json data", 190 | Pretty: false, 191 | }, 192 | }, 193 | es.CurlRequest{}, 194 | true, 195 | }, 196 | } 197 | for _, tt := range tests { 198 | t.Run(tt.name, func(t *testing.T) { 199 | gotResult, err := CommandToCurlRequestParameter(tt.args.request) 200 | if (err != nil) != tt.wantErr { 201 | t.Errorf("CommandToCurlRequestParameter() error = %v, wantErr %v", err, tt.wantErr) 202 | return 203 | } 204 | if !reflect.DeepEqual(gotResult, tt.wantResult) { 205 | t.Errorf("CommandToCurlRequestParameter() gotResult = %v, want %v", gotResult, tt.wantResult) 206 | } 207 | }) 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /mapper/es/testdata/index.json: -------------------------------------------------------------------------------- 1 | { 2 | "settings" : { 3 | "number_of_shards" : 1, 4 | "number_of_replicas" : 0, 5 | "index.knn" : true 6 | }, 7 | "mappings": { 8 | "properties": { 9 | "my_dense_vector": { 10 | "type": "knn_vector", 11 | "dimension": 2 12 | }, 13 | "color" : { 14 | "type" : "keyword" 15 | } 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /mapper/mapper.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * A copy of the License is located at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * or in the "license" file accompanying this file. This file is distributed 11 | * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 12 | * express or implied. See the License for the specific language governing 13 | * permissions and limitations under the License. 14 | */ 15 | 16 | // Package mapper provides a collection of simple mapper functions. 17 | package mapper 18 | 19 | import ( 20 | "fmt" 21 | "math" 22 | ) 23 | 24 | // IntToInt32 maps an int to an int32. 25 | func IntToInt32(r int) (int32, error) { 26 | if r < math.MinInt32 || r > math.MaxInt32 { 27 | return 0, fmt.Errorf("integer overflow, cannot map %d to int32", r) 28 | } 29 | return int32(r), nil 30 | } 31 | 32 | // IntToInt32Ptr maps an int to an *int32. 33 | func IntToInt32Ptr(r int) (*int32, error) { 34 | rr, err := IntToInt32(r) 35 | return &rr, err 36 | } 37 | 38 | // Int32PtrToInt32 maps an *int32 to an int32, 39 | // defaulting to 0 if the pointer is nil. 40 | func Int32PtrToInt32(r *int32) int32 { 41 | if r == nil { 42 | return 0 43 | } 44 | return *r 45 | } 46 | 47 | // StringToStringPtr maps a string to a *string. 48 | func StringToStringPtr(r string) *string { 49 | return &r 50 | } 51 | 52 | // BoolToBoolPtr maps a bool to a *bool. 53 | func BoolToBoolPtr(r bool) *bool { 54 | return &r 55 | } 56 | 57 | // StringPtrToString maps a *string to a string, 58 | // defaulting to "" if the pointer is nil. 59 | func StringPtrToString(r *string) string { 60 | if r == nil { 61 | return "" 62 | } 63 | return *r 64 | } 65 | -------------------------------------------------------------------------------- /release-notes/opendistro-odfe-cli-release-notes-1.1.0.md: -------------------------------------------------------------------------------- 1 | ## Version 1.1.0 Release Notes 2 | 3 | ### Features 4 | 5 | * Add knn to odfe-cli ([#20](https://github.com/opendistro-for-elasticsearch/odfe-cli/pull/20)) 6 | * Enable shell auto completion. ([#40](https://github.com/opendistro-for-elasticsearch/odfe-cli/pull/40)) 7 | * Add support for ES REST API as generic commands ([#43](https://github.com/opendistro-for-elasticsearch/odfe-cli/pull/43)) 8 | * Support AWS IAM Authentication ([#56](https://github.com/opendistro-for-elasticsearch/odfe-cli/pull/56)) 9 | 10 | ### Enhancements 11 | 12 | * Allow non-secured cluster to be added to profile ([#41](https://github.com/opendistro-for-elasticsearch/odfe-cli/pull/41)) 13 | 14 | ### Infrastructure 15 | 16 | * Add release draft workflow ([#35](https://github.com/opendistro-for-elasticsearch/odfe-cli/pull/35)) 17 | 18 | ### Documentation 19 | 20 | * Add compatibility matrix section to README ([#31](https://github.com/opendistro-for-elasticsearch/odfe-cli/pull/31)) 21 | * Add RFC for Rest API as commands. ([#42](https://github.com/opendistro-for-elasticsearch/odfe-cli/pull/42)) 22 | * Add release guidelines ([#38](https://github.com/opendistro-for-elasticsearch/odfe-cli/pull/38)) 23 | * Add doc reference as badge ([#46](https://github.com/opendistro-for-elasticsearch/odfe-cli/pull/46)) 24 | * Update compatibility matrix ([#58](https://github.com/opendistro-for-elasticsearch/odfe-cli/pull/58)) 25 | * Update README & Version ([#59](https://github.com/opendistro-for-elasticsearch/odfe-cli/pull/59)) 26 | --------------------------------------------------------------------------------