├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── nebula-ci.yml │ ├── nebula-publish.yml │ ├── nebula-snapshot.yml │ └── push-docker-image.yml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── OSSMETADATA ├── README.md ├── build.gradle ├── codequality ├── checkstyle.xml ├── netflix-intellij-code-style.xml └── pmd.xml ├── dependencies.lock ├── docs └── ARCHITECTURE.md ├── gradle ├── check.gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle └── src ├── main ├── java │ └── io │ │ └── mantisrx │ │ └── api │ │ ├── Bootstrap.java │ │ ├── Constants.java │ │ ├── MantisAPIModule.java │ │ ├── MantisConfigurationBasedServerList.java │ │ ├── MantisServerStartup.java │ │ ├── Util.java │ │ ├── filters │ │ ├── AppStreamDiscovery.java │ │ ├── Artifacts.java │ │ ├── Favicon.java │ │ ├── Healthcheck.java │ │ ├── JobDiscoveryCacheLoader.java │ │ ├── JobDiscoveryInfoCacheHitChecker.java │ │ ├── MQLParser.java │ │ ├── MREAppStreamToJobClusterMapping.java │ │ ├── MasterCacheHitChecker.java │ │ ├── MasterCacheLoader.java │ │ ├── MetricsReporting.java │ │ ├── Options.java │ │ ├── OutboundHeaders.java │ │ └── Routes.java │ │ ├── initializers │ │ └── MantisApiServerChannelInitializer.java │ │ ├── proto │ │ ├── AppDiscoveryMap.java │ │ └── Artifact.java │ │ ├── push │ │ ├── ConnectionBroker.java │ │ ├── MantisSSEHandler.java │ │ ├── MantisWebSocketFrameHandler.java │ │ └── PushConnectionDetails.java │ │ ├── services │ │ ├── AppStreamDiscoveryService.java │ │ ├── AppStreamStore.java │ │ ├── ConfigurationBasedAppStreamStore.java │ │ ├── JobDiscoveryService.java │ │ └── artifacts │ │ │ ├── ArtifactManager.java │ │ │ └── InMemoryArtifactManager.java │ │ └── tunnel │ │ ├── CrossRegionHandler.java │ │ ├── MantisCrossRegionalClient.java │ │ ├── NoOpCrossRegionalClient.java │ │ └── RegionData.java └── resources │ ├── api-docker.properties │ ├── api-local.properties │ └── log4j.properties └── test └── java └── io └── mantisrx └── api ├── UtilTest.java └── tunnel └── CrossRegionHandlerTest.java /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @calvin681 @sundargates @Andyz26 @hmitnflx @liuml07 @fdc-ntflx 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Context 2 | 3 | Explain symptoms and other details for the issue. 4 | 5 | ### Steps to reproduce 6 | 7 | Explain the steps to reliably reproduce the issue. 8 | 9 | ### Expected behavior 10 | 11 | Explain what you think should happen. 12 | 13 | ### Actual Behavior 14 | 15 | Explain what actually happens instead. 16 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Context 2 | 3 | Explain context and other details for this pull request. 4 | 5 | ### Checklist 6 | 7 | - [ ] `./gradlew build` compiles code correctly 8 | - [ ] Added new tests where applicable 9 | - [ ] `./gradlew test` passes all tests 10 | - [ ] Extended README or added javadocs where applicable 11 | - [ ] Added copyright headers for new files from `CONTRIBUTING.md` 12 | -------------------------------------------------------------------------------- /.github/workflows/nebula-ci.yml: -------------------------------------------------------------------------------- 1 | name: "CI" 2 | on: 3 | push: 4 | branches: 5 | - '*' 6 | tags-ignore: 7 | - '*' 8 | pull_request: 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | # test against JDK 8 16 | java: [ 8 ] 17 | name: CI with Java ${{ matrix.java }} 18 | steps: 19 | - uses: actions/checkout@v1 20 | - run: | 21 | git config --global user.name "Mantis OSS Maintainers" 22 | git config --global user.email "mantis-oss-dev@googlegroups.com" 23 | - name: Setup jdk 24 | uses: actions/setup-java@v1 25 | with: 26 | java-version: ${{ matrix.java }} 27 | - uses: actions/cache@v4 28 | id: gradle-cache 29 | with: 30 | path: ~/.gradle/caches 31 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }} 32 | restore-keys: | 33 | - ${{ runner.os }}-gradle- 34 | - uses: actions/cache@v4 35 | id: gradle-wrapper-cache 36 | with: 37 | path: ~/.gradle/wrapper 38 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} 39 | restore-keys: | 40 | - ${{ runner.os }}-gradlewrapper- 41 | - name: Build with Gradle 42 | run: ./gradlew --info --stacktrace build 43 | env: 44 | CI_NAME: github_actions 45 | CI_BUILD_NUMBER: ${{ github.sha }} 46 | CI_BUILD_URL: 'https://github.com/${{ github.repository }}' 47 | CI_BRANCH: ${{ github.ref }} 48 | COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} 49 | -------------------------------------------------------------------------------- /.github/workflows/nebula-publish.yml: -------------------------------------------------------------------------------- 1 | name: "Publish candidate/release to NetflixOSS and Maven Central" 2 | on: 3 | push: 4 | tags: 5 | - v*.*.* 6 | - v*.*.*-rc.* 7 | release: 8 | types: 9 | - published 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v1 16 | - run: | 17 | git config --global user.name "Mantis OSS Maintainers" 18 | git config --global user.email "mantis-oss-dev@googlegroups.com" 19 | - name: Setup jdk 8 20 | uses: actions/setup-java@v1 21 | with: 22 | java-version: 1.8 23 | - uses: actions/cache@v4 24 | id: gradle-cache 25 | with: 26 | path: ~/.gradle/caches 27 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }} 28 | restore-keys: | 29 | - ${{ runner.os }}-gradle- 30 | - uses: actions/cache@v4 31 | id: gradle-wrapper-cache 32 | with: 33 | path: ~/.gradle/wrapper 34 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} 35 | restore-keys: | 36 | - ${{ runner.os }}-gradlewrapper- 37 | - name: Publish candidate 38 | if: contains(github.ref, '-rc.') 39 | run: ./gradlew --info --stacktrace -Prelease.useLastTag=true candidate 40 | env: 41 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} 42 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} 43 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} 44 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} 45 | - name: Publish release 46 | if: (!contains(github.ref, '-rc.')) 47 | run: ./gradlew --info -Prelease.useLastTag=true final 48 | env: 49 | NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }} 50 | NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }} 51 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} 52 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} 53 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} 54 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} 55 | NETFLIX_OSS_SONATYPE_STAGING_PROFILE_ID: "c3547130240327" 56 | -------------------------------------------------------------------------------- /.github/workflows/nebula-snapshot.yml: -------------------------------------------------------------------------------- 1 | name: "Publish snapshot to NetflixOSS and Maven Central" 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | with: 14 | fetch-depth: 0 15 | - run: | 16 | git config --global user.name "Mantis OSS Maintainers" 17 | git config --global user.email "mantis-oss-dev@googlegroups.com" 18 | - name: Set up JDK 19 | uses: actions/setup-java@v1 20 | with: 21 | java-version: 8 22 | - uses: actions/cache@v4 23 | id: gradle-cache 24 | with: 25 | path: | 26 | ~/.gradle/caches 27 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} 28 | - uses: actions/cache@v4 29 | id: gradle-wrapper-cache 30 | with: 31 | path: | 32 | ~/.gradle/wrapper 33 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }} 34 | - name: Build 35 | run: ./gradlew build snapshot 36 | env: 37 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} 38 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} 39 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} 40 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} 41 | -------------------------------------------------------------------------------- /.github/workflows/push-docker-image.yml: -------------------------------------------------------------------------------- 1 | name: "Publish Docker images to Docker Registry" 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - ./build.gradle 7 | - .github/workflows/push-docker-image.yml 8 | push: 9 | tags: 10 | - v*.*.* 11 | - v*.*.*-rc.* 12 | release: 13 | types: 14 | - published 15 | 16 | jobs: 17 | build-and-push-image: 18 | runs-on: ubuntu-latest 19 | strategy: 20 | fail-fast: false 21 | permissions: 22 | contents: read 23 | packages: write 24 | 25 | steps: 26 | - name: Checkout PR 27 | uses: actions/checkout@v3 28 | - name: Setup jdk 29 | uses: actions/setup-java@v1 30 | with: 31 | java-version: 8 32 | 33 | - name: Generate dockerfiles 34 | uses: gradle/gradle-build-action@v2 35 | env: 36 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }} 37 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }} 38 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }} 39 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }} 40 | with: 41 | arguments: --info --stacktrace :dockerCreateDockerfile 42 | 43 | - name: Fetch tags 44 | run: | 45 | git fetch --prune --unshallow --tags 46 | - name: Login to Docker Hub 47 | id: login-docker-hub 48 | uses: docker/login-action@v2 49 | with: 50 | username: ${{ secrets.ORG_NETFLIXOSS_DOCKERHUB_USERNAME }} 51 | password: ${{ secrets.ORG_NETFLIXOSS_DOCKERHUB_PASSWORD }} 52 | 53 | - name: Docker meta 54 | id: meta 55 | uses: docker/metadata-action@v4 56 | with: 57 | # list of Docker images to use as base name for tags 58 | images: "netflixoss/mantisapi" 59 | # generate Docker tags based on the following events/attributes 60 | # we generate the latest tag off the beta branch 61 | tags: | 62 | type=ref,event=branch 63 | type=ref,event=pr 64 | type=semver,pattern={{version}} 65 | type=semver,pattern={{major}}.{{minor}} 66 | type=raw,value=latest,enable=${{ github.event_name == 'release' }} 67 | 68 | - name: Set up QEMU 69 | uses: docker/setup-qemu-action@v2 70 | 71 | - name: Set up Docker Buildx 72 | uses: docker/setup-buildx-action@v2 73 | 74 | - name: Build and push Docker images 75 | uses: docker/build-push-action@v3 76 | with: 77 | context: ./build/docker 78 | file: ./build/docker/Dockerfile 79 | push: ${{ github.event_name != 'pull_request' }} 80 | tags: ${{ steps.meta.outputs.tags }} 81 | labels: ${{ steps.meta.outputs.labels }} 82 | platforms: linux/amd64,linux/arm64 83 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/vim,java,linux,macos,gradle,intellij+all 3 | # Edit at https://www.gitignore.io/?templates=vim,java,linux,macos,gradle,intellij+all 4 | 5 | ### Intellij+all ### 6 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 7 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 8 | 9 | # User-specific stuff 10 | .idea/**/workspace.xml 11 | .idea/**/tasks.xml 12 | .idea/**/usage.statistics.xml 13 | .idea/**/dictionaries 14 | .idea/**/shelf 15 | 16 | # Generated files 17 | .idea/**/contentModel.xml 18 | 19 | # Sensitive or high-churn files 20 | .idea/**/dataSources/ 21 | .idea/**/dataSources.ids 22 | .idea/**/dataSources.local.xml 23 | .idea/**/sqlDataSources.xml 24 | .idea/**/dynamic.xml 25 | .idea/**/uiDesigner.xml 26 | .idea/**/dbnavigator.xml 27 | 28 | # Gradle 29 | .idea/**/gradle.xml 30 | .idea/**/libraries 31 | 32 | # Gradle and Maven with auto-import 33 | # When using Gradle or Maven with auto-import, you should exclude module files, 34 | # since they will be recreated, and may cause churn. Uncomment if using 35 | # auto-import. 36 | # .idea/modules.xml 37 | # .idea/*.iml 38 | # .idea/modules 39 | 40 | # CMake 41 | cmake-build-*/ 42 | 43 | # Mongo Explorer plugin 44 | .idea/**/mongoSettings.xml 45 | 46 | # File-based project format 47 | *.iws 48 | 49 | # IntelliJ 50 | out/ 51 | 52 | # mpeltonen/sbt-idea plugin 53 | .idea_modules/ 54 | 55 | # JIRA plugin 56 | atlassian-ide-plugin.xml 57 | 58 | # Cursive Clojure plugin 59 | .idea/replstate.xml 60 | 61 | # Crashlytics plugin (for Android Studio and IntelliJ) 62 | com_crashlytics_export_strings.xml 63 | crashlytics.properties 64 | crashlytics-build.properties 65 | fabric.properties 66 | 67 | # Editor-based Rest Client 68 | .idea/httpRequests 69 | 70 | # Android studio 3.1+ serialized cache file 71 | .idea/caches/build_file_checksums.ser 72 | 73 | # JetBrains templates 74 | **___jb_tmp___ 75 | 76 | ### Intellij+all Patch ### 77 | # Ignores the whole .idea folder and all .iml files 78 | # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 79 | 80 | .idea/ 81 | 82 | # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 83 | 84 | *.iml 85 | modules.xml 86 | .idea/misc.xml 87 | *.ipr 88 | 89 | # Sonarlint plugin 90 | .idea/sonarlint 91 | 92 | ### Java ### 93 | # Compiled class file 94 | *.class 95 | 96 | # Log file 97 | *.log 98 | 99 | # BlueJ files 100 | *.ctxt 101 | 102 | # Mobile Tools for Java (J2ME) 103 | .mtj.tmp/ 104 | 105 | # Package Files # 106 | *.jar 107 | *.war 108 | *.nar 109 | *.ear 110 | *.zip 111 | *.tar.gz 112 | *.rar 113 | 114 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 115 | hs_err_pid* 116 | 117 | ### Linux ### 118 | *~ 119 | 120 | # temporary files which can be created if a process still has a handle open of a deleted file 121 | .fuse_hidden* 122 | 123 | # KDE directory preferences 124 | .directory 125 | 126 | # Linux trash folder which might appear on any partition or disk 127 | .Trash-* 128 | 129 | # .nfs files are created when an open file is removed but is still being accessed 130 | .nfs* 131 | 132 | ### macOS ### 133 | # General 134 | .DS_Store 135 | .AppleDouble 136 | .LSOverride 137 | 138 | # Icon must end with two \r 139 | Icon 140 | 141 | # Thumbnails 142 | ._* 143 | 144 | # Files that might appear in the root of a volume 145 | .DocumentRevisions-V100 146 | .fseventsd 147 | .Spotlight-V100 148 | .TemporaryItems 149 | .Trashes 150 | .VolumeIcon.icns 151 | .com.apple.timemachine.donotpresent 152 | 153 | # Directories potentially created on remote AFP share 154 | .AppleDB 155 | .AppleDesktop 156 | Network Trash Folder 157 | Temporary Items 158 | .apdisk 159 | 160 | ### Vim ### 161 | # Swap 162 | [._]*.s[a-v][a-z] 163 | [._]*.sw[a-p] 164 | [._]s[a-rt-v][a-z] 165 | [._]ss[a-gi-z] 166 | [._]sw[a-p] 167 | 168 | # Session 169 | Session.vim 170 | 171 | # Temporary 172 | .netrwhist 173 | # Auto-generated tag files 174 | tags 175 | # Persistent undo 176 | [._]*.un~ 177 | 178 | ### Gradle ### 179 | .gradle 180 | build/ 181 | 182 | # Ignore Gradle GUI config 183 | gradle-app.setting 184 | 185 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 186 | !gradle-wrapper.jar 187 | 188 | # Cache of project 189 | .gradletasknamecache 190 | 191 | # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 192 | # gradle/wrapper/gradle-wrapper.properties 193 | 194 | ### Gradle Patch ### 195 | **/build/ 196 | 197 | # End of https://www.gitignore.io/api/vim,java,linux,macos,gradle,intellij+all 198 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Mantis 2 | 3 | If you would like to contribute code you can do so through GitHub by forking the repository and sending a pull request. When submitting code, please make every effort to follow existing conventions and style in order to keep the code as readable as possible. 4 | 5 | ## License 6 | 7 | By contributing your code, you agree to license your contribution under the terms of the [Apache License v2.0](http://www.apache.org/licenses/LICENSE-2.0). Your contributions should also include the following header: 8 | 9 | ``` 10 | /** 11 | * Copyright 2019 Netflix, Inc. 12 | * 13 | * Licensed under the Apache License, Version 2.0 (the "License"); 14 | * you may not use this file except in compliance with the License. 15 | * You may obtain a copy of the License at 16 | * 17 | * http://www.apache.org/licenses/LICENSE-2.0 18 | * 19 | * Unless required by applicable law or agreed to in writing, software 20 | * distributed under the License is distributed on an "AS IS" BASIS, 21 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 22 | * See the License for the specific language governing permissions and 23 | * limitations under the License. 24 | */ 25 | ``` 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/mantis-api/d83dc0623ff12a7755ba9557d569411edb02656b/NOTICE -------------------------------------------------------------------------------- /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=active 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mantis-api 2 | 3 | [![Build Status](https://img.shields.io/travis/com/Netflix/mantis-api.svg)](https://travis-ci.com/Netflix/mantis-api) 4 | [![OSS Lifecycle](https://img.shields.io/osslifecycle/Netflix/mantis-api.svg)](https://github.com/Netflix/mantis-api) 5 | [![License](https://img.shields.io/github/license/Netflix/mantis-api.svg)](https://www.apache.org/licenses/LICENSE-2.0) 6 | 7 | ## Development 8 | 9 | ### Building 10 | 11 | ```sh 12 | $ ./gradlew clean build 13 | ``` 14 | 15 | ### Testing 16 | 17 | ```sh 18 | $ ./gradlew clean test 19 | ``` 20 | 21 | ### Building deployment into local Maven cache 22 | 23 | ```sh 24 | $ ./gradlew clean publishNebulaPublicationToMavenLocal 25 | ``` 26 | 27 | ### Releasing 28 | 29 | We release by tagging which kicks off a CI build. The CI build will run tests, integration tests, 30 | static analysis, checkstyle, build, and then publish to the public Bintray repo to be synced into Maven Central. 31 | 32 | Tag format: 33 | 34 | ``` 35 | vMajor.Minor.Patch 36 | ``` 37 | 38 | You can tag via git or through Github's Release UI. 39 | 40 | ## Contributing 41 | 42 | Mantis is interested in building the community. We welcome any forms of contributions through discussions on any 43 | of our [mailing lists](https://netflix.github.io/mantis/community/#mailing-lists) or through patches. 44 | 45 | For more information on contribution, check out the contributions file here: 46 | 47 | - [https://github.com/Netflix/mantis/blob/master/CONTRIBUTING.md](https://github.com/Netflix/mantis/blob/master/CONTRIBUTING.md) 48 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | import org.gradle.api.internal.project.ProjectInternal 2 | 3 | /* 4 | * Copyright 2019 Netflix, Inc. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | buildscript { 20 | repositories { 21 | mavenCentral() 22 | maven { 23 | url "https://plugins.gradle.org/m2/" 24 | } 25 | gradlePluginPortal() 26 | } 27 | 28 | dependencies { 29 | classpath 'com.netflix.nebula:gradle-netflixoss-project-plugin:11.5.0' 30 | classpath 'com.bmuschko:gradle-docker-plugin:6.7.0' 31 | classpath 'com.palantir.gradle.gitversion:gradle-git-version:3.0.0' 32 | } 33 | } 34 | 35 | apply plugin: 'nebula.netflixoss' 36 | apply plugin: 'idea' 37 | apply plugin: 'java-library' 38 | apply plugin: 'pmd' 39 | apply plugin: 'application' 40 | apply plugin: 'com.bmuschko.docker-java-application' 41 | apply plugin: 'com.palantir.git-version' 42 | 43 | ext { 44 | archaiusVersion = '2.3.+' 45 | assetjCoreVersion = '3.11.1' 46 | guavaVersion = '19.+' 47 | gsonVersion = '2.8.+' 48 | guiceVersion = '4.1.0' 49 | jettyVersion = '9.4.12.v20180830' 50 | jsonVersion = '20160810' 51 | junitVersion = '4.10' 52 | lombokVersion = '1.18.10' 53 | mantisVersion = '2.0.98' 54 | mockitoVersion = '3.+' 55 | s3Version = '1.11.566' 56 | servletApiVersion = '3.1.0' 57 | spectatorVersion = '0.92.+' 58 | vavrVersion = '0.10.2' 59 | zuulVersion = '2.3.0' 60 | } 61 | 62 | group = 'io.mantisrx' 63 | 64 | sourceCompatibility = JavaVersion.VERSION_1_8 65 | targetCompatibility = JavaVersion.VERSION_1_8 66 | 67 | repositories { 68 | mavenCentral() 69 | maven { 70 | url "https://netflixoss.jfrog.io/artifactory/maven-oss-candidates" 71 | } 72 | maven { 73 | url "https://artifacts-oss.netflix.net/artifactory/maven-oss-snapshots" 74 | } 75 | } 76 | 77 | tasks.withType(Javadoc).all { 78 | enabled = false 79 | } 80 | 81 | 82 | dependencies { 83 | api("com.netflix.zuul:zuul-core:${zuulVersion}") { 84 | attributes { attribute(ProjectInternal.STATUS_ATTRIBUTE, 'release') } 85 | exclude group: 'com.google.guava', module: 'guava' 86 | exclude group: 'com.google.inject', module: 'guice' 87 | } 88 | api("com.netflix.zuul:zuul-guice:${zuulVersion}") { 89 | attributes { attribute(ProjectInternal.STATUS_ATTRIBUTE, 'release') } 90 | exclude group: 'com.google.guava', module: 'guava' 91 | exclude group: 'com.google.inject', module: 'guice' 92 | } 93 | implementation("com.netflix.zuul:zuul-groovy:${zuulVersion}") { 94 | exclude group: 'com.google.guava', module: 'guava' 95 | exclude group: 'com.google.inject', module: 'guice' 96 | } 97 | implementation 'com.netflix.blitz4j:blitz4j:1.37.2' 98 | implementation ("com.netflix.governator:governator:1.+") { 99 | attributes { attribute(ProjectInternal.STATUS_ATTRIBUTE, 'release') } 100 | } 101 | implementation "org.apache.commons:commons-lang3:3.+" 102 | implementation "commons-configuration:commons-configuration:1.8" 103 | compileOnly "com.netflix.ribbon:ribbon-loadbalancer:2.4.4" 104 | compileOnly "com.netflix.ribbon:ribbon-eureka:2.4.4" 105 | api "io.mantisrx:mantis-discovery-proto:$mantisVersion" 106 | api "io.mantisrx:mantis-client:$mantisVersion" 107 | api "io.mantisrx:mantis-runtime:$mantisVersion" 108 | api "io.mantisrx:mantis-control-plane-client:$mantisVersion" 109 | api "io.mantisrx:mantis-server-worker-client:$mantisVersion" 110 | api "io.mantisrx:mql-jvm:3.+" 111 | api "io.vavr:vavr:$vavrVersion" 112 | api "com.google.inject:guice:$guiceVersion" 113 | api "org.projectlombok:lombok:$lombokVersion" 114 | annotationProcessor "org.projectlombok:lombok:$lombokVersion" 115 | 116 | testImplementation "junit:junit:$junitVersion" 117 | testImplementation "org.mockito:mockito-core:$mockitoVersion" 118 | testImplementation"org.projectlombok:lombok:$lombokVersion" 119 | testAnnotationProcessor "org.projectlombok:lombok:$lombokVersion" 120 | } 121 | 122 | 123 | configurations.all { 124 | resolutionStrategy { 125 | force "com.google.guava:guava:$guavaVersion" 126 | force "com.google.inject:guice:$guiceVersion" 127 | force "com.google.inject.extensions:guice-multibindings:$guiceVersion" 128 | force "com.google.inject.extensions:guice-grapher:$guiceVersion" 129 | force "com.google.inject.extensions:guice-assistedinject:$guiceVersion" 130 | force "javax.servlet:javax.servlet-api:$servletApiVersion" 131 | } 132 | } 133 | 134 | /* 135 | * Run regular: ./gradlew run 136 | * Run benchmark: ./gradlew run -Pbench 137 | */ 138 | run { 139 | mainClassName = "io.mantisrx.api.Bootstrap" 140 | 141 | applicationDefaultJvmArgs = ["-DTZ=GMT", 142 | "-Darchaius.deployment.environment=test", 143 | "-Dcom.sun.management.jmxremote", 144 | "-Dcom.sun.management.jmxremote.local.only=false", 145 | "-Deureka.validateInstanceId=false", 146 | "-Deureka.mt.num_retries=1"] 147 | 148 | if (project.hasProperty('bench')) { 149 | println 'Running benchmark configuration...' 150 | jvmArgs "-Darchaius.deployment.environment=benchmark" 151 | } 152 | } 153 | 154 | def installDir = file("${buildDir}/install") 155 | def resourcesDir = file("${buildDir}/resources") 156 | def ci = System.getenv('GITHUB_ACTIONS') 157 | def imageRepository = ci ? 'netflixoss' : 'localhost:5001/netflixoss' 158 | def dockerVersionTag = gitVersion() 159 | 160 | docker { 161 | dockerSyncBuildContext { 162 | from installDir 163 | from resourcesDir 164 | } 165 | 166 | dockerCreateDockerfile { 167 | // root directory to store all the files 168 | instruction 'WORKDIR /apps/mantis/mantis-api' 169 | // copy the files from the build context to the image 170 | instruction 'COPY mantis-api/bin/* /apps/mantis/mantis-api/bin/' 171 | instruction 'COPY mantis-api/lib/* /apps/mantis/mantis-api/lib/' 172 | instruction 'COPY resources/* conf/' 173 | entryPoint 'bin/mantis-api', '-p', 'api-docker.properties' 174 | } 175 | 176 | javaApplication { 177 | baseImage = 'azul/zulu-openjdk:8-latest' 178 | maintainer = 'Mantis Developers "mantis-oss-dev@netflix.com"' 179 | mainClassName = 'io.mantisrx.api.Bootstrap' 180 | images = ["$imageRepository/mantisapi:latest", "$imageRepository/mantisapi:$dockerVersionTag"] 181 | ports = [5050] 182 | } 183 | } 184 | 185 | dockerSyncBuildContext.dependsOn(installDist) 186 | 187 | pmd { 188 | ignoreFailures = true 189 | } 190 | -------------------------------------------------------------------------------- /codequality/checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 21 | 22 | 23 | 24 | 25 | 26 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 158 | 159 | 160 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | -------------------------------------------------------------------------------- /codequality/netflix-intellij-code-style.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 23 | 28 | 41 | -------------------------------------------------------------------------------- /codequality/pmd.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 23 | 24 | Exclude noisy rules. 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | -------------------------------------------------------------------------------- /dependencies.lock: -------------------------------------------------------------------------------- 1 | { 2 | "annotationProcessor": { 3 | "org.projectlombok:lombok": { 4 | "locked": "1.18.10" 5 | } 6 | }, 7 | "compileClasspath": { 8 | "com.google.inject:guice": { 9 | "locked": "4.1.0" 10 | }, 11 | "com.netflix.blitz4j:blitz4j": { 12 | "locked": "1.37.2" 13 | }, 14 | "com.netflix.governator:governator": { 15 | "locked": "1.17.13" 16 | }, 17 | "com.netflix.ribbon:ribbon-eureka": { 18 | "locked": "2.4.4" 19 | }, 20 | "com.netflix.ribbon:ribbon-loadbalancer": { 21 | "locked": "2.4.4" 22 | }, 23 | "com.netflix.zuul:zuul-core": { 24 | "locked": "2.3.0" 25 | }, 26 | "com.netflix.zuul:zuul-groovy": { 27 | "locked": "2.3.0" 28 | }, 29 | "com.netflix.zuul:zuul-guice": { 30 | "locked": "2.3.0" 31 | }, 32 | "commons-configuration:commons-configuration": { 33 | "locked": "1.8" 34 | }, 35 | "io.mantisrx:mantis-client": { 36 | "locked": "2.0.98" 37 | }, 38 | "io.mantisrx:mantis-control-plane-client": { 39 | "locked": "2.0.98" 40 | }, 41 | "io.mantisrx:mantis-discovery-proto": { 42 | "locked": "2.0.98" 43 | }, 44 | "io.mantisrx:mantis-runtime": { 45 | "locked": "2.0.98" 46 | }, 47 | "io.mantisrx:mantis-server-worker-client": { 48 | "locked": "2.0.98" 49 | }, 50 | "io.mantisrx:mql-jvm": { 51 | "locked": "3.4.0" 52 | }, 53 | "io.vavr:vavr": { 54 | "locked": "0.10.2" 55 | }, 56 | "org.apache.commons:commons-lang3": { 57 | "locked": "3.13.0" 58 | }, 59 | "org.projectlombok:lombok": { 60 | "locked": "1.18.10" 61 | } 62 | }, 63 | "pmd": { 64 | "net.sourceforge.pmd:pmd-java": { 65 | "locked": "6.26.0" 66 | } 67 | }, 68 | "runtimeClasspath": { 69 | "com.google.inject:guice": { 70 | "locked": "4.1.0" 71 | }, 72 | "com.netflix.blitz4j:blitz4j": { 73 | "locked": "1.37.2" 74 | }, 75 | "com.netflix.governator:governator": { 76 | "locked": "1.17.13" 77 | }, 78 | "com.netflix.zuul:zuul-core": { 79 | "locked": "2.3.0" 80 | }, 81 | "com.netflix.zuul:zuul-groovy": { 82 | "locked": "2.3.0" 83 | }, 84 | "com.netflix.zuul:zuul-guice": { 85 | "locked": "2.3.0" 86 | }, 87 | "commons-configuration:commons-configuration": { 88 | "locked": "1.8" 89 | }, 90 | "io.mantisrx:mantis-client": { 91 | "locked": "2.0.98" 92 | }, 93 | "io.mantisrx:mantis-control-plane-client": { 94 | "locked": "2.0.98" 95 | }, 96 | "io.mantisrx:mantis-discovery-proto": { 97 | "locked": "2.0.98" 98 | }, 99 | "io.mantisrx:mantis-runtime": { 100 | "locked": "2.0.98" 101 | }, 102 | "io.mantisrx:mantis-server-worker-client": { 103 | "locked": "2.0.98" 104 | }, 105 | "io.mantisrx:mql-jvm": { 106 | "locked": "3.4.0" 107 | }, 108 | "io.vavr:vavr": { 109 | "locked": "0.10.2" 110 | }, 111 | "org.apache.commons:commons-lang3": { 112 | "locked": "3.13.0" 113 | }, 114 | "org.projectlombok:lombok": { 115 | "locked": "1.18.10" 116 | } 117 | }, 118 | "testAnnotationProcessor": { 119 | "org.projectlombok:lombok": { 120 | "locked": "1.18.10" 121 | } 122 | }, 123 | "testCompileClasspath": { 124 | "com.google.inject:guice": { 125 | "locked": "4.1.0" 126 | }, 127 | "com.netflix.blitz4j:blitz4j": { 128 | "locked": "1.37.2" 129 | }, 130 | "com.netflix.governator:governator": { 131 | "locked": "1.17.13" 132 | }, 133 | "com.netflix.zuul:zuul-core": { 134 | "locked": "2.3.0" 135 | }, 136 | "com.netflix.zuul:zuul-groovy": { 137 | "locked": "2.3.0" 138 | }, 139 | "com.netflix.zuul:zuul-guice": { 140 | "locked": "2.3.0" 141 | }, 142 | "commons-configuration:commons-configuration": { 143 | "locked": "1.8" 144 | }, 145 | "io.mantisrx:mantis-client": { 146 | "locked": "2.0.98" 147 | }, 148 | "io.mantisrx:mantis-control-plane-client": { 149 | "locked": "2.0.98" 150 | }, 151 | "io.mantisrx:mantis-discovery-proto": { 152 | "locked": "2.0.98" 153 | }, 154 | "io.mantisrx:mantis-runtime": { 155 | "locked": "2.0.98" 156 | }, 157 | "io.mantisrx:mantis-server-worker-client": { 158 | "locked": "2.0.98" 159 | }, 160 | "io.mantisrx:mql-jvm": { 161 | "locked": "3.4.0" 162 | }, 163 | "io.vavr:vavr": { 164 | "locked": "0.10.2" 165 | }, 166 | "junit:junit": { 167 | "locked": "4.13.1" 168 | }, 169 | "org.apache.commons:commons-lang3": { 170 | "locked": "3.13.0" 171 | }, 172 | "org.mockito:mockito-core": { 173 | "locked": "3.12.4" 174 | }, 175 | "org.projectlombok:lombok": { 176 | "locked": "1.18.10" 177 | } 178 | }, 179 | "testRuntimeClasspath": { 180 | "com.google.inject:guice": { 181 | "locked": "4.1.0" 182 | }, 183 | "com.netflix.blitz4j:blitz4j": { 184 | "locked": "1.37.2" 185 | }, 186 | "com.netflix.governator:governator": { 187 | "locked": "1.17.13" 188 | }, 189 | "com.netflix.zuul:zuul-core": { 190 | "locked": "2.3.0" 191 | }, 192 | "com.netflix.zuul:zuul-groovy": { 193 | "locked": "2.3.0" 194 | }, 195 | "com.netflix.zuul:zuul-guice": { 196 | "locked": "2.3.0" 197 | }, 198 | "commons-configuration:commons-configuration": { 199 | "locked": "1.8" 200 | }, 201 | "io.mantisrx:mantis-client": { 202 | "locked": "2.0.98" 203 | }, 204 | "io.mantisrx:mantis-control-plane-client": { 205 | "locked": "2.0.98" 206 | }, 207 | "io.mantisrx:mantis-discovery-proto": { 208 | "locked": "2.0.98" 209 | }, 210 | "io.mantisrx:mantis-runtime": { 211 | "locked": "2.0.98" 212 | }, 213 | "io.mantisrx:mantis-server-worker-client": { 214 | "locked": "2.0.98" 215 | }, 216 | "io.mantisrx:mql-jvm": { 217 | "locked": "3.4.0" 218 | }, 219 | "io.vavr:vavr": { 220 | "locked": "0.10.2" 221 | }, 222 | "junit:junit": { 223 | "locked": "4.13.1" 224 | }, 225 | "org.apache.commons:commons-lang3": { 226 | "locked": "3.13.0" 227 | }, 228 | "org.mockito:mockito-core": { 229 | "locked": "3.12.4" 230 | }, 231 | "org.projectlombok:lombok": { 232 | "locked": "1.18.10" 233 | } 234 | } 235 | } -------------------------------------------------------------------------------- /docs/ARCHITECTURE.md: -------------------------------------------------------------------------------- 1 | # Mantis API Architecture 2 | The primary objective of Mantis API is to provide a horizontally scalable front to the Mantis Master. The role of the API is to relieve pressure from the Master bvy reducing the total number of incoming connections. 3 | 4 | There are three primary pieces of functionality; 5 | 6 | 1. Proxying Calls to Mantis Master 7 | 1. Streaming data from Mantis Jobs on behalf of client 8 | 1. Proxying data between regions using the tunnel 9 | 10 | These pieces are wired together using Zuul's concept of a `ChannelInitializer` which is a subclass of Netty's concept of the same name. The [MantisApiServerChannelInitializer](src/main/java/io/mantisrx/api/initializers/MantisApiServerChannelInitializer.java) 11 | sets up the Netty channel and then adds handlers based on the route. Each of these pieces of functionality are discussed in turn below. 12 | 13 | ## Proxying calls to Mantis Master 14 | The API by default will proxy any unintercepted HTTP request to the Mantis Master. This is handled by the underlying [Zuul](https://github.com/netflix/zuul) implementation, we 15 | treat the Mantis Master as the Zuul Origin `api`, which treats all requests beginning with `/api` as calls to the specified origin. We always want this origin to be our current leader in the Manits Master cluster. 16 | 17 | There are three aspects of the api which are germain to proxying to Mantis Master: 18 | 19 | ### Static Server List and Leader Election 20 | Zuul typically uses Eureka to discover instances of the service to which it is proxying. This doesn't work for Mantis as we have a specific leader to which we want to proxy. 21 | To this end we use Zuul's static server list feature, which we update via the `masterClient.getMasterMonitor()` in [MantisServerStartup.java](../src/main/java/io/mantisrx/api/MantisServerStartup.java). 22 | Here we observe the Master Monitor and set the `api.ribbon.listOfServers` Note the origin name `api` at the beginning of this. 23 | 24 | ### Routes 25 | [Routes](../src/main/java/io/mantisrx/api/filters/Routes.java) affect this, and may cause a call to be routed to other "filters". Anything directed to `ZuulEndPointRunner.PROXY_ENDPOINT_FILTER_NAME` will 26 | be proxied to the master. 27 | 28 | ### Filters 29 | Zuul Filters may manipulate either incoming requests before they're proxied, or outgoinmg requests after they are proxied. We have as a policy attempted to minimize the amount of manipulation. The reason 30 | for this is simple, we want the API defined by `mantismasterv2` to define the standard, however we must support existing client expectations. Therefore minimizing differences in master responses and API responses 31 | achieves this. 32 | 33 | See [src/main/java/io/mantisrx/api/filters](../src/main/java/io/mantisrx/api/filters) for a list of all filters loaded at runtime. Note that these may be incoming, outgoing, or endpoints. 34 | 35 | ## Streaming Data 36 | The api is tasked with providing an interface to end users, via websocket/sse, to access data streaming from jobs. 37 | This is achieved through the [io.mantisrx.api.push](../src/main/java/io/mantisrx/api/push) namespace which contains a few critical components. 38 | 39 | * PushConnectionDetails is a domain object which fully specifies the target (including cross regional) of a streaming data request. 40 | * ConnectionBroker is responsible for acquiring (and potentially multiplexing) data for a PushConnectionDetails instance and returning an `Observable` 41 | * MantisSSEHandler is responsible for determining if the request is a websocket upgrade and the `Observable` serving server sent events if it is not an upgrade. 42 | * MantisWebSocketFrameHandler is responsible for serving the `Observable` over websocket. 43 | 44 | ## Cross Regional Data 45 | The ConnectionBroker also handles cross-regional data streams with the help of the contents of the [io.mantisrx.api.tunnel](../src/main/java/io/mantisrx/api/tunnel) namespace. 46 | In the open source world we do not have a tunnel provider, it is up to the user to implement `MantisCrossRegionalClient`. Internally we have a Metatron based implementation which mutually auths the servers before 47 | exchanging data over SSL. 48 | -------------------------------------------------------------------------------- /gradle/check.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | checkstyle { 18 | toolVersion = '8.14' 19 | // TODO: Don't ignore failures. 20 | ignoreFailures = true 21 | configFile = rootProject.file('codequality/checkstyle.xml') 22 | sourceSets = [sourceSets.main] 23 | } 24 | 25 | pmd { 26 | toolVersion = '6.9.0' 27 | // TODO: Don't ignore failures. 28 | ignoreFailures = true 29 | sourceSets = [sourceSets.main] 30 | ruleSets = [] 31 | ruleSetFiles = rootProject.files("codequality/pmd.xml") 32 | } 33 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Netflix/mantis-api/d83dc0623ff12a7755ba9557d569411edb02656b/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2019 Netflix, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | distributionBase=GRADLE_USER_HOME 17 | distributionPath=wrapper/dists 18 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip 19 | zipStoreBase=GRADLE_USER_HOME 20 | zipStorePath=wrapper/dists 21 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | rootProject.name = 'mantis-api' 18 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/Bootstrap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.api; 18 | 19 | import com.google.inject.Injector; 20 | import com.netflix.config.ConfigurationManager; 21 | import com.netflix.governator.InjectorBuilder; 22 | import com.netflix.zuul.netty.server.BaseServerStartup; 23 | import com.netflix.zuul.netty.server.Server; 24 | 25 | /** 26 | * Bootstrap 27 | * 28 | * Author: Arthur Gonigberg 29 | * Date: November 20, 2017 30 | */ 31 | public class Bootstrap { 32 | 33 | public static void main(String[] args) { 34 | String propertiesFile = null; 35 | if (args.length >= 2 && "-p".equals(args[0])) { 36 | propertiesFile = args[1]; 37 | if (propertiesFile.endsWith(".properties")) { 38 | propertiesFile = propertiesFile.substring(0, propertiesFile.length() - 11); 39 | } 40 | } 41 | new Bootstrap().start(propertiesFile); 42 | } 43 | 44 | public void start(String configName) { 45 | System.out.println("Mantis API: starting up."); 46 | long startTime = System.currentTimeMillis(); 47 | int exitCode = 0; 48 | 49 | Server server = null; 50 | 51 | try { 52 | ConfigurationManager.loadCascadedPropertiesFromResources(configName); 53 | Injector injector = InjectorBuilder.fromModule(new MantisAPIModule()).createInjector(); 54 | BaseServerStartup serverStartup = injector.getInstance(BaseServerStartup.class); 55 | server = serverStartup.server(); 56 | 57 | long startupDuration = System.currentTimeMillis() - startTime; 58 | System.out.println("Mantis API: finished startup. Duration = " + startupDuration + " ms"); 59 | 60 | server.start(); 61 | server.awaitTermination(); 62 | } 63 | catch (Throwable t) { 64 | t.printStackTrace(); 65 | System.err.println("###############"); 66 | System.err.println("Mantis API: initialization failed. Forcing shutdown now."); 67 | System.err.println("###############"); 68 | exitCode = 1; 69 | } 70 | finally { 71 | // server shutdown 72 | if (server != null) server.stop(); 73 | 74 | System.exit(exitCode); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/Constants.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api; 2 | 3 | import lombok.experimental.UtilityClass; 4 | 5 | @UtilityClass 6 | public class Constants { 7 | public static final String numMessagesCounterName = "numSinkMessages"; 8 | public static final String numDroppedMessagesCounterName = "numDroppedSinkMessages"; 9 | public static final String numBytesCounterName = "numSinkBytes"; 10 | public static final String numDroppedBytesCounterName = "numDroppedSinkBytes"; 11 | public static final String drainTriggeredCounterName = "drainTriggered"; 12 | public static final String numIncomingMessagesCounterName = "numIncomingMessages"; 13 | 14 | public static final String SSE_DATA_SUFFIX = "\r\n\r\n"; 15 | public static final String SSE_DATA_PREFIX = "data: "; 16 | 17 | public static final long TunnelPingIntervalSecs = 12; 18 | public static final String TunnelPingMessage = "MantisApiTunnelPing"; 19 | public static final String TunnelPingParamName = "MantisApiTunnelPingEnabled"; 20 | 21 | public static final String OriginRegionTagName = "originRegion"; 22 | public static final String ClientIdTagName = "clientId"; 23 | public static final String TagsParamName = "MantisApiTag"; 24 | public static final String TagNameValDelimiter = ":"; 25 | 26 | public static final String metaErrorMsgHeader = "mantis.meta.error.message"; 27 | public static final String metaOriginName = "mantis.meta.origin"; 28 | 29 | public static final String numRemoteBytesCounterName = "numRemoteSinkBytes"; 30 | public static final String numRemoteMessagesCounterName = "numRemoteMessages"; 31 | public static final String numSseErrorsCounterName = "numSseErrors"; 32 | 33 | public static final String DUMMY_TIMER_DATA = "DUMMY_TIMER_DATA"; 34 | 35 | public static final String MANTISAPI_CACHED_HEADER = "x-nflx-mantisapi-cached"; 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/MantisAPIModule.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.api; 18 | 19 | import com.google.inject.Scopes; 20 | import com.google.inject.util.Modules; 21 | import com.netflix.appinfo.EurekaInstanceConfig; 22 | import com.netflix.appinfo.providers.MyDataCenterInstanceConfigProvider; 23 | import com.netflix.discovery.guice.EurekaModule; 24 | import com.netflix.zuul.*; 25 | import com.netflix.zuul.filters.FilterRegistry; 26 | import com.netflix.zuul.filters.MutableFilterRegistry; 27 | import com.netflix.zuul.groovy.GroovyCompiler; 28 | import com.netflix.zuul.groovy.GroovyFileFilter; 29 | import io.mantisrx.api.services.AppStreamDiscoveryService; 30 | import io.mantisrx.api.services.AppStreamStore; 31 | import io.mantisrx.api.services.ConfigurationBasedAppStreamStore; 32 | import io.mantisrx.server.core.Configurations; 33 | import io.mantisrx.server.core.CoreConfiguration; 34 | import io.mantisrx.server.master.client.HighAvailabilityServices; 35 | import io.mantisrx.server.master.client.HighAvailabilityServicesUtil; 36 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.DeserializationFeature; 37 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper; 38 | import com.google.inject.AbstractModule; 39 | import com.google.inject.Provides; 40 | import com.google.inject.Singleton; 41 | import com.google.inject.name.Named; 42 | import com.netflix.config.ConfigurationManager; 43 | import com.netflix.netty.common.accesslog.AccessLogPublisher; 44 | import com.netflix.netty.common.status.ServerStatusManager; 45 | import com.netflix.spectator.api.DefaultRegistry; 46 | import com.netflix.spectator.api.Registry; 47 | import com.netflix.spectator.api.patterns.ThreadPoolMonitor; 48 | import com.netflix.zuul.context.SessionContextDecorator; 49 | import com.netflix.zuul.context.ZuulSessionContextDecorator; 50 | import com.netflix.zuul.init.ZuulFiltersModule; 51 | import com.netflix.zuul.netty.server.BaseServerStartup; 52 | import com.netflix.zuul.netty.server.ClientRequestReceiver; 53 | import com.netflix.zuul.origins.BasicNettyOriginManager; 54 | import com.netflix.zuul.origins.OriginManager; 55 | import io.mantisrx.api.services.artifacts.ArtifactManager; 56 | import io.mantisrx.api.services.artifacts.InMemoryArtifactManager; 57 | import com.netflix.zuul.stats.BasicRequestMetricsPublisher; 58 | import com.netflix.zuul.stats.RequestMetricsPublisher; 59 | import io.mantisrx.api.tunnel.MantisCrossRegionalClient; 60 | import io.mantisrx.api.tunnel.NoOpCrossRegionalClient; 61 | import io.mantisrx.client.MantisClient; 62 | import io.mantisrx.server.worker.client.WorkerMetricsClient; 63 | import io.mantisrx.shaded.org.apache.curator.framework.listen.Listenable; 64 | import io.mantisrx.shaded.org.apache.curator.framework.listen.ListenerContainer; 65 | import org.apache.commons.configuration.AbstractConfiguration; 66 | import rx.Scheduler; 67 | import rx.schedulers.Schedulers; 68 | 69 | import java.io.FilenameFilter; 70 | import java.util.ArrayList; 71 | import java.util.List; 72 | import java.util.Properties; 73 | import java.util.concurrent.*; 74 | 75 | public class MantisAPIModule extends AbstractModule { 76 | @Override 77 | protected void configure() { 78 | bind(AbstractConfiguration.class).toInstance(ConfigurationManager.getConfigInstance()); 79 | 80 | bind(BaseServerStartup.class).to(MantisServerStartup.class); 81 | 82 | // use provided basic netty origin manager 83 | bind(OriginManager.class).to(BasicNettyOriginManager.class); 84 | 85 | // zuul filter loading 86 | bind(DynamicCodeCompiler.class).to(GroovyCompiler.class); 87 | bind(FilenameFilter.class).to(GroovyFileFilter.class); 88 | 89 | install(Modules.override(new EurekaModule()).with(new AbstractModule() { 90 | @Override 91 | protected void configure() { 92 | bind(EurekaInstanceConfig.class).toProvider(MyDataCenterInstanceConfigProvider.class).in(Scopes.SINGLETON); 93 | } 94 | })); 95 | 96 | install(new ZuulFiltersModule()); 97 | bind(FilterLoader.class).to(DynamicFilterLoader.class); 98 | bind(FilterRegistry.class).to(MutableFilterRegistry.class); 99 | bind(FilterFileManager.class).asEagerSingleton(); 100 | 101 | // general server bindings 102 | bind(ServerStatusManager.class); // health/discovery status 103 | bind(SessionContextDecorator.class).to(ZuulSessionContextDecorator.class); // decorate new sessions when requests come in 104 | bind(Registry.class).to(DefaultRegistry.class); // atlas metrics registry 105 | bind(RequestCompleteHandler.class).to(BasicRequestCompleteHandler.class); // metrics post-request completion 106 | bind(RequestMetricsPublisher.class).to(BasicRequestMetricsPublisher.class); // timings publisher 107 | 108 | // access logger, including request ID generator 109 | bind(AccessLogPublisher.class).toInstance(new AccessLogPublisher("ACCESS", 110 | (channel, httpRequest) -> ClientRequestReceiver.getRequestFromChannel(channel).getContext().getUUID())); 111 | 112 | bind(ArtifactManager.class).to(InMemoryArtifactManager.class); 113 | bind(MantisCrossRegionalClient.class).to(NoOpCrossRegionalClient.class); 114 | 115 | bind(ObjectMapper.class).toInstance(new ObjectMapper() 116 | .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)); 117 | } 118 | 119 | @Provides 120 | @Singleton 121 | HighAvailabilityServices provideHighAvailabilityServices(AbstractConfiguration configuration) { 122 | Properties props = new Properties(); 123 | configuration.getKeys("mantis").forEachRemaining(key -> { 124 | props.put(key, configuration.getString(key)); 125 | }); 126 | 127 | return HighAvailabilityServicesUtil.createHAServices( 128 | Configurations.frmProperties(props, CoreConfiguration.class)); 129 | } 130 | 131 | @Provides @Singleton MantisClient provideMantisClient(AbstractConfiguration configuration) { 132 | Properties props = new Properties(); 133 | configuration.getKeys("mantis").forEachRemaining(key -> { 134 | props.put(key, configuration.getString(key)); 135 | }); 136 | 137 | return new MantisClient(props); 138 | } 139 | 140 | @Provides 141 | @Singleton 142 | @Named("io-scheduler") 143 | Scheduler provideIoScheduler(Registry registry) { 144 | ThreadPoolExecutor executor = new ThreadPoolExecutor(16, 128, 60, 145 | TimeUnit.SECONDS, new LinkedBlockingQueue<>()); 146 | ThreadPoolMonitor.attach(registry, executor, "io-thread-pool"); 147 | return Schedulers.from(executor); 148 | } 149 | 150 | @Provides 151 | @Singleton 152 | ConfigurationBasedAppStreamStore.ConfigSource provideConfigSource(AbstractConfiguration configuration) { 153 | return new ConfigurationBasedAppStreamStore.ConfigSource() { 154 | @Override 155 | public Listenable getListenable() { 156 | return new ListenerContainer<>(); 157 | } 158 | 159 | @Override 160 | public String get() { 161 | return String.join(",", configuration.getStringArray("mreAppJobClusterMap")); 162 | } 163 | }; 164 | } 165 | 166 | @Provides 167 | @Singleton 168 | AppStreamDiscoveryService provideAppStreamDiscoveryService(MantisClient mantisClient, 169 | @Named("io-scheduler") Scheduler ioScheduler, 170 | ConfigurationBasedAppStreamStore.ConfigSource configSource) { 171 | AppStreamStore appStreamStore = new ConfigurationBasedAppStreamStore(configSource); 172 | return new AppStreamDiscoveryService(mantisClient, ioScheduler, appStreamStore); 173 | } 174 | 175 | @Provides @Singleton 176 | WorkerMetricsClient provideWorkerMetricsClient(AbstractConfiguration configuration) { 177 | Properties props = new Properties(); 178 | configuration.getKeys("mantis").forEachRemaining(key -> { 179 | props.put(key, configuration.getString(key)); 180 | }); 181 | return new WorkerMetricsClient(props); 182 | } 183 | 184 | @Provides 185 | @Singleton 186 | @Named("push-prefixes") 187 | List providePushPrefixes() { 188 | List pushPrefixes = new ArrayList<>(20); 189 | pushPrefixes.add("/jobconnectbyid"); 190 | pushPrefixes.add("/api/v1/jobconnectbyid"); 191 | pushPrefixes.add("/jobconnectbyname"); 192 | pushPrefixes.add("/api/v1/jobconnectbyname"); 193 | pushPrefixes.add("/jobsubmitandconnect"); 194 | pushPrefixes.add("/api/v1/jobsubmitandconnect"); 195 | pushPrefixes.add("/jobClusters/discoveryInfoStream"); 196 | pushPrefixes.add("/jobstatus"); 197 | pushPrefixes.add("/api/v1/jobstatus"); 198 | pushPrefixes.add("/api/v1/jobs/schedulingInfo/"); 199 | pushPrefixes.add("/api/v1/metrics"); 200 | 201 | return pushPrefixes; 202 | } 203 | 204 | } 205 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/MantisConfigurationBasedServerList.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.mantisrx.api; 17 | 18 | import com.google.common.base.Strings; 19 | import com.google.common.collect.Lists; 20 | import com.netflix.appinfo.InstanceInfo; 21 | import com.netflix.loadbalancer.ConfigurationBasedServerList; 22 | import com.netflix.loadbalancer.Server; 23 | import com.netflix.niws.loadbalancer.DiscoveryEnabledServer; 24 | 25 | import java.util.List; 26 | 27 | public class MantisConfigurationBasedServerList extends ConfigurationBasedServerList { 28 | @Override 29 | protected List derive(String value) { 30 | List list = Lists.newArrayList(); 31 | if (!Strings.isNullOrEmpty(value)) { 32 | for (String s : value.split(",")) { 33 | Server server = new Server(s.trim()); 34 | InstanceInfo instanceInfo = 35 | InstanceInfo.Builder.newBuilder() 36 | .setAppName("mantismasterv2") 37 | .setIPAddr(server.getHost()) 38 | .setPort(server.getPort()) 39 | .build(); 40 | list.add(new DiscoveryEnabledServer(instanceInfo, false, true)); 41 | } 42 | } 43 | return list; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/MantisServerStartup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.api; 18 | 19 | import com.google.inject.name.Named; 20 | import com.netflix.appinfo.ApplicationInfoManager; 21 | import com.netflix.config.DynamicIntProperty; 22 | import com.netflix.discovery.EurekaClient; 23 | import com.netflix.netty.common.accesslog.AccessLogPublisher; 24 | import com.netflix.netty.common.channel.config.ChannelConfig; 25 | import com.netflix.netty.common.channel.config.CommonChannelConfigKeys; 26 | import com.netflix.netty.common.metrics.EventLoopGroupMetrics; 27 | import com.netflix.netty.common.proxyprotocol.StripUntrustedProxyHeadersHandler; 28 | import com.netflix.netty.common.status.ServerStatusManager; 29 | import com.netflix.spectator.api.Registry; 30 | import com.netflix.zuul.FilterLoader; 31 | import com.netflix.zuul.FilterUsageNotifier; 32 | import com.netflix.zuul.RequestCompleteHandler; 33 | import com.netflix.zuul.context.SessionContextDecorator; 34 | import com.netflix.zuul.netty.server.BaseServerStartup; 35 | import com.netflix.zuul.netty.server.DirectMemoryMonitor; 36 | import com.netflix.zuul.netty.server.NamedSocketAddress; 37 | 38 | import io.mantisrx.api.initializers.MantisApiServerChannelInitializer; 39 | import io.mantisrx.api.push.ConnectionBroker; 40 | import io.mantisrx.api.tunnel.MantisCrossRegionalClient; 41 | import io.mantisrx.server.master.client.HighAvailabilityServices; 42 | import io.netty.channel.ChannelInitializer; 43 | import io.netty.channel.group.ChannelGroup; 44 | import org.apache.commons.configuration.AbstractConfiguration; 45 | import rx.Scheduler; 46 | 47 | import javax.inject.Inject; 48 | import javax.inject.Singleton; 49 | import java.net.InetSocketAddress; 50 | import java.util.*; 51 | 52 | @Singleton 53 | public class MantisServerStartup extends BaseServerStartup { 54 | 55 | private final HighAvailabilityServices highAvailabilityServices; 56 | private final MantisCrossRegionalClient mantisCrossRegionalClient; 57 | private final ConnectionBroker connectionBroker; 58 | private final Scheduler scheduler; 59 | private final List pushPrefixes; 60 | 61 | @Inject 62 | public MantisServerStartup(ServerStatusManager serverStatusManager, FilterLoader filterLoader, 63 | SessionContextDecorator sessionCtxDecorator, FilterUsageNotifier usageNotifier, 64 | RequestCompleteHandler reqCompleteHandler, Registry registry, 65 | DirectMemoryMonitor directMemoryMonitor, EventLoopGroupMetrics eventLoopGroupMetrics, 66 | EurekaClient discoveryClient, ApplicationInfoManager applicationInfoManager, 67 | AccessLogPublisher accessLogPublisher, 68 | AbstractConfiguration configurationManager, 69 | HighAvailabilityServices highAvailabilityServices, 70 | MantisCrossRegionalClient mantisCrossRegionalClient, 71 | ConnectionBroker connectionBroker, 72 | @Named("io-scheduler") Scheduler scheduler, 73 | @Named("push-prefixes") List pushPrefixes 74 | ) { 75 | super(serverStatusManager, filterLoader, sessionCtxDecorator, usageNotifier, reqCompleteHandler, registry, 76 | directMemoryMonitor, eventLoopGroupMetrics, discoveryClient, applicationInfoManager, 77 | accessLogPublisher); 78 | this.highAvailabilityServices = highAvailabilityServices; 79 | this.mantisCrossRegionalClient = mantisCrossRegionalClient; 80 | this.connectionBroker = connectionBroker; 81 | this.scheduler = scheduler; 82 | this.pushPrefixes = pushPrefixes; 83 | 84 | // Mantis Master Listener 85 | highAvailabilityServices 86 | .getMasterMonitor() 87 | .getMasterObservable() 88 | .filter(x -> x != null) 89 | .forEach(masterDescription -> { 90 | LOG.info("Received new Mantis Master: " + masterDescription); 91 | configurationManager.setProperty("api.ribbon.listOfServers", 92 | masterDescription.getHostIP() + ":" + masterDescription.getApiPort()); 93 | }); 94 | } 95 | 96 | @Override 97 | protected Map> chooseAddrsAndChannels(ChannelGroup clientChannels) { 98 | Map> addrsToChannels = new HashMap<>(); 99 | 100 | String mainPortName = "main"; 101 | int port = new DynamicIntProperty("zuul.server.port.main", 7001).get(); 102 | NamedSocketAddress sockAddr = new NamedSocketAddress(mainPortName, new InetSocketAddress(port)); 103 | 104 | ChannelConfig channelConfig = defaultChannelConfig(mainPortName); 105 | ChannelConfig channelDependencies = defaultChannelDependencies(mainPortName); 106 | 107 | /* These settings may need to be tweaked depending if you're running behind an ELB HTTP listener, TCP listener, 108 | * or directly on the internet. 109 | */ 110 | channelConfig.set(CommonChannelConfigKeys.allowProxyHeadersWhen, 111 | StripUntrustedProxyHeadersHandler.AllowWhen.ALWAYS); 112 | channelConfig.set(CommonChannelConfigKeys.preferProxyProtocolForClientIp, false); 113 | channelConfig.set(CommonChannelConfigKeys.isSSlFromIntermediary, false); 114 | channelConfig.set(CommonChannelConfigKeys.withProxyProtocol, false); 115 | 116 | addrsToChannels.put( 117 | sockAddr, 118 | new MantisApiServerChannelInitializer( 119 | String.valueOf(port), channelConfig, channelDependencies, clientChannels, pushPrefixes, 120 | highAvailabilityServices, mantisCrossRegionalClient, connectionBroker, 121 | scheduler, false)); 122 | logAddrConfigured(sockAddr); 123 | 124 | return Collections.unmodifiableMap(addrsToChannels); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/Util.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.mantisrx.api; 17 | 18 | import com.google.common.base.Strings; 19 | import io.netty.handler.codec.http.QueryStringDecoder; 20 | import lombok.experimental.UtilityClass; 21 | import lombok.extern.slf4j.Slf4j; 22 | import org.apache.commons.lang3.tuple.ImmutablePair; 23 | import org.slf4j.Logger; 24 | import rx.Observable; 25 | import rx.functions.Func1; 26 | 27 | import java.util.*; 28 | import java.util.concurrent.TimeUnit; 29 | 30 | import static io.mantisrx.api.Constants.*; 31 | 32 | @UtilityClass 33 | @Slf4j 34 | public class Util { 35 | private static final int defaultNumRetries = 2; 36 | 37 | public static boolean startsWithAnyOf(final String target, List prefixes) { 38 | for (String prefix : prefixes) { 39 | if (target.startsWith(prefix)) { 40 | return true; 41 | } 42 | } 43 | return false; 44 | } 45 | 46 | // 47 | // Regions 48 | // 49 | 50 | public static String getLocalRegion() { 51 | return System.getenv("EC2_REGION"); 52 | } 53 | 54 | // 55 | // Query Params 56 | // 57 | 58 | public static String[] getTaglist(String uri, String id) { 59 | return getTaglist(uri, id, null); 60 | } 61 | 62 | public static String[] getTaglist(String uri, String id, String region) { 63 | QueryStringDecoder queryStringDecoder = new QueryStringDecoder(uri); 64 | Map> queryParameters = queryStringDecoder.parameters(); 65 | boolean isClientIdSet = false; 66 | 67 | final List tags = new LinkedList<>(); 68 | if (queryParameters != null) { 69 | List tagVals = queryParameters.get(TagsParamName); 70 | if (tagVals != null) { 71 | for (String s : tagVals) { 72 | StringTokenizer tokenizer = new StringTokenizer(s, TagNameValDelimiter); 73 | if (tokenizer.countTokens() == 2) { 74 | String s1 = tokenizer.nextToken(); 75 | String s2 = tokenizer.nextToken(); 76 | if (s1 != null && !s1.isEmpty() && s2 != null && !s2.isEmpty()) { 77 | tags.add(s1); 78 | tags.add(s2); 79 | if (ClientIdTagName.equals(s1)) { 80 | isClientIdSet = true; 81 | } 82 | } 83 | } 84 | } 85 | } 86 | tagVals = queryParameters.get(ClientIdTagName); 87 | if (!isClientIdSet && tagVals != null && !tagVals.isEmpty()) { 88 | tags.add(ClientIdTagName); 89 | tags.add(tagVals.get(0)); 90 | } 91 | } 92 | 93 | tags.add("SessionId"); 94 | tags.add(id); 95 | 96 | tags.add("urlPath"); 97 | tags.add(queryStringDecoder.path()); 98 | 99 | if (!Strings.isNullOrEmpty(region)) { 100 | tags.add("region"); 101 | tags.add(region); 102 | } 103 | 104 | return tags.toArray(new String[]{}); 105 | } 106 | 107 | // 108 | // Retries 109 | // 110 | 111 | public static Func1, Observable> getRetryFunc(final Logger logger, String name) { 112 | return getRetryFunc(logger, name, defaultNumRetries); 113 | } 114 | 115 | public static Func1, Observable> getRetryFunc(final Logger logger, String name, final int retries) { 116 | final int limit = retries == Integer.MAX_VALUE ? retries : retries + 1; 117 | return attempts -> attempts 118 | .zipWith(Observable.range(1, limit), (t1, integer) -> { 119 | logger.warn("Caught exception connecting for {}.", name, t1); 120 | return new ImmutablePair(t1, integer); 121 | }) 122 | .flatMap(pair -> { 123 | Throwable t = pair.left; 124 | int retryIter = pair.right; 125 | long delay = Math.round(Math.pow(2, retryIter)); 126 | 127 | if (retryIter > retries) { 128 | logger.error("Exceeded maximum retries ({}) for {} with exception: {}", retries, name, t.getMessage(), t); 129 | return Observable.error(new Exception("Timeout after " + retries + " retries")); 130 | } 131 | logger.info("Retrying connection to {} after sleeping for {} seconds.", name, delay, t); 132 | return Observable.timer(delay, TimeUnit.SECONDS); 133 | }); 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/AppStreamDiscovery.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.mantisrx.api.filters; 17 | 18 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper; 19 | import com.google.inject.Inject; 20 | import com.netflix.zuul.filters.http.HttpSyncEndpoint; 21 | import com.netflix.zuul.message.http.HttpHeaderNames; 22 | import com.netflix.zuul.message.http.HttpRequestMessage; 23 | import com.netflix.zuul.message.http.HttpResponseMessage; 24 | import com.netflix.zuul.message.http.HttpResponseMessageImpl; 25 | import io.mantisrx.api.proto.AppDiscoveryMap; 26 | import io.mantisrx.api.services.AppStreamDiscoveryService; 27 | import com.netflix.zuul.stats.status.StatusCategoryUtils; 28 | import com.netflix.zuul.stats.status.ZuulStatusCategory; 29 | import io.netty.handler.codec.http.HttpHeaderValues; 30 | import io.vavr.control.Either; 31 | import io.vavr.control.Try; 32 | import lombok.extern.slf4j.Slf4j; 33 | 34 | import java.util.List; 35 | import java.util.function.Function; 36 | 37 | @Slf4j 38 | public class AppStreamDiscovery extends HttpSyncEndpoint { 39 | 40 | private final AppStreamDiscoveryService appStreamDiscoveryService; 41 | private final ObjectMapper objectMapper; 42 | private static final String APPNAME_QUERY_PARAM = "app"; 43 | 44 | @Inject 45 | public AppStreamDiscovery(AppStreamDiscoveryService appStreamDiscoveryService, 46 | ObjectMapper objectMapper) { 47 | this.appStreamDiscoveryService = appStreamDiscoveryService; 48 | this.objectMapper = objectMapper; 49 | } 50 | 51 | @Override 52 | public HttpResponseMessage apply(HttpRequestMessage request) { 53 | 54 | List apps = request.getQueryParams().get(APPNAME_QUERY_PARAM); 55 | Either result = appStreamDiscoveryService.getAppDiscoveryMap(apps); 56 | 57 | return result.bimap(errorMessage -> { 58 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 500); 59 | resp.setBodyAsText(errorMessage); 60 | StatusCategoryUtils.setStatusCategory(request.getContext(), ZuulStatusCategory.FAILURE_LOCAL); 61 | return resp; 62 | }, appDiscoveryMap -> { 63 | 64 | Try serialized = Try.of(() -> objectMapper.writeValueAsString(appDiscoveryMap)); 65 | 66 | if (serialized.isSuccess()) { 67 | StatusCategoryUtils.setStatusCategory(request.getContext(), ZuulStatusCategory.SUCCESS); 68 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 200); 69 | resp.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_JSON.toString()); 70 | resp.setBodyAsText(serialized.get()); 71 | return resp; 72 | } else { 73 | StatusCategoryUtils.setStatusCategory(request.getContext(), ZuulStatusCategory.FAILURE_LOCAL); 74 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 500); 75 | resp.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.TEXT_PLAIN.toString()); 76 | resp.setBodyAsText(serialized.getOrElseGet(Throwable::getMessage)); 77 | return resp; 78 | } 79 | }).getOrElseGet(Function.identity()); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/Artifacts.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.filters; 2 | 3 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.google.common.base.Strings; 5 | import com.netflix.zuul.filters.http.HttpSyncEndpoint; 6 | import com.netflix.zuul.message.http.HttpHeaderNames; 7 | import com.netflix.zuul.message.http.HttpRequestMessage; 8 | import com.netflix.zuul.message.http.HttpResponseMessage; 9 | import com.netflix.zuul.message.http.HttpResponseMessageImpl; 10 | import io.mantisrx.api.proto.Artifact; 11 | import io.mantisrx.api.services.artifacts.ArtifactManager; 12 | import io.netty.handler.codec.http.HttpHeaderValues; 13 | import io.netty.handler.codec.http.HttpResponseStatus; 14 | import io.vavr.control.Try; 15 | import lombok.extern.slf4j.Slf4j; 16 | 17 | import javax.inject.Inject; 18 | import java.util.List; 19 | import java.util.Optional; 20 | 21 | @Slf4j 22 | public class Artifacts extends HttpSyncEndpoint { 23 | 24 | private final ArtifactManager artifactManager; 25 | private final ObjectMapper objectMapper; 26 | public static final String PATH_SPEC = "/api/v1/artifacts"; 27 | 28 | @Override 29 | public boolean needsBodyBuffered(HttpRequestMessage input) { 30 | return input.getMethod().toLowerCase().equals("post"); 31 | } 32 | 33 | @Inject 34 | public Artifacts(ArtifactManager artifactManager, ObjectMapper objectMapper) { 35 | this.artifactManager = artifactManager; 36 | this.objectMapper = objectMapper; 37 | artifactManager.putArtifact(new Artifact("mantis.json", 0, new byte[0])); 38 | artifactManager.putArtifact(new Artifact("mantis.zip", 0, new byte[0])); 39 | } 40 | 41 | @Override 42 | public HttpResponseMessage apply(HttpRequestMessage request) { 43 | 44 | if (request.getMethod().toLowerCase().equals("get")) { 45 | 46 | String fileName = request.getPath().replaceFirst("^" + PATH_SPEC + "/?", ""); 47 | if (Strings.isNullOrEmpty(fileName)) { 48 | List files = artifactManager 49 | .getArtifacts(); 50 | Try serialized = Try.of(() -> objectMapper.writeValueAsString(files)); 51 | 52 | return serialized.map(body -> { 53 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request, 200); 54 | response.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_JSON.toString()); 55 | response.setBodyAsText(body); 56 | return response; 57 | }).getOrElseGet(t -> { 58 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request, 500); 59 | response.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.TEXT_PLAIN.toString()); 60 | response.setBodyAsText(t.getMessage()); 61 | return response; 62 | }); 63 | } else { 64 | Optional artifact = artifactManager.getArtifact(fileName); 65 | 66 | return artifact.map(art -> { 67 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request, 68 | HttpResponseStatus.OK.code()); 69 | response.setBody(art.getContent()); 70 | response.getHeaders().set(HttpHeaderNames.CONTENT_TYPE, 71 | fileName.endsWith("json") 72 | ? HttpHeaderValues.APPLICATION_JSON.toString() 73 | : HttpHeaderValues.APPLICATION_OCTET_STREAM.toString()); 74 | response.getHeaders().set("Content-Disposition", 75 | String.format("attachment; filename=\"%s\"", fileName)); 76 | return response; 77 | }).orElseGet(() -> { 78 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request, 79 | HttpResponseStatus.NOT_FOUND.code()); 80 | response.setBody(new byte[]{}); 81 | return response; 82 | }); 83 | 84 | } 85 | } else if (request.getMethod().toLowerCase().equals("post")) { 86 | 87 | byte[] body = request.getBody(); 88 | artifactManager.putArtifact(new Artifact("testing.json", body.length, body)); 89 | 90 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request, 91 | HttpResponseStatus.OK.code()); 92 | return response; 93 | 94 | } 95 | 96 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request, HttpResponseStatus.METHOD_NOT_ALLOWED.code()); 97 | response.setBodyAsText(HttpResponseStatus.METHOD_NOT_ALLOWED.reasonPhrase()); 98 | response.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.TEXT_PLAIN.toString()); 99 | return response; 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/Favicon.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.mantisrx.api.filters; 17 | 18 | import com.netflix.zuul.filters.http.HttpSyncEndpoint; 19 | import com.netflix.zuul.message.http.HttpRequestMessage; 20 | import com.netflix.zuul.message.http.HttpResponseMessage; 21 | import com.netflix.zuul.message.http.HttpResponseMessageImpl; 22 | import com.netflix.zuul.stats.status.StatusCategoryUtils; 23 | import com.netflix.zuul.stats.status.ZuulStatusCategory; 24 | 25 | /** 26 | * Returns an empty 200 response to prevent 404s on Favicon. 27 | */ 28 | public class Favicon extends HttpSyncEndpoint { 29 | @Override 30 | public HttpResponseMessage apply(HttpRequestMessage request) { 31 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 200); 32 | resp.setBody(new byte[0]); 33 | StatusCategoryUtils.setStatusCategory(request.getContext(), ZuulStatusCategory.SUCCESS); 34 | return resp; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/Healthcheck.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.mantisrx.api.filters; 17 | 18 | import com.netflix.zuul.filters.http.HttpSyncEndpoint; 19 | import com.netflix.zuul.message.http.HttpRequestMessage; 20 | import com.netflix.zuul.message.http.HttpResponseMessage; 21 | import com.netflix.zuul.message.http.HttpResponseMessageImpl; 22 | import com.netflix.zuul.stats.status.StatusCategoryUtils; 23 | import com.netflix.zuul.stats.status.ZuulStatusCategory; 24 | 25 | public class Healthcheck extends HttpSyncEndpoint { 26 | 27 | @Override 28 | public HttpResponseMessage apply(HttpRequestMessage request) { 29 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 200); 30 | resp.setBodyAsText("mantisapi healthy"); 31 | StatusCategoryUtils.setStatusCategory(request.getContext(), ZuulStatusCategory.SUCCESS); 32 | return resp; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/JobDiscoveryCacheLoader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.mantisrx.api.filters; 17 | 18 | import com.netflix.config.DynamicBooleanProperty; 19 | import com.netflix.zuul.filters.http.HttpOutboundSyncFilter; 20 | import com.netflix.zuul.message.http.HttpResponseMessage; 21 | import io.mantisrx.api.Constants; 22 | import io.mantisrx.api.services.JobDiscoveryService; 23 | import lombok.extern.slf4j.Slf4j; 24 | 25 | @Slf4j 26 | public class JobDiscoveryCacheLoader extends HttpOutboundSyncFilter { 27 | 28 | private static DynamicBooleanProperty cacheEnabled = new DynamicBooleanProperty("mantisapi.cache.enabled", false); 29 | 30 | @Override 31 | public boolean needsBodyBuffered(HttpResponseMessage message) { 32 | return true; 33 | } 34 | 35 | @Override 36 | public int filterOrder() { 37 | return 999; // Don't really care. 38 | } 39 | 40 | @Override 41 | public boolean shouldFilter(HttpResponseMessage response) { 42 | return response.getOutboundRequest().getPath().matches("^/api/v1/jobClusters/.*/latestJobDiscoveryInfo$") 43 | && response.getHeaders().getAll(Constants.MANTISAPI_CACHED_HEADER).isEmpty() 44 | && cacheEnabled.get(); 45 | } 46 | 47 | @Override 48 | public HttpResponseMessage apply(HttpResponseMessage response) { 49 | String jobCluster = response.getOutboundRequest().getPath() 50 | .replaceFirst("^/api/v1/jobClusters/", "") 51 | .replaceFirst("/latestJobDiscoveryInfo$", ""); 52 | 53 | String responseBody = response.getBodyAsText(); 54 | 55 | if (null != responseBody) { 56 | log.info("Caching latest job discovery info for {}.", jobCluster); 57 | JobDiscoveryService.jobDiscoveryInfoCache.put(jobCluster, response.getBodyAsText()); 58 | } 59 | return response; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/JobDiscoveryInfoCacheHitChecker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.mantisrx.api.filters; 17 | 18 | import com.google.common.base.Strings; 19 | import com.netflix.config.DynamicBooleanProperty; 20 | import com.netflix.zuul.filters.http.HttpInboundSyncFilter; 21 | import com.netflix.zuul.message.http.HttpHeaderNames; 22 | import com.netflix.zuul.message.http.HttpRequestMessage; 23 | import com.netflix.zuul.message.http.HttpResponseMessage; 24 | import com.netflix.zuul.message.http.HttpResponseMessageImpl; 25 | import io.mantisrx.api.Constants; 26 | import io.mantisrx.api.services.JobDiscoveryService; 27 | import io.netty.handler.codec.http.HttpHeaderValues; 28 | import lombok.extern.slf4j.Slf4j; 29 | 30 | @Slf4j 31 | public class JobDiscoveryInfoCacheHitChecker extends HttpInboundSyncFilter { 32 | 33 | public static final String PATH_SPEC = "/jobClusters/discoveryInfo"; 34 | private static DynamicBooleanProperty cacheEnabled = new DynamicBooleanProperty("mantisapi.cache.enabled", false); 35 | 36 | @Override 37 | public int filterOrder() { 38 | return -1; 39 | } 40 | 41 | @Override 42 | public boolean shouldFilter(HttpRequestMessage httpRequestMessage) { 43 | String jobCluster = httpRequestMessage.getPath().replaceFirst(PATH_SPEC + "/", ""); 44 | return httpRequestMessage.getPath().startsWith(PATH_SPEC) 45 | && JobDiscoveryService.jobDiscoveryInfoCache.getIfPresent(jobCluster) != null; 46 | } 47 | 48 | @Override 49 | public HttpRequestMessage apply(HttpRequestMessage request) { 50 | String jobCluster = request.getPath().replaceFirst(PATH_SPEC + "/", ""); 51 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 200); 52 | String bodyText = JobDiscoveryService.jobDiscoveryInfoCache.getIfPresent(jobCluster) ; 53 | if (cacheEnabled.get() && !Strings.isNullOrEmpty(bodyText)) { 54 | log.info("Serving cached job discovery info for {}.", jobCluster); 55 | resp.setBodyAsText(bodyText); 56 | resp.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_JSON.toString()); 57 | resp.getHeaders().set(Constants.MANTISAPI_CACHED_HEADER, "true"); 58 | request.getContext().setStaticResponse(resp); 59 | } 60 | return request; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/MQLParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.api.filters; 18 | 19 | import io.mantisrx.shaded.com.fasterxml.jackson.core.JsonProcessingException; 20 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper; 21 | import com.netflix.zuul.filters.http.HttpSyncEndpoint; 22 | import com.netflix.zuul.message.http.HttpRequestMessage; 23 | import com.netflix.zuul.message.http.HttpResponseMessage; 24 | import com.netflix.zuul.message.http.HttpResponseMessageImpl; 25 | import io.mantisrx.mql.shaded.clojure.java.api.Clojure; 26 | import io.mantisrx.mql.shaded.clojure.lang.IFn; 27 | import lombok.Value; 28 | import lombok.extern.slf4j.Slf4j; 29 | 30 | import java.nio.charset.Charset; 31 | 32 | 33 | @Slf4j 34 | public class MQLParser extends HttpSyncEndpoint { 35 | 36 | private static IFn require = Clojure.var("io.mantisrx.mql.shaded.clojure.core", "require"); 37 | static { 38 | require.invoke(Clojure.read("io.mantisrx.mql.core")); 39 | require.invoke(Clojure.read("io.mantisrx.mql.jvm.interfaces.server")); 40 | require.invoke(Clojure.read("io.mantisrx.mql.jvm.interfaces.core")); 41 | } 42 | private static IFn parses = Clojure.var("io.mantisrx.mql.jvm.interfaces.core", "parses?"); 43 | private static IFn getParseError = Clojure.var("io.mantisrx.mql.jvm.interfaces.core", "get-parse-error"); 44 | 45 | private static final ObjectMapper objectMapper = new ObjectMapper(); 46 | 47 | public @Value class MQLParseResult { 48 | private boolean success; 49 | private String criterion; 50 | private String message; 51 | } 52 | 53 | @Override 54 | public HttpResponseMessage apply(HttpRequestMessage input) { 55 | String query = input.getQueryParams().getFirst("criterion"); 56 | 57 | boolean parses = parses(query); 58 | String parseError = getParseError(query); 59 | MQLParseResult result = new MQLParseResult(parses, query, parses ? "" : parseError); 60 | 61 | try { 62 | HttpResponseMessage response = new HttpResponseMessageImpl(input.getContext(), input, 200); 63 | response.setBody(objectMapper.writeValueAsBytes(result)); 64 | return response; 65 | 66 | } catch (JsonProcessingException ex) { 67 | HttpResponseMessage response = new HttpResponseMessageImpl(input.getContext(), input, 500); 68 | response.setBody(getErrorResponse(ex.getMessage()).getBytes(Charset.defaultCharset())); 69 | return response; 70 | } 71 | } 72 | 73 | /** 74 | * A predicate which indicates whether or not the MQL parser considers query to be a valid query. 75 | * @param query A String representing the MQL query. 76 | * @return A boolean indicating whether or not the query successfully parses. 77 | */ 78 | public static Boolean parses(String query) { 79 | return (Boolean) parses.invoke(query); 80 | } 81 | 82 | /** 83 | * A convenience function allowing a caller to determine what went wrong if a call to #parses(String query) returns 84 | * false. 85 | * @param query A String representing the MQL query. 86 | * @return A String representing the parse error for an MQL query, null if no parse error occurred. 87 | */ 88 | public static String getParseError(String query) { 89 | return (String) getParseError.invoke(query); 90 | } 91 | 92 | private String getErrorResponse(String exceptionMessage) { 93 | StringBuilder sb = new StringBuilder(50); 94 | sb.append("{\"success\": false, \"messages\": \""); 95 | sb.append(exceptionMessage); 96 | sb.append("\"}"); 97 | return sb.toString(); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/MREAppStreamToJobClusterMapping.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.mantisrx.api.filters; 17 | 18 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper; 19 | import io.mantisrx.discovery.proto.AppJobClustersMap; 20 | import com.netflix.zuul.filters.http.HttpSyncEndpoint; 21 | import com.netflix.zuul.message.http.HttpHeaderNames; 22 | import com.netflix.zuul.message.http.HttpRequestMessage; 23 | import com.netflix.zuul.message.http.HttpResponseMessage; 24 | import com.netflix.zuul.message.http.HttpResponseMessageImpl; 25 | import io.mantisrx.api.services.AppStreamDiscoveryService; 26 | import io.mantisrx.shaded.com.google.common.base.Preconditions; 27 | import io.netty.handler.codec.http.HttpHeaderValues; 28 | import io.vavr.control.Try; 29 | 30 | import java.util.List; 31 | import javax.inject.Inject; 32 | 33 | public class MREAppStreamToJobClusterMapping extends HttpSyncEndpoint { 34 | 35 | private final AppStreamDiscoveryService appStreamDiscoveryService; 36 | private final ObjectMapper objectMapper; 37 | 38 | private static final String APPNAME_QUERY_PARAM = "app"; 39 | public static final String PATH_SPEC = "/api/v1/mantis/publish/streamJobClusterMap"; 40 | 41 | @Inject 42 | public MREAppStreamToJobClusterMapping(AppStreamDiscoveryService appStreamDiscoveryService, 43 | ObjectMapper objectMapper) { 44 | Preconditions.checkArgument(appStreamDiscoveryService != null, "appStreamDiscoveryService cannot be null"); 45 | this.appStreamDiscoveryService = appStreamDiscoveryService; 46 | Preconditions.checkArgument(objectMapper != null, "objectMapper cannot be null"); 47 | this.objectMapper = objectMapper; 48 | } 49 | 50 | @Override 51 | public HttpResponseMessage apply(HttpRequestMessage request) { 52 | List apps = request.getQueryParams().get(APPNAME_QUERY_PARAM); 53 | Try payloadTry = Try.ofCallable(() -> appStreamDiscoveryService.getAppJobClustersMap(apps)); 54 | 55 | Try serialized = payloadTry.flatMap(payload -> Try.of(() -> objectMapper.writeValueAsString(payload))); 56 | 57 | return serialized.map(body -> { 58 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 200); 59 | resp.setBodyAsText(body); 60 | resp.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_JSON.toString()); 61 | return resp; 62 | }).getOrElseGet(t -> { 63 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 200); 64 | resp.setBodyAsText(t.getMessage()); 65 | resp.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.TEXT_PLAIN.toString()); 66 | return resp; 67 | }); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/MasterCacheHitChecker.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.filters; 2 | 3 | import com.google.inject.Inject; 4 | import com.google.inject.name.Named; 5 | import com.netflix.config.DynamicBooleanProperty; 6 | import com.netflix.spectator.api.Counter; 7 | import com.netflix.zuul.filters.http.HttpInboundSyncFilter; 8 | import com.netflix.zuul.message.http.HttpHeaderNames; 9 | import com.netflix.zuul.message.http.HttpRequestMessage; 10 | import com.netflix.zuul.message.http.HttpResponseMessage; 11 | import com.netflix.zuul.message.http.HttpResponseMessageImpl; 12 | import com.netflix.zuul.netty.SpectatorUtils; 13 | import io.mantisrx.api.Constants; 14 | import io.mantisrx.api.Util; 15 | import io.netty.handler.codec.http.HttpHeaderValues; 16 | import lombok.extern.slf4j.Slf4j; 17 | 18 | import java.util.List; 19 | import java.util.concurrent.ConcurrentHashMap; 20 | 21 | @Slf4j 22 | public class MasterCacheHitChecker extends HttpInboundSyncFilter { 23 | 24 | private static DynamicBooleanProperty cacheEnabled = new DynamicBooleanProperty("mantisapi.cache.enabled", false); 25 | private static final ConcurrentHashMap cacheHitCounters = new ConcurrentHashMap<>(500); 26 | private static final ConcurrentHashMap cacheMissCounters = new ConcurrentHashMap<>(500); 27 | private static final String CACHE_HIT_COUNTER_NAME = "mantis.api.cache.count"; 28 | private final List pushPrefixes; 29 | 30 | @Inject 31 | public MasterCacheHitChecker(@Named("push-prefixes") List pushPrefixes) { 32 | super(); 33 | this.pushPrefixes = pushPrefixes; 34 | } 35 | 36 | @Override 37 | public HttpRequestMessage apply(HttpRequestMessage request) { 38 | if(cacheEnabled.get()) { 39 | String key = request.getPathAndQuery(); 40 | String bodyText = MasterCacheLoader.masterCache.getIfPresent(key); 41 | 42 | if (bodyText != null) { // Cache Hit 43 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request, 200); 44 | response.setBodyAsText(bodyText); 45 | response.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_JSON.toString()); 46 | response.getHeaders().set(Constants.MANTISAPI_CACHED_HEADER, "true"); 47 | request.getContext().setStaticResponse(response); 48 | 49 | cacheHitCounters.computeIfAbsent(key, 50 | k -> SpectatorUtils.newCounter(CACHE_HIT_COUNTER_NAME, "api", "endpoint", k, "class", "hit")) 51 | .increment(); 52 | } else { // Cache Miss 53 | cacheMissCounters.computeIfAbsent(key, 54 | k -> SpectatorUtils.newCounter(CACHE_HIT_COUNTER_NAME, "api", "endpoint", k, "class", "miss")) 55 | .increment(); 56 | } 57 | } 58 | 59 | return request; 60 | } 61 | 62 | @Override 63 | public int filterOrder() { 64 | return 0; 65 | } 66 | 67 | @Override 68 | public boolean shouldFilter(HttpRequestMessage msg) { 69 | String key = msg.getPathAndQuery(); 70 | 71 | return msg.getMethod().equalsIgnoreCase("get") 72 | && key.startsWith("/api") 73 | && !Util.startsWithAnyOf(key, pushPrefixes); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/MasterCacheLoader.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.filters; 2 | 3 | import com.google.common.cache.Cache; 4 | import com.google.common.cache.CacheBuilder; 5 | import com.google.common.cache.CacheStats; 6 | import com.google.inject.Inject; 7 | import com.netflix.config.DynamicBooleanProperty; 8 | import com.netflix.config.DynamicIntProperty; 9 | import com.netflix.spectator.api.BasicTag; 10 | import com.netflix.spectator.api.Registry; 11 | import com.netflix.spectator.api.Spectator; 12 | import com.netflix.spectator.api.patterns.PolledMeter; 13 | import com.netflix.zuul.filters.http.HttpOutboundSyncFilter; 14 | import com.netflix.zuul.message.http.HttpResponseMessage; 15 | import io.mantisrx.api.Constants; 16 | import lombok.extern.slf4j.Slf4j; 17 | 18 | import java.util.concurrent.TimeUnit; 19 | 20 | @Slf4j 21 | public class MasterCacheLoader extends HttpOutboundSyncFilter { 22 | 23 | @Override 24 | public boolean needsBodyBuffered(HttpResponseMessage message) { 25 | return true; 26 | } 27 | 28 | private static DynamicBooleanProperty cacheEnabled = new DynamicBooleanProperty("mantisapi.cache.enabled", false); 29 | private static DynamicIntProperty cacheSize = new DynamicIntProperty("mantisapi.cache.size", 1000); 30 | private static DynamicIntProperty cacheDurationSeconds = new DynamicIntProperty("mantisapi.cache.seconds", 1); 31 | 32 | public static final Cache masterCache = CacheBuilder.newBuilder() 33 | .maximumSize(cacheSize.get()) 34 | .expireAfterWrite(cacheDurationSeconds.get(), TimeUnit.SECONDS) 35 | .build(); 36 | 37 | @Inject 38 | public MasterCacheLoader(Registry registry) { 39 | CacheStats stats = masterCache.stats(); 40 | PolledMeter.using(registry) 41 | .withName("mantis.api.cache.size") 42 | .withTag(new BasicTag("id", "api")) 43 | .monitorMonotonicCounter(masterCache, Cache::size); 44 | 45 | PolledMeter.using(registry) 46 | .withName("mantis.api.cache.hitCount") 47 | .withTag(new BasicTag("id", "api")) 48 | .monitorMonotonicCounter(stats, CacheStats::hitCount); 49 | 50 | PolledMeter.using(registry) 51 | .withName("mantis.api.cache.missCount") 52 | .withTag(new BasicTag("id", "api")) 53 | .monitorMonotonicCounter(stats, CacheStats::missCount); 54 | } 55 | 56 | @Override 57 | public HttpResponseMessage apply(HttpResponseMessage input) { 58 | String key = input.getInboundRequest().getPathAndQuery(); 59 | String responseBody = input.getBodyAsText(); 60 | 61 | if (null != responseBody && cacheEnabled.get()) { 62 | masterCache.put(key, responseBody); 63 | } 64 | 65 | return input; 66 | } 67 | 68 | @Override 69 | public int filterOrder() { 70 | return 999; 71 | } 72 | 73 | @Override 74 | public boolean shouldFilter(HttpResponseMessage msg) { 75 | return msg.getOutboundRequest().getContext().getRouteVIP() != null 76 | && msg.getOutboundRequest().getContext().getRouteVIP().equalsIgnoreCase("api") 77 | && msg.getInboundRequest().getMethod().equalsIgnoreCase("get") 78 | && msg.getHeaders().getAll(Constants.MANTISAPI_CACHED_HEADER).size() == 0; // Set by the MasterCacheHitChecker, ensures we aren't re-caching. 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/MetricsReporting.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.filters; 2 | 3 | import com.netflix.spectator.api.Counter; 4 | import com.netflix.spectator.api.Timer; 5 | import com.netflix.zuul.filters.http.HttpOutboundSyncFilter; 6 | import com.netflix.zuul.message.http.HttpResponseMessage; 7 | import com.netflix.zuul.netty.SpectatorUtils; 8 | import io.vavr.Tuple; 9 | import io.vavr.Tuple2; 10 | 11 | import java.util.concurrent.ConcurrentHashMap; 12 | import java.util.concurrent.TimeUnit; 13 | 14 | public class MetricsReporting extends HttpOutboundSyncFilter { 15 | 16 | private static final ConcurrentHashMap, Timer> timerCache = new ConcurrentHashMap<>(500); 17 | private static final ConcurrentHashMap, Counter> counterCache = new ConcurrentHashMap<>(500); 18 | 19 | @Override 20 | public HttpResponseMessage apply(HttpResponseMessage input) { 21 | String path = input.getInboundRequest().getPath(); 22 | String status = statusCodeToStringRepresentation(input.getStatus()); 23 | 24 | // Record Latency. Zuul no longer record total request time. 25 | timerCache.computeIfAbsent(Tuple.of(path, status), 26 | tuple -> SpectatorUtils.newTimer("latency", path,"status", status)) 27 | .record(input.getContext().getOriginReportedDuration(), TimeUnit.NANOSECONDS); 28 | 29 | // Record Request 30 | counterCache.computeIfAbsent(Tuple.of(path, status), 31 | tuple -> SpectatorUtils.newCounter("requests", path, "status", status)) 32 | .increment(); 33 | 34 | return input; 35 | } 36 | 37 | private String statusCodeToStringRepresentation(Integer statusCode) { 38 | return (statusCode / 100) + "xx"; 39 | } 40 | 41 | @Override 42 | public int filterOrder() { 43 | return -100; 44 | } 45 | 46 | @Override 47 | public boolean shouldFilter(HttpResponseMessage msg) { 48 | return true; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/Options.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.filters; 2 | 3 | import com.netflix.zuul.filters.http.HttpSyncEndpoint; 4 | import com.netflix.zuul.message.http.HttpRequestMessage; 5 | import com.netflix.zuul.message.http.HttpResponseMessage; 6 | import com.netflix.zuul.message.http.HttpResponseMessageImpl; 7 | import com.netflix.zuul.stats.status.StatusCategoryUtils; 8 | import com.netflix.zuul.stats.status.ZuulStatusCategory; 9 | import io.netty.handler.codec.http.HttpHeaderNames; 10 | import io.netty.handler.codec.http.HttpResponseStatus; 11 | 12 | public class Options extends HttpSyncEndpoint { 13 | 14 | @Override 15 | public HttpResponseMessage apply(HttpRequestMessage request) { 16 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, HttpResponseStatus.OK.code()); 17 | resp.setBodyAsText(""); 18 | StatusCategoryUtils.setStatusCategory(request.getContext(), ZuulStatusCategory.SUCCESS); 19 | return resp; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/OutboundHeaders.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.filters; 2 | 3 | import com.netflix.zuul.filters.http.HttpOutboundSyncFilter; 4 | import com.netflix.zuul.message.HeaderName; 5 | import com.netflix.zuul.message.http.HttpResponseMessage; 6 | import io.netty.handler.codec.http.HttpHeaderNames; 7 | import io.netty.util.AsciiString; 8 | 9 | public class OutboundHeaders extends HttpOutboundSyncFilter { 10 | 11 | @Override 12 | public boolean shouldFilter(HttpResponseMessage msg) { 13 | return true; 14 | } 15 | 16 | @Override 17 | public HttpResponseMessage apply(HttpResponseMessage resp) { 18 | upsert(resp, HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN, "*"); 19 | addHeaderIfMissing(resp, HttpHeaderNames.ACCESS_CONTROL_ALLOW_HEADERS, 20 | "Origin, X-Requested-With, Accept, Content-Type, Cache-Control"); 21 | addHeaderIfMissing(resp, HttpHeaderNames.ACCESS_CONTROL_ALLOW_METHODS, 22 | "GET, OPTIONS, PUT, POST, DELETE, CONNECT"); 23 | addHeaderIfMissing(resp, HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS, "true"); 24 | return resp; 25 | } 26 | 27 | private void upsert(HttpResponseMessage resp, AsciiString name, String value) { 28 | resp.getHeaders().remove(new HeaderName(name.toString())); 29 | resp.getHeaders().add(new HeaderName(name.toString()), value); 30 | } 31 | 32 | private void addHeaderIfMissing(HttpResponseMessage resp, AsciiString name, String value) { 33 | if (resp.getHeaders().getAll(name.toString()).size() == 0) { 34 | resp.getHeaders().add(name.toString(), value); 35 | } 36 | } 37 | 38 | @Override 39 | public int filterOrder() { 40 | return 0; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/filters/Routes.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.mantisrx.api.filters; 17 | 18 | import com.netflix.zuul.context.SessionContext; 19 | import com.netflix.zuul.filters.http.HttpInboundSyncFilter; 20 | import com.netflix.zuul.message.http.HttpRequestMessage; 21 | import com.netflix.zuul.netty.filter.ZuulEndPointRunner; 22 | import lombok.extern.slf4j.Slf4j; 23 | 24 | @Slf4j 25 | public class Routes extends HttpInboundSyncFilter { 26 | 27 | @Override 28 | public int filterOrder() { 29 | return 0; 30 | } 31 | 32 | @Override 33 | public boolean shouldFilter(HttpRequestMessage httpRequestMessage) { 34 | return true; 35 | } 36 | 37 | @Override 38 | public HttpRequestMessage apply(HttpRequestMessage request) { 39 | SessionContext context = request.getContext(); 40 | String path = request.getPath(); 41 | String host = request.getOriginalHost(); 42 | 43 | if (request.getMethod().toLowerCase().equals("options")) { 44 | context.setEndpoint(Options.class.getCanonicalName()); 45 | } else if (path.equalsIgnoreCase("/healthcheck")) { 46 | context.setEndpoint(Healthcheck.class.getCanonicalName()); 47 | } else if (path.equalsIgnoreCase("/favicon.ico")) { 48 | context.setEndpoint(Favicon.class.getCanonicalName()); 49 | } else if (path.startsWith(Artifacts.PATH_SPEC)) { 50 | context.setEndpoint(Artifacts.class.getCanonicalName()); 51 | } else if (path.equalsIgnoreCase("/api/v1/mantis/publish/streamDiscovery")) { 52 | context.setEndpoint(AppStreamDiscovery.class.getCanonicalName()); 53 | } else if (path.startsWith("/jobClusters/discoveryInfo")) { 54 | String jobCluster = request.getPath().replaceFirst(JobDiscoveryInfoCacheHitChecker.PATH_SPEC + "/", ""); 55 | String newUrl = "/api/v1/jobClusters/" + jobCluster + "/latestJobDiscoveryInfo"; 56 | request.setPath(newUrl); 57 | context.setEndpoint(ZuulEndPointRunner.PROXY_ENDPOINT_FILTER_NAME); 58 | context.setRouteVIP("api"); 59 | } else if (path.equalsIgnoreCase("/api/v1/mql/parse")) { 60 | context.setEndpoint(MQLParser.class.getCanonicalName()); 61 | } else if (path.equals(MREAppStreamToJobClusterMapping.PATH_SPEC)) { 62 | context.setEndpoint(MREAppStreamToJobClusterMapping.class.getCanonicalName()); 63 | } else { 64 | context.setEndpoint(ZuulEndPointRunner.PROXY_ENDPOINT_FILTER_NAME); 65 | context.setRouteVIP("api"); 66 | } 67 | 68 | return request; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/initializers/MantisApiServerChannelInitializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.mantisrx.api.initializers; 17 | 18 | import com.netflix.netty.common.HttpLifecycleChannelHandler; 19 | import com.netflix.netty.common.channel.config.ChannelConfig; 20 | import com.netflix.netty.common.channel.config.CommonChannelConfigKeys; 21 | import com.netflix.zuul.netty.server.BaseZuulChannelInitializer; 22 | import com.netflix.zuul.netty.ssl.SslContextFactory; 23 | import io.mantisrx.api.Util; 24 | import io.mantisrx.api.push.ConnectionBroker; 25 | import io.mantisrx.api.push.MantisSSEHandler; 26 | import io.mantisrx.api.push.MantisWebSocketFrameHandler; 27 | import io.mantisrx.api.tunnel.CrossRegionHandler; 28 | import io.mantisrx.api.tunnel.MantisCrossRegionalClient; 29 | import io.mantisrx.server.master.client.HighAvailabilityServices; 30 | import io.netty.channel.Channel; 31 | import io.netty.channel.ChannelHandlerContext; 32 | import io.netty.channel.ChannelInboundHandlerAdapter; 33 | import io.netty.channel.ChannelPipeline; 34 | import io.netty.channel.group.ChannelGroup; 35 | import io.netty.handler.codec.http.HttpObjectAggregator; 36 | import io.netty.handler.codec.http.websocketx.WebSocketServerProtocolHandler; 37 | import io.netty.handler.ssl.SslContext; 38 | import io.netty.handler.ssl.SslHandler; 39 | import io.netty.handler.stream.ChunkedWriteHandler; 40 | import java.util.List; 41 | import javax.net.ssl.SSLException; 42 | import rx.Scheduler; 43 | 44 | 45 | public class MantisApiServerChannelInitializer extends BaseZuulChannelInitializer 46 | { 47 | private final SslContextFactory sslContextFactory; 48 | private final SslContext sslContext; 49 | private final boolean isSSlFromIntermediary; 50 | 51 | private final ConnectionBroker connectionBroker; 52 | private final HighAvailabilityServices highAvailabilityServices; 53 | private final MantisCrossRegionalClient mantisCrossRegionalClient; 54 | private final Scheduler scheduler; 55 | private final List pushPrefixes; 56 | private final boolean sslEnabled; 57 | 58 | public MantisApiServerChannelInitializer( 59 | String metricId, 60 | ChannelConfig channelConfig, 61 | ChannelConfig channelDependencies, 62 | ChannelGroup channels, 63 | List pushPrefixes, 64 | HighAvailabilityServices highAvailabilityServices, 65 | MantisCrossRegionalClient mantisCrossRegionalClient, 66 | ConnectionBroker connectionBroker, 67 | Scheduler scheduler, 68 | boolean sslEnabled) { 69 | super(metricId, channelConfig, channelDependencies, channels); 70 | 71 | this.pushPrefixes = pushPrefixes; 72 | this.connectionBroker = connectionBroker; 73 | this.highAvailabilityServices = highAvailabilityServices; 74 | this.mantisCrossRegionalClient = mantisCrossRegionalClient; 75 | this.scheduler = scheduler; 76 | this.sslEnabled = sslEnabled; 77 | 78 | this.isSSlFromIntermediary = channelConfig.get(CommonChannelConfigKeys.isSSlFromIntermediary); 79 | this.sslContextFactory = channelConfig.get(CommonChannelConfigKeys.sslContextFactory); 80 | 81 | if (sslEnabled) { 82 | try { 83 | sslContext = sslContextFactory.createBuilderForServer().build(); 84 | } catch (SSLException e) { 85 | throw new RuntimeException("Error configuring SslContext!", e); 86 | } 87 | 88 | // Enable TLS Session Tickets support. 89 | sslContextFactory.enableSessionTickets(sslContext); 90 | 91 | // Setup metrics tracking the OpenSSL stats. 92 | sslContextFactory.configureOpenSslStatsMetrics(sslContext, metricId); 93 | } else { 94 | sslContext = null; 95 | } 96 | } 97 | 98 | 99 | 100 | 101 | @Override 102 | protected void initChannel(Channel ch) throws Exception 103 | { 104 | 105 | // Configure our pipeline of ChannelHandlerS. 106 | ChannelPipeline pipeline = ch.pipeline(); 107 | 108 | storeChannel(ch); 109 | addTimeoutHandlers(pipeline); 110 | addPassportHandler(pipeline); 111 | addTcpRelatedHandlers(pipeline); 112 | 113 | if (sslEnabled) { 114 | SslHandler sslHandler = sslContext.newHandler(ch.alloc()); 115 | sslHandler.engine().setEnabledProtocols(sslContextFactory.getProtocols()); 116 | pipeline.addLast("ssl", sslHandler); 117 | addSslInfoHandlers(pipeline, isSSlFromIntermediary); 118 | addSslClientCertChecks(pipeline); 119 | } 120 | 121 | addHttp1Handlers(pipeline); 122 | addHttpRelatedHandlers(pipeline); 123 | 124 | pipeline.addLast("mantishandler", new MantisChannelHandler(pushPrefixes)); 125 | } 126 | 127 | /** 128 | * Adds a series of handlers for providing SSE/Websocket connections 129 | * to Mantis Jobs. 130 | * 131 | * @param pipeline The netty pipeline to which push handlers should be added. 132 | * @param url The url with which to initiate the websocket handler. 133 | */ 134 | protected void addPushHandlers(final ChannelPipeline pipeline, String url) { 135 | pipeline.addLast(new ChunkedWriteHandler()); 136 | pipeline.addLast(new HttpObjectAggregator(64 * 1024)); 137 | pipeline.addLast(new MantisSSEHandler(connectionBroker, highAvailabilityServices, pushPrefixes)); 138 | pipeline.addLast(new WebSocketServerProtocolHandler(url, true)); 139 | pipeline.addLast(new MantisWebSocketFrameHandler(connectionBroker)); 140 | } 141 | 142 | /** 143 | * Adds a series of handlers for providing SSE/Websocket connections 144 | * to Mantis Jobs. 145 | * 146 | * @param pipeline The netty pipeline to which regional handlers should be added. 147 | */ 148 | protected void addRegionalHandlers(final ChannelPipeline pipeline) { 149 | pipeline.addLast(new ChunkedWriteHandler()); 150 | pipeline.addLast(new HttpObjectAggregator(10 * 1024 * 1024)); 151 | pipeline.addLast(new CrossRegionHandler(pushPrefixes, mantisCrossRegionalClient, connectionBroker, scheduler)); 152 | } 153 | 154 | /** 155 | * The MantisChannelHandler's job is to initialize the tail end of the pipeline differently 156 | * depending on the URI of the request. This is largely to circumvent issues with endpoint responses 157 | * when the push handlers preceed the Zuul handlers. 158 | */ 159 | @Sharable 160 | public class MantisChannelHandler extends ChannelInboundHandlerAdapter { 161 | 162 | private final List pushPrefixes; 163 | 164 | public MantisChannelHandler(List pushPrefixes) { 165 | this.pushPrefixes = pushPrefixes; 166 | } 167 | 168 | @Override 169 | public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { 170 | if (evt instanceof HttpLifecycleChannelHandler.StartEvent) { 171 | HttpLifecycleChannelHandler.StartEvent startEvent = (HttpLifecycleChannelHandler.StartEvent) evt; 172 | String uri = startEvent.getRequest().uri(); 173 | ChannelPipeline pipeline = ctx.pipeline(); 174 | 175 | removeEverythingAfterThis(pipeline); 176 | 177 | if (Util.startsWithAnyOf(uri, this.pushPrefixes)) { 178 | addPushHandlers(pipeline, uri); 179 | } else if(uri.startsWith("/region/")) { 180 | addRegionalHandlers(pipeline); 181 | } else { 182 | addZuulHandlers(pipeline); 183 | } 184 | } 185 | ctx.fireUserEventTriggered(evt); 186 | } 187 | } 188 | 189 | private void removeEverythingAfterThis(ChannelPipeline pipeline) { 190 | while (pipeline.last().getClass() != MantisChannelHandler.class) { 191 | pipeline.removeLast(); 192 | } 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/proto/AppDiscoveryMap.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.mantisrx.api.proto; 17 | 18 | import io.mantisrx.server.core.JobSchedulingInfo; 19 | 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | 23 | public class AppDiscoveryMap { 24 | public final String version; 25 | public final Long timestamp; 26 | public final Map> mappings = new HashMap<>(); 27 | 28 | public AppDiscoveryMap(String version, Long timestamp) { 29 | this.version = version; 30 | this.timestamp = timestamp; 31 | } 32 | 33 | public void addMapping(String app, String stream, JobSchedulingInfo schedulingInfo) { 34 | if(!mappings.containsKey(app)) { 35 | mappings.put(app, new HashMap()); 36 | } 37 | mappings.get(app).put(stream, schedulingInfo); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/proto/Artifact.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.api.proto; 18 | 19 | import java.util.Objects; 20 | 21 | 22 | public class Artifact { 23 | private long sizeInBytes; 24 | private String fileName; 25 | private byte[] content; 26 | 27 | public Artifact(String fileName, long sizeInBytes, byte[] content) { 28 | Objects.requireNonNull(fileName, "File name cannot be null"); 29 | Objects.requireNonNull(content, "Content cannot be null"); 30 | this.fileName = fileName; 31 | this.sizeInBytes = sizeInBytes; 32 | this.content = content; 33 | } 34 | 35 | public long getSizeInBytes() { 36 | return this.sizeInBytes; 37 | } 38 | 39 | public byte[] getContent() { 40 | return content; 41 | } 42 | 43 | public String getFileName() { 44 | return fileName; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/push/ConnectionBroker.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.push; 2 | 3 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.google.inject.Inject; 5 | import com.google.inject.Singleton; 6 | import com.google.inject.name.Named; 7 | import com.netflix.spectator.api.Counter; 8 | import com.netflix.zuul.netty.SpectatorUtils; 9 | import io.mantisrx.api.Constants; 10 | import io.mantisrx.api.Util; 11 | import io.mantisrx.api.services.JobDiscoveryService; 12 | import io.mantisrx.api.tunnel.MantisCrossRegionalClient; 13 | import io.mantisrx.client.MantisClient; 14 | import io.mantisrx.client.SinkConnectionFunc; 15 | import io.mantisrx.client.SseSinkConnectionFunction; 16 | import io.mantisrx.common.MantisServerSentEvent; 17 | import io.mantisrx.runtime.parameter.SinkParameters; 18 | import io.mantisrx.server.worker.client.MetricsClient; 19 | import io.mantisrx.server.worker.client.SseWorkerConnectionFunction; 20 | import io.mantisrx.server.worker.client.WorkerConnectionsStatus; 21 | import io.mantisrx.server.worker.client.WorkerMetricsClient; 22 | import io.vavr.control.Try; 23 | import lombok.extern.slf4j.Slf4j; 24 | import mantis.io.reactivex.netty.protocol.http.client.HttpClientRequest; 25 | import mantis.io.reactivex.netty.protocol.http.client.HttpClientResponse; 26 | import mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent; 27 | import rx.Observable; 28 | import rx.Observer; 29 | import rx.Scheduler; 30 | import rx.functions.Action1; 31 | import rx.schedulers.Schedulers; 32 | 33 | import java.util.List; 34 | import java.util.Map; 35 | import java.util.WeakHashMap; 36 | import java.util.concurrent.TimeUnit; 37 | import java.util.concurrent.atomic.AtomicBoolean; 38 | 39 | import static io.mantisrx.api.Constants.TunnelPingMessage; 40 | import static io.mantisrx.api.Util.getLocalRegion; 41 | 42 | @Slf4j 43 | @Singleton 44 | public class ConnectionBroker { 45 | 46 | private final MantisClient mantisClient; 47 | private final MantisCrossRegionalClient mantisCrossRegionalClient; 48 | private final WorkerMetricsClient workerMetricsClient; 49 | private final JobDiscoveryService jobDiscoveryService; 50 | private final Scheduler scheduler; 51 | private final ObjectMapper objectMapper; 52 | 53 | private final Map> connectionCache = new WeakHashMap<>(); 54 | 55 | @Inject 56 | public ConnectionBroker(MantisClient mantisClient, 57 | MantisCrossRegionalClient mantisCrossRegionalClient, 58 | WorkerMetricsClient workerMetricsClient, 59 | @Named("io-scheduler") Scheduler scheduler, 60 | ObjectMapper objectMapper) { 61 | this.mantisClient = mantisClient; 62 | this.mantisCrossRegionalClient = mantisCrossRegionalClient; 63 | this.workerMetricsClient = workerMetricsClient; 64 | this.jobDiscoveryService = JobDiscoveryService.getInstance(mantisClient, scheduler); 65 | this.scheduler = scheduler; 66 | this.objectMapper = objectMapper; 67 | } 68 | 69 | public Observable connect(PushConnectionDetails details) { 70 | 71 | if (!connectionCache.containsKey(details)) { 72 | switch (details.type) { 73 | case CONNECT_BY_NAME: 74 | return getConnectByNameFor(details) 75 | .subscribeOn(scheduler) 76 | .doOnUnsubscribe(() -> { 77 | log.info("Purging {} from cache.", details); 78 | connectionCache.remove(details); 79 | }) 80 | .doOnCompleted(() -> { 81 | log.info("Purging {} from cache.", details); 82 | connectionCache.remove(details); 83 | }) 84 | .share(); 85 | case CONNECT_BY_ID: 86 | return getConnectByIdFor(details) 87 | .subscribeOn(scheduler) 88 | .doOnUnsubscribe(() -> { 89 | log.info("Purging {} from cache.", details); 90 | connectionCache.remove(details); 91 | }) 92 | .doOnCompleted(() -> { 93 | log.info("Purging {} from cache.", details); 94 | connectionCache.remove(details); 95 | }) 96 | .share(); 97 | 98 | case METRICS: 99 | return getWorkerMetrics(details) 100 | .subscribeOn(scheduler) 101 | .doOnUnsubscribe(() -> { 102 | log.info("Purging {} from cache.", details); 103 | connectionCache.remove(details); 104 | }) 105 | .doOnCompleted(() -> { 106 | log.info("Purging {} from cache.", details); 107 | connectionCache.remove(details); 108 | }); 109 | 110 | case JOB_STATUS: 111 | connectionCache.put(details, 112 | mantisClient 113 | .getJobStatusObservable(details.target) 114 | .subscribeOn(scheduler) 115 | .doOnCompleted(() -> { 116 | log.info("Purging {} from cache.", details); 117 | connectionCache.remove(details); 118 | }) 119 | .doOnUnsubscribe(() -> { 120 | log.info("Purging {} from cache.", details); 121 | connectionCache.remove(details); 122 | }) 123 | .replay(25) 124 | .autoConnect()); 125 | break; 126 | case JOB_SCHEDULING_INFO: 127 | connectionCache.put(details, 128 | mantisClient.getSchedulingChanges(details.target) 129 | .subscribeOn(scheduler) 130 | .map(changes -> Try.of(() -> objectMapper.writeValueAsString(changes)).getOrElse("Error")) 131 | .doOnCompleted(() -> { 132 | log.info("Purging {} from cache.", details); 133 | connectionCache.remove(details); 134 | }) 135 | .doOnUnsubscribe(() -> { 136 | log.info("Purging {} from cache.", details); 137 | connectionCache.remove(details); 138 | }) 139 | .replay(1) 140 | .autoConnect()); 141 | break; 142 | 143 | case JOB_CLUSTER_DISCOVERY: 144 | connectionCache.put(details, 145 | jobDiscoveryService.jobDiscoveryInfoStream(jobDiscoveryService.key(JobDiscoveryService.LookupType.JOB_CLUSTER, details.target)) 146 | .subscribeOn(scheduler) 147 | .map(jdi ->Try.of(() -> objectMapper.writeValueAsString(jdi)).getOrElse("Error")) 148 | .doOnCompleted(() -> { 149 | log.info("Purging {} from cache.", details); 150 | connectionCache.remove(details); 151 | }) 152 | .doOnUnsubscribe(() -> { 153 | log.info("Purging {} from cache.", details); 154 | connectionCache.remove(details); 155 | }) 156 | .replay(1) 157 | .autoConnect()); 158 | break; 159 | } 160 | log.info("Caching connection for: {}", details); 161 | } 162 | return connectionCache.get(details); 163 | } 164 | 165 | // 166 | // Helpers 167 | // 168 | 169 | private Observable getConnectByNameFor(PushConnectionDetails details) { 170 | return details.regions.isEmpty() 171 | ? getResults(false, this.mantisClient, details.target, details.getSinkparameters()) 172 | .flatMap(m -> m) 173 | .map(MantisServerSentEvent::getEventAsString) 174 | : getRemoteDataObservable(details.getUri(), details.target, details.getRegions().asJava()); 175 | } 176 | 177 | private Observable getConnectByIdFor(PushConnectionDetails details) { 178 | return details.getRegions().isEmpty() 179 | ? getResults(true, this.mantisClient, details.target, details.getSinkparameters()) 180 | .flatMap(m -> m) 181 | .map(MantisServerSentEvent::getEventAsString) 182 | : getRemoteDataObservable(details.getUri(), details.target, details.getRegions().asJava()); 183 | } 184 | 185 | 186 | private static SinkConnectionFunc getSseConnFunc(final String target, SinkParameters sinkParameters) { 187 | return new SseSinkConnectionFunction(true, 188 | t -> log.warn("Reconnecting to sink of job " + target + " after error: " + t.getMessage()), 189 | sinkParameters); 190 | } 191 | 192 | private static Observable> getResults(boolean isJobId, MantisClient mantisClient, 193 | final String target, SinkParameters sinkParameters) { 194 | final AtomicBoolean hasError = new AtomicBoolean(); 195 | return isJobId ? 196 | mantisClient.getSinkClientByJobId(target, getSseConnFunc(target, sinkParameters), null).getResults() : 197 | mantisClient.getSinkClientByJobName(target, getSseConnFunc(target, sinkParameters), null) 198 | .switchMap(serverSentEventSinkClient -> { 199 | if (serverSentEventSinkClient.hasError()) { 200 | hasError.set(true); 201 | return Observable.error(new Exception(serverSentEventSinkClient.getError())); 202 | } 203 | return serverSentEventSinkClient.getResults(); 204 | }) 205 | .takeWhile(o -> !hasError.get()); 206 | } 207 | 208 | // 209 | // Tunnel 210 | // 211 | 212 | private Observable getRemoteDataObservable(String uri, String target, List regions) { 213 | return Observable.from(regions) 214 | .flatMap(region -> { 215 | final String originReplacement = "\\{\"" + Constants.metaOriginName + "\": \"" + region + "\", "; 216 | if (region.equalsIgnoreCase(getLocalRegion())) { 217 | return this.connect(PushConnectionDetails.from(uri)) 218 | .map(datum -> datum.replaceFirst("^\\{", originReplacement)); 219 | } else { 220 | log.info("Connecting to remote region {} at {}.", region, uri); 221 | return mantisCrossRegionalClient.getSecureSseClient(region) 222 | .submit(HttpClientRequest.createGet(uri)) 223 | .retryWhen(Util.getRetryFunc(log, uri + " in " + region)) 224 | .doOnError(throwable -> log.warn( 225 | "Error getting response from remote SSE server for uri {} in region {}: {}", 226 | uri, region, throwable.getMessage(), throwable) 227 | ).flatMap(remoteResponse -> { 228 | if (!remoteResponse.getStatus().reasonPhrase().equals("OK")) { 229 | log.warn("Unexpected response from remote sink for uri {} region {}: {}", uri, region, remoteResponse.getStatus().reasonPhrase()); 230 | String err = remoteResponse.getHeaders().get(Constants.metaErrorMsgHeader); 231 | if (err == null || err.isEmpty()) 232 | err = remoteResponse.getStatus().reasonPhrase(); 233 | return Observable.error(new Exception(err)) 234 | .map(datum -> datum.getEventAsString()); 235 | } 236 | return clientResponseToObservable(remoteResponse, target, region, uri) 237 | .map(datum -> datum.replaceFirst("^\\{", originReplacement)) 238 | .doOnError(t -> log.error(t.getMessage())); 239 | }) 240 | .subscribeOn(scheduler) 241 | .observeOn(scheduler) 242 | .doOnError(t -> log.warn("Error streaming in remote data ({}). Will retry: {}", region, t.getMessage(), t)) 243 | .doOnCompleted(() -> log.info(String.format("remote sink connection complete for uri %s, region=%s", uri, region))); 244 | } 245 | }) 246 | .observeOn(scheduler) 247 | .subscribeOn(scheduler) 248 | .doOnError(t -> log.error("Error in flatMapped cross-regional observable for {}", uri, t)); 249 | } 250 | 251 | private Observable clientResponseToObservable(HttpClientResponse response, String target, String 252 | region, String uri) { 253 | 254 | Counter numRemoteBytes = SpectatorUtils.newCounter(Constants.numRemoteBytesCounterName, target, "region", region); 255 | Counter numRemoteMessages = SpectatorUtils.newCounter(Constants.numRemoteMessagesCounterName, target, "region", region); 256 | Counter numSseErrors = SpectatorUtils.newCounter(Constants.numSseErrorsCounterName, target, "region", region); 257 | 258 | return response.getContent() 259 | .doOnError(t -> log.warn(t.getMessage())) 260 | .timeout(3 * Constants.TunnelPingIntervalSecs, TimeUnit.SECONDS) 261 | .doOnError(t -> log.warn("Timeout getting data from remote {} connection for {}", region, uri)) 262 | .filter(sse -> !(!sse.hasEventType() || !sse.getEventTypeAsString().startsWith("error:")) || 263 | !TunnelPingMessage.equals(sse.contentAsString())) 264 | .map(t1 -> { 265 | String data = ""; 266 | if (t1.hasEventType() && t1.getEventTypeAsString().startsWith("error:")) { 267 | log.error("SSE has error, type=" + t1.getEventTypeAsString() + ", content=" + t1.contentAsString()); 268 | numSseErrors.increment(); 269 | throw new RuntimeException("Got error SSE event: " + t1.contentAsString()); 270 | } 271 | try { 272 | data = t1.contentAsString(); 273 | if (data != null) { 274 | numRemoteBytes.increment(data.length()); 275 | numRemoteMessages.increment(); 276 | } 277 | } catch (Exception e) { 278 | log.error("Could not extract data from SSE " + e.getMessage(), e); 279 | } 280 | return data; 281 | }); 282 | } 283 | 284 | private Observable getWorkerMetrics(PushConnectionDetails details) { 285 | 286 | final String jobId = details.target; 287 | 288 | SinkParameters metricNamesFilter = details.getSinkparameters(); 289 | 290 | final MetricsClient metricsClient = workerMetricsClient.getMetricsClientByJobId(jobId, 291 | new SseWorkerConnectionFunction(true, new Action1() { 292 | @Override 293 | public void call(Throwable throwable) { 294 | log.error("Metric connection error: " + throwable.getMessage()); 295 | try { 296 | Thread.sleep(500); 297 | } catch (InterruptedException ie) { 298 | log.error("Interrupted waiting for retrying connection"); 299 | } 300 | } 301 | }, metricNamesFilter), 302 | new Observer() { 303 | @Override 304 | public void onCompleted() { 305 | log.info("got onCompleted in WorkerConnStatus obs"); 306 | } 307 | 308 | @Override 309 | public void onError(Throwable e) { 310 | log.info("got onError in WorkerConnStatus obs"); 311 | } 312 | 313 | @Override 314 | public void onNext(WorkerConnectionsStatus workerConnectionsStatus) { 315 | log.info("got WorkerConnStatus {}", workerConnectionsStatus); 316 | } 317 | }); 318 | 319 | return metricsClient 320 | .getResults() 321 | .flatMap(metrics -> metrics 322 | .map(MantisServerSentEvent::getEventAsString)); 323 | 324 | } 325 | } 326 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/push/MantisSSEHandler.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.push; 2 | 3 | import com.netflix.config.DynamicIntProperty; 4 | import com.netflix.spectator.api.Counter; 5 | import com.netflix.zuul.netty.SpectatorUtils; 6 | import io.mantisrx.api.Constants; 7 | import io.mantisrx.api.Util; 8 | import io.mantisrx.server.core.master.MasterDescription; 9 | import io.mantisrx.server.master.client.HighAvailabilityServices; 10 | import io.mantisrx.shaded.com.google.common.util.concurrent.ThreadFactoryBuilder; 11 | import io.netty.buffer.ByteBuf; 12 | import io.netty.buffer.Unpooled; 13 | import io.netty.channel.ChannelHandlerContext; 14 | import io.netty.channel.SimpleChannelInboundHandler; 15 | import io.netty.handler.codec.http.*; 16 | import lombok.extern.slf4j.Slf4j; 17 | import mantis.io.reactivex.netty.RxNetty; 18 | import mantis.io.reactivex.netty.channel.StringTransformer; 19 | import mantis.io.reactivex.netty.pipeline.PipelineConfigurator; 20 | import mantis.io.reactivex.netty.pipeline.PipelineConfigurators; 21 | import mantis.io.reactivex.netty.protocol.http.client.HttpClient; 22 | import mantis.io.reactivex.netty.protocol.http.client.HttpClientRequest; 23 | import mantis.io.reactivex.netty.protocol.http.client.HttpClientResponse; 24 | import mantis.io.reactivex.netty.protocol.http.client.HttpResponseHeaders; 25 | import rx.Observable; 26 | import rx.Subscription; 27 | 28 | import java.nio.charset.StandardCharsets; 29 | import java.util.*; 30 | import java.util.concurrent.BlockingQueue; 31 | import java.util.concurrent.LinkedBlockingQueue; 32 | import java.util.concurrent.ScheduledExecutorService; 33 | import java.util.concurrent.ScheduledFuture; 34 | import java.util.concurrent.ScheduledThreadPoolExecutor; 35 | import java.util.concurrent.TimeUnit; 36 | 37 | /** 38 | * Http handler for the WebSocket/SSE paths. 39 | */ 40 | @Slf4j 41 | public class MantisSSEHandler extends SimpleChannelInboundHandler { 42 | private final DynamicIntProperty queueCapacity = new DynamicIntProperty("io.mantisrx.api.push.queueCapacity", 1000); 43 | private final DynamicIntProperty writeIntervalMillis = new DynamicIntProperty("io.mantisrx.api.push.writeIntervalMillis", 50); 44 | 45 | private final ConnectionBroker connectionBroker; 46 | private final HighAvailabilityServices highAvailabilityServices; 47 | private final List pushPrefixes; 48 | 49 | private Subscription subscription; 50 | private ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, 51 | new ThreadFactoryBuilder().setNameFormat("sse-handler-drainer-%d").build()); 52 | private ScheduledFuture drainFuture; 53 | private String uri; 54 | 55 | public MantisSSEHandler(ConnectionBroker connectionBroker, HighAvailabilityServices highAvailabilityServices, 56 | List pushPrefixes) { 57 | super(true); 58 | this.connectionBroker = connectionBroker; 59 | this.highAvailabilityServices = highAvailabilityServices; 60 | this.pushPrefixes = pushPrefixes; 61 | } 62 | 63 | @Override 64 | protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) throws Exception { 65 | if (Util.startsWithAnyOf(request.uri(), pushPrefixes) 66 | && !isWebsocketUpgrade(request)) { 67 | 68 | if (HttpUtil.is100ContinueExpected(request)) { 69 | send100Contine(ctx); 70 | } 71 | 72 | HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, 73 | HttpResponseStatus.OK); 74 | HttpHeaders headers = response.headers(); 75 | headers.add(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN, "*"); 76 | headers.add(HttpHeaderNames.ACCESS_CONTROL_ALLOW_HEADERS, "Origin, X-Requested-With, Accept, Content-Type, Cache-Control"); 77 | headers.set(HttpHeaderNames.CONTENT_TYPE, "text/event-stream"); 78 | headers.set(HttpHeaderNames.CACHE_CONTROL, "no-cache, no-store, max-age=0, must-revalidate"); 79 | headers.set(HttpHeaderNames.PRAGMA, HttpHeaderValues.NO_CACHE); 80 | headers.set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED); 81 | response.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE); 82 | ctx.writeAndFlush(response); 83 | 84 | uri = request.uri(); 85 | final PushConnectionDetails pcd = 86 | isSubmitAndConnect(request) 87 | ? new PushConnectionDetails(uri, jobSubmit(request), PushConnectionDetails.TARGET_TYPE.CONNECT_BY_ID, io.vavr.collection.List.empty()) 88 | : PushConnectionDetails.from(uri); 89 | log.info("SSE Connecting for: {}", pcd); 90 | 91 | boolean tunnelPingsEnabled = isTunnelPingsEnabled(uri); 92 | 93 | final String[] tags = Util.getTaglist(uri, pcd.target); 94 | Counter numDroppedBytesCounter = SpectatorUtils.newCounter(Constants.numDroppedBytesCounterName, pcd.target, tags); 95 | Counter numDroppedMessagesCounter = SpectatorUtils.newCounter(Constants.numDroppedMessagesCounterName, pcd.target, tags); 96 | Counter numMessagesCounter = SpectatorUtils.newCounter(Constants.numMessagesCounterName, pcd.target, tags); 97 | Counter numBytesCounter = SpectatorUtils.newCounter(Constants.numBytesCounterName, pcd.target, tags); 98 | Counter drainTriggeredCounter = SpectatorUtils.newCounter(Constants.drainTriggeredCounterName, pcd.target, tags); 99 | Counter numIncomingMessagesCounter = SpectatorUtils.newCounter(Constants.numIncomingMessagesCounterName, pcd.target, tags); 100 | 101 | BlockingQueue queue = new LinkedBlockingQueue<>(queueCapacity.get()); 102 | 103 | drainFuture = scheduledExecutorService.scheduleAtFixedRate(() -> { 104 | try { 105 | if (queue.size() > 0 && ctx.channel().isWritable()) { 106 | drainTriggeredCounter.increment(); 107 | final List items = new ArrayList<>(queue.size()); 108 | synchronized (queue) { 109 | queue.drainTo(items); 110 | } 111 | for (String data : items) { 112 | ctx.write(Unpooled.copiedBuffer(data, StandardCharsets.UTF_8)); 113 | numMessagesCounter.increment(); 114 | numBytesCounter.increment(data.length()); 115 | } 116 | ctx.flush(); 117 | } 118 | } catch (Exception ex) { 119 | log.error("Error writing to channel", ex); 120 | } 121 | }, writeIntervalMillis.get(), writeIntervalMillis.get(), TimeUnit.MILLISECONDS); 122 | 123 | this.subscription = this.connectionBroker.connect(pcd) 124 | .doOnNext(event -> numIncomingMessagesCounter.increment()) 125 | .mergeWith(tunnelPingsEnabled 126 | ? Observable.interval(Constants.TunnelPingIntervalSecs, Constants.TunnelPingIntervalSecs, 127 | TimeUnit.SECONDS) 128 | .map(l -> Constants.TunnelPingMessage) 129 | : Observable.empty()) 130 | .doOnNext(event -> { 131 | if (!Constants.DUMMY_TIMER_DATA.equals(event)) { 132 | String data = Constants.SSE_DATA_PREFIX + event + Constants.SSE_DATA_SUFFIX; 133 | boolean offer = false; 134 | synchronized (queue) { 135 | offer = queue.offer(data); 136 | } 137 | if (!offer) { 138 | numDroppedBytesCounter.increment(data.length()); 139 | numDroppedMessagesCounter.increment(); 140 | } 141 | } 142 | }) 143 | .subscribe(); 144 | } else { 145 | ctx.fireChannelRead(request.retain()); 146 | } 147 | } 148 | 149 | private static void send100Contine(ChannelHandlerContext ctx) { 150 | FullHttpResponse response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, 151 | HttpResponseStatus.CONTINUE); 152 | ctx.writeAndFlush(response); 153 | } 154 | 155 | private boolean isTunnelPingsEnabled(String uri) { 156 | QueryStringDecoder queryStringDecoder = new QueryStringDecoder(uri); 157 | return queryStringDecoder.parameters() 158 | .getOrDefault(Constants.TunnelPingParamName, Arrays.asList("false")) 159 | .get(0) 160 | .equalsIgnoreCase("true"); 161 | } 162 | 163 | private boolean isWebsocketUpgrade(HttpRequest request) { 164 | HttpHeaders headers = request.headers(); 165 | // Header "Connection" contains "upgrade" (case insensitive) and 166 | // Header "Upgrade" equals "websocket" (case insensitive) 167 | String connection = headers.get(HttpHeaderNames.CONNECTION); 168 | String upgrade = headers.get(HttpHeaderNames.UPGRADE); 169 | return connection != null && connection.toLowerCase().contains("upgrade") && 170 | upgrade != null && upgrade.toLowerCase().equals("websocket"); 171 | } 172 | 173 | 174 | private boolean isSubmitAndConnect(HttpRequest request) { 175 | return request.method().equals(HttpMethod.POST) && request.uri().contains("jobsubmitandconnect"); 176 | } 177 | 178 | @Override 179 | public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { 180 | log.info("Channel {} is unregistered. URI: {}", ctx.channel(), uri); 181 | unsubscribeIfSubscribed(); 182 | super.channelUnregistered(ctx); 183 | } 184 | 185 | @Override 186 | public void channelInactive(ChannelHandlerContext ctx) throws Exception { 187 | log.info("Channel {} is inactive. URI: {}", ctx.channel(), uri); 188 | unsubscribeIfSubscribed(); 189 | super.channelInactive(ctx); 190 | } 191 | 192 | @Override 193 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { 194 | log.warn("Exception caught by channel {}. URI: {}", ctx.channel(), uri, cause); 195 | unsubscribeIfSubscribed(); 196 | ctx.close(); 197 | } 198 | 199 | /** Unsubscribe if it's subscribed. */ 200 | private void unsubscribeIfSubscribed() { 201 | if (subscription != null && !subscription.isUnsubscribed()) { 202 | log.info("SSE unsubscribing subscription with URI: {}", uri); 203 | subscription.unsubscribe(); 204 | } 205 | if (drainFuture != null) { 206 | drainFuture.cancel(false); 207 | } 208 | if (scheduledExecutorService != null) { 209 | scheduledExecutorService.shutdown(); 210 | } 211 | } 212 | 213 | public String jobSubmit(FullHttpRequest request) { 214 | final String API_JOB_SUBMIT_PATH = "/api/submit"; 215 | 216 | String content = request.content().toString(StandardCharsets.UTF_8); 217 | return callPostOnMaster(highAvailabilityServices.getMasterMonitor().getMasterObservable(), API_JOB_SUBMIT_PATH, content) 218 | .retryWhen(Util.getRetryFunc(log, API_JOB_SUBMIT_PATH)) 219 | .flatMap(masterResponse -> masterResponse.getByteBuf() 220 | .take(1) 221 | .map(byteBuf -> { 222 | final String s = byteBuf.toString(StandardCharsets.UTF_8); 223 | log.info("response: " + s); 224 | return s; 225 | })) 226 | .take(1) 227 | .toBlocking() 228 | .first(); 229 | } 230 | 231 | public static class MasterResponse { 232 | 233 | private final HttpResponseStatus status; 234 | private final Observable byteBuf; 235 | private final HttpResponseHeaders responseHeaders; 236 | 237 | public MasterResponse(HttpResponseStatus status, Observable byteBuf, HttpResponseHeaders responseHeaders) { 238 | this.status = status; 239 | this.byteBuf = byteBuf; 240 | this.responseHeaders = responseHeaders; 241 | } 242 | 243 | public HttpResponseStatus getStatus() { 244 | return status; 245 | } 246 | 247 | public Observable getByteBuf() { 248 | return byteBuf; 249 | } 250 | 251 | public HttpResponseHeaders getResponseHeaders() { return responseHeaders; } 252 | } 253 | 254 | public static Observable callPostOnMaster(Observable masterObservable, String uri, String content) { 255 | PipelineConfigurator, HttpClientRequest> pipelineConfigurator 256 | = PipelineConfigurators.httpClientConfigurator(); 257 | 258 | return masterObservable 259 | .filter(Objects::nonNull) 260 | .flatMap(masterDesc -> { 261 | HttpClient client = 262 | RxNetty.newHttpClientBuilder(masterDesc.getHostname(), masterDesc.getApiPort()) 263 | .pipelineConfigurator(pipelineConfigurator) 264 | .build(); 265 | HttpClientRequest request = HttpClientRequest.create(HttpMethod.POST, uri); 266 | request = request.withHeader(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_JSON.toString()); 267 | request.withRawContent(content, StringTransformer.DEFAULT_INSTANCE); 268 | return client.submit(request) 269 | .map(response -> new MasterResponse(response.getStatus(), response.getContent(), response.getHeaders())); 270 | }) 271 | .take(1); 272 | } 273 | } 274 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/push/MantisWebSocketFrameHandler.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.push; 2 | 3 | import java.util.concurrent.BlockingQueue; 4 | import java.util.concurrent.LinkedBlockingQueue; 5 | import java.util.concurrent.ScheduledExecutorService; 6 | import java.util.concurrent.ScheduledFuture; 7 | import java.util.concurrent.ScheduledThreadPoolExecutor; 8 | import java.util.concurrent.TimeUnit; 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | 12 | import com.netflix.config.DynamicIntProperty; 13 | import com.netflix.spectator.api.Counter; 14 | import com.netflix.zuul.netty.SpectatorUtils; 15 | import io.mantisrx.api.Constants; 16 | import io.mantisrx.api.Util; 17 | import io.mantisrx.shaded.com.google.common.util.concurrent.ThreadFactoryBuilder; 18 | import io.netty.channel.ChannelHandlerContext; 19 | import io.netty.channel.SimpleChannelInboundHandler; 20 | import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; 21 | import io.netty.handler.codec.http.websocketx.WebSocketServerProtocolHandler; 22 | import io.netty.util.ReferenceCountUtil; 23 | import lombok.extern.slf4j.Slf4j; 24 | import rx.Subscription; 25 | 26 | @Slf4j 27 | public class MantisWebSocketFrameHandler extends SimpleChannelInboundHandler { 28 | private final ConnectionBroker connectionBroker; 29 | private final DynamicIntProperty queueCapacity = new DynamicIntProperty("io.mantisrx.api.push.queueCapacity", 1000); 30 | private final DynamicIntProperty writeIntervalMillis = new DynamicIntProperty("io.mantisrx.api.push.writeIntervalMillis", 50); 31 | 32 | private Subscription subscription; 33 | private String uri; 34 | private ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, 35 | new ThreadFactoryBuilder().setNameFormat("websocket-handler-drainer-%d").build()); 36 | private ScheduledFuture drainFuture; 37 | 38 | public MantisWebSocketFrameHandler(ConnectionBroker broker) { 39 | super(true); 40 | this.connectionBroker = broker; 41 | } 42 | 43 | @Override 44 | public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { 45 | if (evt.getClass() == WebSocketServerProtocolHandler.HandshakeComplete.class) { 46 | WebSocketServerProtocolHandler.HandshakeComplete complete = (WebSocketServerProtocolHandler.HandshakeComplete) evt; 47 | 48 | uri = complete.requestUri(); 49 | final PushConnectionDetails pcd = PushConnectionDetails.from(uri); 50 | 51 | log.info("Request to URI '{}' is a WebSSocket upgrade, removing the SSE handler", uri); 52 | if (ctx.pipeline().get(MantisSSEHandler.class) != null) { 53 | ctx.pipeline().remove(MantisSSEHandler.class); 54 | } 55 | 56 | final String[] tags = Util.getTaglist(uri, pcd.target); 57 | Counter numDroppedBytesCounter = SpectatorUtils.newCounter(Constants.numDroppedBytesCounterName, pcd.target, tags); 58 | Counter numDroppedMessagesCounter = SpectatorUtils.newCounter(Constants.numDroppedMessagesCounterName, pcd.target, tags); 59 | Counter numMessagesCounter = SpectatorUtils.newCounter(Constants.numMessagesCounterName, pcd.target, tags); 60 | Counter numBytesCounter = SpectatorUtils.newCounter(Constants.numBytesCounterName, pcd.target, tags); 61 | Counter drainTriggeredCounter = SpectatorUtils.newCounter(Constants.drainTriggeredCounterName, pcd.target, tags); 62 | Counter numIncomingMessagesCounter = SpectatorUtils.newCounter(Constants.numIncomingMessagesCounterName, pcd.target, tags); 63 | 64 | BlockingQueue queue = new LinkedBlockingQueue<>(queueCapacity.get()); 65 | 66 | drainFuture = scheduledExecutorService.scheduleAtFixedRate(() -> { 67 | try { 68 | if (queue.size() > 0 && ctx.channel().isWritable()) { 69 | drainTriggeredCounter.increment(); 70 | final List items = new ArrayList<>(queue.size()); 71 | synchronized (queue) { 72 | queue.drainTo(items); 73 | } 74 | for (String data : items) { 75 | ctx.write(new TextWebSocketFrame(data)); 76 | numMessagesCounter.increment(); 77 | numBytesCounter.increment(data.length()); 78 | } 79 | ctx.flush(); 80 | } 81 | } catch (Exception ex) { 82 | log.error("Error writing to channel", ex); 83 | } 84 | }, writeIntervalMillis.get(), writeIntervalMillis.get(), TimeUnit.MILLISECONDS); 85 | 86 | this.subscription = this.connectionBroker.connect(pcd) 87 | .doOnNext(event -> { 88 | numIncomingMessagesCounter.increment(); 89 | if (!Constants.DUMMY_TIMER_DATA.equals(event)) { 90 | boolean offer = false; 91 | synchronized (queue) { 92 | offer = queue.offer(event); 93 | } 94 | if (!offer) { 95 | numDroppedBytesCounter.increment(event.length()); 96 | numDroppedMessagesCounter.increment(); 97 | } 98 | } 99 | }) 100 | .subscribe(); 101 | } else { 102 | ReferenceCountUtil.retain(evt); 103 | super.userEventTriggered(ctx, evt); 104 | } 105 | } 106 | 107 | @Override 108 | public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { 109 | log.info("Channel {} is unregistered. URI: {}", ctx.channel(), uri); 110 | unsubscribeIfSubscribed(); 111 | super.channelUnregistered(ctx); 112 | } 113 | 114 | @Override 115 | public void channelInactive(ChannelHandlerContext ctx) throws Exception { 116 | log.info("Channel {} is inactive. URI: {}", ctx.channel(), uri); 117 | unsubscribeIfSubscribed(); 118 | super.channelInactive(ctx); 119 | } 120 | 121 | @Override 122 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { 123 | log.warn("Exception caught by channel {}. URI: {}", ctx.channel(), uri, cause); 124 | unsubscribeIfSubscribed(); 125 | // This is the tail of handlers. We should close the channel between the server and the client, 126 | // essentially causing the client to disconnect and terminate. 127 | ctx.close(); 128 | } 129 | 130 | @Override 131 | protected void channelRead0(ChannelHandlerContext ctx, TextWebSocketFrame msg) { 132 | // No op. 133 | } 134 | 135 | /** Unsubscribe if it's subscribed. */ 136 | private void unsubscribeIfSubscribed() { 137 | if (subscription != null && !subscription.isUnsubscribed()) { 138 | log.info("WebSocket unsubscribing subscription with URI: {}", uri); 139 | subscription.unsubscribe(); 140 | } 141 | if (drainFuture != null) { 142 | drainFuture.cancel(false); 143 | } 144 | if (scheduledExecutorService != null) { 145 | scheduledExecutorService.shutdown(); 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/push/PushConnectionDetails.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.push; 2 | 3 | import io.mantisrx.runtime.parameter.SinkParameter; 4 | import io.mantisrx.runtime.parameter.SinkParameters; 5 | import io.netty.handler.codec.http.QueryStringDecoder; 6 | import io.vavr.collection.List; 7 | import io.vavr.control.Try; 8 | import lombok.Value; 9 | 10 | import java.util.stream.Collectors; 11 | 12 | public @Value class PushConnectionDetails { 13 | 14 | public enum TARGET_TYPE { 15 | CONNECT_BY_NAME, 16 | CONNECT_BY_ID, 17 | JOB_STATUS, 18 | JOB_SCHEDULING_INFO, 19 | JOB_CLUSTER_DISCOVERY, 20 | METRICS 21 | } 22 | 23 | private final String uri; 24 | public final String target; 25 | public final TARGET_TYPE type; 26 | public final List regions; 27 | 28 | 29 | /** 30 | * Determines the connection type for a given push connection. 31 | * 32 | * @param uri Request URI as returned by Netty's requestUri() methods. Expects leading slash. 33 | * @return The CONNECTION_TYPE requested by the URI. 34 | */ 35 | public static TARGET_TYPE determineTargetType(final String uri) { 36 | if (uri.startsWith("/jobconnectbyname") || uri.startsWith("/api/v1/jobconnectbyname")) { 37 | return TARGET_TYPE.CONNECT_BY_NAME; 38 | } else if (uri.startsWith("/jobconnectbyid") || uri.startsWith("/api/v1/jobconnectbyid")) { 39 | return TARGET_TYPE.CONNECT_BY_ID; 40 | } else if (uri.startsWith("/jobstatus/") || uri.startsWith("/api/v1/jobstatus/")) { 41 | return TARGET_TYPE.JOB_STATUS; 42 | } else if (uri.startsWith("/api/v1/jobs/schedulingInfo/")) { 43 | return TARGET_TYPE.JOB_SCHEDULING_INFO; 44 | } else if (uri.startsWith("/jobClusters/discoveryInfoStream/")) { 45 | return TARGET_TYPE.JOB_CLUSTER_DISCOVERY; 46 | } else if (uri.startsWith("/api/v1/metrics/")) { 47 | return TARGET_TYPE.METRICS; 48 | } else { 49 | throw new IllegalArgumentException("Unable to determine push connection type from URI: " + uri); 50 | } 51 | } 52 | 53 | /** 54 | * Determines the target for a push connection request. Typically a job name or id. 55 | * 56 | * @param uri Request URI as returned by Netty's requestUri() methods. Expects leading slash. 57 | * @return The target requested by the URI. 58 | */ 59 | public static String determineTarget(final String uri) { 60 | String sanitized = uri.replaceFirst("^/(api/v1/)?(jobconnectbyid|jobconnectbyname|jobstatus|jobs/schedulingInfo|jobClusters/discoveryInfoStream|metrics)/", ""); 61 | QueryStringDecoder queryStringDecoder = new QueryStringDecoder(sanitized); 62 | return queryStringDecoder.path(); 63 | } 64 | 65 | // 66 | // Computed Properties 67 | // 68 | 69 | public SinkParameters getSinkparameters() { 70 | SinkParameters.Builder builder = new SinkParameters.Builder(); 71 | QueryStringDecoder queryStringDecoder = new QueryStringDecoder(uri); 72 | 73 | builder.parameters(queryStringDecoder 74 | .parameters() 75 | .entrySet() 76 | .stream() 77 | .flatMap(entry -> entry.getValue() 78 | .stream() 79 | .map(val -> Try.of(() -> new SinkParameter(entry.getKey(), val))) 80 | .filter(Try::isSuccess) 81 | .map(Try::get)) 82 | .collect(Collectors.toList()) 83 | .toArray(new SinkParameter[]{})); 84 | 85 | return builder.build(); 86 | } 87 | 88 | // 89 | // Static Factories 90 | // 91 | 92 | public static PushConnectionDetails from(String uri) { 93 | return from(uri, List.empty()); 94 | } 95 | 96 | public static PushConnectionDetails from(String uri, List regions) { 97 | return new PushConnectionDetails(uri, determineTarget(uri), determineTargetType(uri), regions); 98 | } 99 | 100 | public static PushConnectionDetails from(String uri, java.util.List regions) { 101 | return new PushConnectionDetails(uri, determineTarget(uri), determineTargetType(uri), List.ofAll(regions)); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/services/AppStreamDiscoveryService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.mantisrx.api.services; 17 | 18 | import com.google.common.base.Preconditions; 19 | import com.netflix.spectator.api.Counter; 20 | import com.netflix.zuul.netty.SpectatorUtils; 21 | import io.mantisrx.api.proto.AppDiscoveryMap; 22 | import io.mantisrx.client.MantisClient; 23 | import io.mantisrx.discovery.proto.AppJobClustersMap; 24 | import io.mantisrx.server.core.JobSchedulingInfo; 25 | import io.vavr.control.Either; 26 | import io.vavr.control.Option; 27 | import java.io.IOException; 28 | import java.util.List; 29 | import java.util.concurrent.TimeUnit; 30 | import lombok.extern.slf4j.Slf4j; 31 | import rx.Observable; 32 | import rx.Scheduler; 33 | 34 | @Slf4j 35 | public class AppStreamDiscoveryService { 36 | 37 | private final MantisClient mantisClient; 38 | private final Scheduler scheduler; 39 | 40 | private final AppStreamStore appStreamStore; 41 | 42 | public AppStreamDiscoveryService( 43 | MantisClient mantisClient, 44 | Scheduler scheduler, 45 | AppStreamStore appStreamStore) { 46 | Preconditions.checkArgument(mantisClient != null); 47 | Preconditions.checkArgument(appStreamStore != null); 48 | Preconditions.checkArgument(scheduler != null); 49 | this.mantisClient = mantisClient; 50 | this.scheduler = scheduler; 51 | this.appStreamStore = appStreamStore; 52 | 53 | Counter appJobClusterMappingNullCount = SpectatorUtils.newCounter( 54 | "appJobClusterMappingNull", "mantisapi"); 55 | Counter appJobClusterMappingRequestCount = SpectatorUtils.newCounter( 56 | "appJobClusterMappingRequest", "mantisapi", "app", "unknown"); 57 | Counter appJobClusterMappingFailCount = SpectatorUtils.newCounter( 58 | "appJobClusterMappingFail", "mantisapi"); 59 | } 60 | 61 | 62 | public Either getAppDiscoveryMap(List appNames) { 63 | try { 64 | 65 | AppJobClustersMap appJobClusters = getAppJobClustersMap(appNames); 66 | 67 | // 68 | // Lookup discovery info per stream and build mapping 69 | // 70 | 71 | AppDiscoveryMap adm = new AppDiscoveryMap(appJobClusters.getVersion(), appJobClusters.getTimestamp()); 72 | 73 | for (String app : appJobClusters.getMappings().keySet()) { 74 | for (String stream : appJobClusters.getMappings().get(app).keySet()) { 75 | String jobCluster = appJobClusters.getMappings().get(app).get(stream); 76 | Option jobSchedulingInfo = getJobDiscoveryInfo(jobCluster); 77 | jobSchedulingInfo.map(jsi -> { 78 | adm.addMapping(app, stream, jsi); 79 | return jsi; 80 | }); 81 | } 82 | } 83 | return Either.right(adm); 84 | } catch (Exception ex) { 85 | log.error(ex.getMessage()); 86 | return Either.left(ex.getMessage()); 87 | } 88 | } 89 | 90 | public AppJobClustersMap getAppJobClustersMap(List appNames) throws IOException { 91 | return appStreamStore.getJobClusterMappings(appNames); 92 | } 93 | 94 | private Option getJobDiscoveryInfo(String jobCluster) { 95 | JobDiscoveryService jdim = JobDiscoveryService.getInstance(mantisClient, scheduler); 96 | return jdim 97 | .jobDiscoveryInfoStream(jdim.key(JobDiscoveryService.LookupType.JOB_CLUSTER, jobCluster)) 98 | .map(Option::of) 99 | .take(1) 100 | .timeout(2, TimeUnit.SECONDS, Observable.just(Option.none())) 101 | .doOnError((t) -> { 102 | log.warn("Timed out looking up job discovery info for cluster: " + jobCluster + "."); 103 | }) 104 | .subscribeOn(scheduler) 105 | .observeOn(scheduler) 106 | .toSingle() 107 | .toBlocking() 108 | .value(); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/services/AppStreamStore.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.services; 2 | 3 | import com.google.common.collect.ImmutableList; 4 | import io.mantisrx.discovery.proto.AppJobClustersMap; 5 | import java.io.IOException; 6 | import java.util.Collection; 7 | 8 | /** 9 | * Interface to get streams associated with a given app or set of apps 10 | */ 11 | public interface AppStreamStore { 12 | default AppJobClustersMap getJobClusterMappings(String app) throws IOException { 13 | return getJobClusterMappings(ImmutableList.of(app)); 14 | } 15 | 16 | AppJobClustersMap getJobClusterMappings(Collection apps) throws IOException; 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/services/ConfigurationBasedAppStreamStore.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.services; 2 | 3 | import com.netflix.spectator.api.Counter; 4 | import com.netflix.zuul.netty.SpectatorUtils; 5 | import io.mantisrx.common.JsonSerializer; 6 | import io.mantisrx.discovery.proto.AppJobClustersMap; 7 | import io.mantisrx.shaded.org.apache.curator.framework.listen.Listenable; 8 | import java.io.IOException; 9 | import java.util.ArrayList; 10 | import java.util.Collection; 11 | import java.util.concurrent.atomic.AtomicReference; 12 | import java.util.function.Supplier; 13 | import javax.annotation.Nullable; 14 | import lombok.extern.slf4j.Slf4j; 15 | 16 | @SuppressWarnings("unused") 17 | @Slf4j 18 | public class ConfigurationBasedAppStreamStore implements AppStreamStore { 19 | 20 | private final JsonSerializer jsonSerializer; 21 | 22 | private final AtomicReference appJobClusterMappings = new AtomicReference<>(); 23 | 24 | private final Counter appJobClusterMappingNullCount; 25 | private final Counter appJobClusterMappingFailCount; 26 | private final Counter appJobClusterMappingRequestCount; 27 | 28 | public ConfigurationBasedAppStreamStore(ConfigSource configSource) { 29 | configSource.getListenable() 30 | .addListener((newConfig) -> updateAppJobClustersMapping(newConfig)); 31 | this.jsonSerializer = new JsonSerializer(); 32 | updateAppJobClustersMapping(configSource.get()); 33 | 34 | this.appJobClusterMappingNullCount = SpectatorUtils.newCounter( 35 | "appJobClusterMappingNull", "mantisapi"); 36 | this.appJobClusterMappingRequestCount = SpectatorUtils.newCounter( 37 | "appJobClusterMappingRequest", "mantisapi", "app", "unknown"); 38 | this.appJobClusterMappingFailCount = SpectatorUtils.newCounter( 39 | "appJobClusterMappingFail", "mantisapi"); 40 | } 41 | 42 | @Override 43 | public AppJobClustersMap getJobClusterMappings(Collection apps) throws IOException { 44 | return getAppJobClustersMap(apps, this.appJobClusterMappings.get()); 45 | } 46 | 47 | private AppJobClustersMap getAppJobClustersMap(Collection appNames, 48 | @Nullable AppJobClustersMap appJobClustersMap) throws IOException { 49 | 50 | if (appJobClustersMap != null) { 51 | final AppJobClustersMap appJobClusters; 52 | if (appNames.size() > 0) { 53 | appJobClusters = appJobClustersMap.getFilteredAppJobClustersMap(new ArrayList<>(appNames)); 54 | } else { 55 | appJobClusterMappingRequestCount.increment(); 56 | appJobClusters = appJobClustersMap; 57 | } 58 | return appJobClusters; 59 | } else { 60 | appJobClusterMappingNullCount.increment(); 61 | throw new IOException("AppJobClustersMap is null"); 62 | } 63 | } 64 | 65 | private void updateAppJobClustersMapping(String appJobClusterStr) { 66 | try { 67 | AppJobClustersMap appJobClustersMap = 68 | jsonSerializer.fromJSON(appJobClusterStr, AppJobClustersMap.class); 69 | log.info("appJobClustersMap updated to {}", appJobClustersMap); 70 | appJobClusterMappings.set(appJobClustersMap); 71 | } catch (Exception ioe) { 72 | log.error("failed to update appJobClustersMap on Property update {}", appJobClusterStr, ioe); 73 | appJobClusterMappingFailCount.increment(); 74 | } 75 | } 76 | 77 | public interface ConfigSource extends Supplier { 78 | 79 | Listenable getListenable(); 80 | } 81 | 82 | public interface ConfigurationChangeListener { 83 | 84 | void onConfigChange(String config); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/services/artifacts/ArtifactManager.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.services.artifacts; 2 | /* 3 | * Copyright 2019 Netflix, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import io.mantisrx.api.proto.Artifact; 19 | 20 | import java.util.List; 21 | import java.util.Optional; 22 | 23 | public interface ArtifactManager { 24 | 25 | List getArtifacts(); 26 | 27 | Optional getArtifact(String name); 28 | 29 | void deleteArtifact(String name); 30 | 31 | void putArtifact(Artifact artifact); 32 | } -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/services/artifacts/InMemoryArtifactManager.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.services.artifacts; 2 | import java.util.HashMap; 3 | import java.util.List; 4 | import java.util.Map; 5 | import java.util.Optional; 6 | import java.util.stream.Collectors; 7 | 8 | import io.mantisrx.api.proto.Artifact; 9 | 10 | 11 | public class InMemoryArtifactManager implements ArtifactManager { 12 | private Map artifacts = new HashMap<>(); 13 | 14 | @Override 15 | public List getArtifacts() { 16 | return artifacts 17 | .values() 18 | .stream() 19 | .map(Artifact::getFileName) 20 | .collect(Collectors.toList()); 21 | } 22 | 23 | @Override 24 | public Optional getArtifact(String name) { 25 | return artifacts 26 | .values() 27 | .stream() 28 | .filter(artifact -> artifact.getFileName().equals(name)) 29 | .findFirst(); 30 | } 31 | 32 | @Override 33 | public void deleteArtifact(String name) { 34 | this.artifacts.remove(name); 35 | } 36 | 37 | @Override 38 | public void putArtifact(Artifact artifact) { 39 | this.artifacts.put(artifact.getFileName(), artifact); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/tunnel/MantisCrossRegionalClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.api.tunnel; 18 | 19 | import io.netty.buffer.ByteBuf; 20 | import mantis.io.reactivex.netty.protocol.http.client.HttpClient; 21 | import mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent; 22 | 23 | public interface MantisCrossRegionalClient { 24 | HttpClient getSecureSseClient(String region); 25 | 26 | HttpClient getSecureRestClient(String region); 27 | } 28 | 29 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/tunnel/NoOpCrossRegionalClient.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.tunnel; 2 | 3 | import io.netty.buffer.ByteBuf; 4 | import mantis.io.reactivex.netty.protocol.http.client.HttpClient; 5 | import mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent; 6 | 7 | public class NoOpCrossRegionalClient implements MantisCrossRegionalClient { 8 | @Override 9 | public HttpClient getSecureSseClient(String region) { 10 | throw new UnsupportedOperationException(); 11 | } 12 | 13 | @Override 14 | public HttpClient getSecureRestClient(String region) { 15 | throw new UnsupportedOperationException(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/io/mantisrx/api/tunnel/RegionData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Netflix, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package io.mantisrx.api.tunnel; 18 | 19 | import lombok.Value; 20 | 21 | public @Value class RegionData { 22 | private final String region; 23 | private final boolean success; 24 | private final String data; 25 | private final int responseCode; 26 | } 27 | -------------------------------------------------------------------------------- /src/main/resources/api-docker.properties: -------------------------------------------------------------------------------- 1 | region=us-east-1 2 | mantis.localmode=false 3 | 4 | eureka.registration.enabled=false 5 | eureka.shouldFetchRegistry=false 6 | 7 | #Zookeeper is necessary for master discovery 8 | 9 | mantis.zookeeper.connectString=zookeeper:2181 10 | mantis.zookeeper.root=/mantis/master/nmahilani 11 | mantis.zookeeper.leader.announcement.path=/leader 12 | 13 | default.nfzookeeper.session-timeout-ms=120000 14 | 15 | api.ribbon.NIWSServerListClassName=io.mantisrx.api.MantisConfigurationBasedServerList 16 | zuul.filters.packages=io.mantisrx.api.filters 17 | 18 | # Controls how long inactive websocket sessions take to timeout 19 | mantisapi.connection.inactive.timeout.secs=300 20 | 21 | mantisapi.submit.instanceLimit=100 22 | 23 | mantis.sse.disablePingFiltering=true 24 | 25 | mantisapi.artifact.disk.cache.location=/apps/nfmantisapi/mantisArtifacts/ 26 | mantisapi.artifact.disk.cache.enabled=true 27 | 28 | mreAppJobClusterMap={"version": "1", "timestamp": 12345, "mappings": {"__default__": {"requestEventStream": "SharedPushRequestEventSource","sentryEventStream": "SentryLogEventSource","__default__": "SharedMrePublishEventSource"},"customApp": {"logEventStream": "CustomAppEventSource","sentryEventStream": "CustomAppSentryLogSource"}}} 29 | -------------------------------------------------------------------------------- /src/main/resources/api-local.properties: -------------------------------------------------------------------------------- 1 | region=us-east-1 2 | mantis.localmode=true 3 | 4 | eureka.registration.enabled=false 5 | eureka.shouldFetchRegistry=false 6 | 7 | #Zookeeper is necessary for master discovery 8 | 9 | mantis.zookeeper.connectString=localhost:8100 10 | mantis.zookeeper.root=/mantis/master/nmahilani 11 | mantis.zookeeper.leader.announcement.path=/leader 12 | mesos.master.location=zk://zookeeper:2181/mantis/mesos/nmahilani 13 | 14 | default.nfzookeeper.session-timeout-ms=120000 15 | 16 | api.ribbon.NIWSServerListClassName=io.mantisrx.api.MantisConfigurationBasedServerList 17 | zuul.filters.packages=io.mantisrx.api.filters 18 | 19 | # Controls how long inactive websocket sessions take to timeout 20 | mantisapi.connection.inactive.timeout.secs=300 21 | 22 | mantisapi.submit.instanceLimit=100 23 | 24 | mantis.sse.disablePingFiltering=true 25 | 26 | mantisapi.artifact.disk.cache.location=/apps/nfmantisapi/mantisArtifacts/ 27 | mantisapi.artifact.disk.cache.enabled=true 28 | 29 | mreAppJobClusterMap={"version": "1", "timestamp": 12345, "mappings": {"__default__": {"requestEventStream": "SharedPushRequestEventSource","sentryEventStream": "SentryLogEventSource","__default__": "SharedPushEventSource"},"customApp": {"logEventStream": "CustomAppEventSource","sentryEventStream": "CustomAppSentryLogSource"}}} 30 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015 Netflix, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | log4j.rootLogger=INFO,stdout 18 | 19 | # raise above INFO to disable access logger 20 | log4j.logger.ACCESS=INFO 21 | 22 | # can be set to INFO for netty wire logging 23 | log4j.logger.zuul.server.nettylog=WARN 24 | log4j.logger.zuul.origin.nettylog=WARN 25 | log4j.logger.zuul.api.nettylog=WARN 26 | 27 | log4j.logger.com.netflix.loadbalancer=WARN 28 | log4j.logger.com.netflix.config=WARN 29 | 30 | # stdout 31 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 32 | log4j.appender.stdout.layout.ConversionPattern=%d %-5p %c [%t] %m%n 33 | 34 | # filter out repeating lines for Rx 35 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 36 | 37 | # async appender 38 | batcher.com.netflix.logging.AsyncAppender.stdout.waitTimeinMillis=120000 39 | log4j.logger.asyncAppenders=INFO,stdout 40 | -------------------------------------------------------------------------------- /src/test/java/io/mantisrx/api/UtilTest.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api; 2 | 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.assertArrayEquals; 6 | 7 | public class UtilTest { 8 | 9 | @Test 10 | public void testGetTagList() { 11 | String[] tags = Util.getTaglist("/jobconnectbyname/rx-sps-tracker?clientId=testClientId", "testTargetId", "us-east-1"); 12 | assertArrayEquals(new String[]{ 13 | "clientId", "testClientId", 14 | "SessionId", "testTargetId", 15 | "urlPath", "/jobconnectbyname/rx-sps-tracker", 16 | "region", "us-east-1"}, tags); 17 | 18 | tags = Util.getTaglist("/jobconnectbyname/rx-sps-tracker?clientId=testClientId&MantisApiTag=tag1:value1", "testTargetId", "us-east-1"); 19 | assertArrayEquals(new String[]{ 20 | "tag1", "value1", 21 | "clientId", "testClientId", 22 | "SessionId", "testTargetId", 23 | "urlPath", "/jobconnectbyname/rx-sps-tracker", 24 | "region", "us-east-1"}, tags); 25 | 26 | tags = Util.getTaglist("/jobconnectbyname/rx-sps-tracker?clientId=testClientId&MantisApiTag=tag1:value1&MantisApiTag=clientId:testClientId2", "testTargetId", "us-east-1"); 27 | assertArrayEquals(new String[]{ 28 | "tag1", "value1", 29 | "clientId", "testClientId2", 30 | "SessionId", "testTargetId", 31 | "urlPath", "/jobconnectbyname/rx-sps-tracker", 32 | "region", "us-east-1"}, tags); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/test/java/io/mantisrx/api/tunnel/CrossRegionHandlerTest.java: -------------------------------------------------------------------------------- 1 | package io.mantisrx.api.tunnel; 2 | 3 | import com.google.common.collect.ImmutableList; 4 | import io.mantisrx.api.push.ConnectionBroker; 5 | import junit.framework.TestCase; 6 | import org.junit.Test; 7 | import rx.Scheduler; 8 | 9 | import static org.mockito.Mockito.doReturn; 10 | import static org.mockito.Mockito.mock; 11 | import static org.mockito.Mockito.spy; 12 | 13 | public class CrossRegionHandlerTest extends TestCase { 14 | 15 | @Test 16 | public void testParseUriRegion() { 17 | 18 | CrossRegionHandler regionHandler = spy(new CrossRegionHandler(ImmutableList.of(), mock(MantisCrossRegionalClient.class), mock(ConnectionBroker.class), mock(Scheduler.class))); 19 | doReturn(ImmutableList.of("us-east-1", "eu-west-1")).when(regionHandler).getTunnelRegions(); 20 | 21 | assertEquals(ImmutableList.of("us-east-1"), regionHandler.parseRegionsInUri("/region/us-east-1/foobar")); 22 | assertEquals(ImmutableList.of("us-east-2"), regionHandler.parseRegionsInUri("/region/us-east-2/foobar")); 23 | assertEquals(ImmutableList.of("us-east-1", "eu-west-1"), regionHandler.parseRegionsInUri("/region/all/foobar")); 24 | assertEquals(ImmutableList.of("us-east-1", "eu-west-1"), regionHandler.parseRegionsInUri("/region/ALL/foobar")); 25 | 26 | doReturn(ImmutableList.of("us-east-1", "eu-west-1", "us-west-2")).when(regionHandler).getTunnelRegions(); 27 | assertEquals(ImmutableList.of("us-east-1", "eu-west-1", "us-west-2"), regionHandler.parseRegionsInUri("/region/ALL/foobar")); 28 | 29 | assertEquals(ImmutableList.of("us-east-1", "us-east-2"), regionHandler.parseRegionsInUri("/region/us-east-1,us-east-2/foobar")); 30 | assertEquals(ImmutableList.of("us-east-1", "us-west-2"), regionHandler.parseRegionsInUri("/region/us-east-1,us-west-2/foobar")); 31 | } 32 | } 33 | --------------------------------------------------------------------------------