├── .github
├── CODEOWNERS
├── ISSUE_TEMPLATE.md
├── PULL_REQUEST_TEMPLATE.md
└── workflows
│ ├── nebula-ci.yml
│ ├── nebula-publish.yml
│ ├── nebula-snapshot.yml
│ └── push-docker-image.yml
├── .gitignore
├── CONTRIBUTING.md
├── LICENSE
├── NOTICE
├── OSSMETADATA
├── README.md
├── build.gradle
├── codequality
├── checkstyle.xml
├── netflix-intellij-code-style.xml
└── pmd.xml
├── dependencies.lock
├── docs
└── ARCHITECTURE.md
├── gradle
├── check.gradle
└── wrapper
│ ├── gradle-wrapper.jar
│ └── gradle-wrapper.properties
├── gradlew
├── gradlew.bat
├── settings.gradle
└── src
├── main
├── java
│ └── io
│ │ └── mantisrx
│ │ └── api
│ │ ├── Bootstrap.java
│ │ ├── Constants.java
│ │ ├── MantisAPIModule.java
│ │ ├── MantisConfigurationBasedServerList.java
│ │ ├── MantisServerStartup.java
│ │ ├── Util.java
│ │ ├── filters
│ │ ├── AppStreamDiscovery.java
│ │ ├── Artifacts.java
│ │ ├── Favicon.java
│ │ ├── Healthcheck.java
│ │ ├── JobDiscoveryCacheLoader.java
│ │ ├── JobDiscoveryInfoCacheHitChecker.java
│ │ ├── MQLParser.java
│ │ ├── MREAppStreamToJobClusterMapping.java
│ │ ├── MasterCacheHitChecker.java
│ │ ├── MasterCacheLoader.java
│ │ ├── MetricsReporting.java
│ │ ├── Options.java
│ │ ├── OutboundHeaders.java
│ │ └── Routes.java
│ │ ├── initializers
│ │ └── MantisApiServerChannelInitializer.java
│ │ ├── proto
│ │ ├── AppDiscoveryMap.java
│ │ └── Artifact.java
│ │ ├── push
│ │ ├── ConnectionBroker.java
│ │ ├── MantisSSEHandler.java
│ │ ├── MantisWebSocketFrameHandler.java
│ │ └── PushConnectionDetails.java
│ │ ├── services
│ │ ├── AppStreamDiscoveryService.java
│ │ ├── AppStreamStore.java
│ │ ├── ConfigurationBasedAppStreamStore.java
│ │ ├── JobDiscoveryService.java
│ │ └── artifacts
│ │ │ ├── ArtifactManager.java
│ │ │ └── InMemoryArtifactManager.java
│ │ └── tunnel
│ │ ├── CrossRegionHandler.java
│ │ ├── MantisCrossRegionalClient.java
│ │ ├── NoOpCrossRegionalClient.java
│ │ └── RegionData.java
└── resources
│ ├── api-docker.properties
│ ├── api-local.properties
│ └── log4j.properties
└── test
└── java
└── io
└── mantisrx
└── api
├── UtilTest.java
└── tunnel
└── CrossRegionHandlerTest.java
/.github/CODEOWNERS:
--------------------------------------------------------------------------------
1 | * @calvin681 @sundargates @Andyz26 @hmitnflx @liuml07 @fdc-ntflx
2 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE.md:
--------------------------------------------------------------------------------
1 | ### Context
2 |
3 | Explain symptoms and other details for the issue.
4 |
5 | ### Steps to reproduce
6 |
7 | Explain the steps to reliably reproduce the issue.
8 |
9 | ### Expected behavior
10 |
11 | Explain what you think should happen.
12 |
13 | ### Actual Behavior
14 |
15 | Explain what actually happens instead.
16 |
--------------------------------------------------------------------------------
/.github/PULL_REQUEST_TEMPLATE.md:
--------------------------------------------------------------------------------
1 | ### Context
2 |
3 | Explain context and other details for this pull request.
4 |
5 | ### Checklist
6 |
7 | - [ ] `./gradlew build` compiles code correctly
8 | - [ ] Added new tests where applicable
9 | - [ ] `./gradlew test` passes all tests
10 | - [ ] Extended README or added javadocs where applicable
11 | - [ ] Added copyright headers for new files from `CONTRIBUTING.md`
12 |
--------------------------------------------------------------------------------
/.github/workflows/nebula-ci.yml:
--------------------------------------------------------------------------------
1 | name: "CI"
2 | on:
3 | push:
4 | branches:
5 | - '*'
6 | tags-ignore:
7 | - '*'
8 | pull_request:
9 |
10 | jobs:
11 | build:
12 | runs-on: ubuntu-latest
13 | strategy:
14 | matrix:
15 | # test against JDK 8
16 | java: [ 8 ]
17 | name: CI with Java ${{ matrix.java }}
18 | steps:
19 | - uses: actions/checkout@v1
20 | - run: |
21 | git config --global user.name "Mantis OSS Maintainers"
22 | git config --global user.email "mantis-oss-dev@googlegroups.com"
23 | - name: Setup jdk
24 | uses: actions/setup-java@v1
25 | with:
26 | java-version: ${{ matrix.java }}
27 | - uses: actions/cache@v4
28 | id: gradle-cache
29 | with:
30 | path: ~/.gradle/caches
31 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }}
32 | restore-keys: |
33 | - ${{ runner.os }}-gradle-
34 | - uses: actions/cache@v4
35 | id: gradle-wrapper-cache
36 | with:
37 | path: ~/.gradle/wrapper
38 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }}
39 | restore-keys: |
40 | - ${{ runner.os }}-gradlewrapper-
41 | - name: Build with Gradle
42 | run: ./gradlew --info --stacktrace build
43 | env:
44 | CI_NAME: github_actions
45 | CI_BUILD_NUMBER: ${{ github.sha }}
46 | CI_BUILD_URL: 'https://github.com/${{ github.repository }}'
47 | CI_BRANCH: ${{ github.ref }}
48 | COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
49 |
--------------------------------------------------------------------------------
/.github/workflows/nebula-publish.yml:
--------------------------------------------------------------------------------
1 | name: "Publish candidate/release to NetflixOSS and Maven Central"
2 | on:
3 | push:
4 | tags:
5 | - v*.*.*
6 | - v*.*.*-rc.*
7 | release:
8 | types:
9 | - published
10 |
11 | jobs:
12 | build:
13 | runs-on: ubuntu-latest
14 | steps:
15 | - uses: actions/checkout@v1
16 | - run: |
17 | git config --global user.name "Mantis OSS Maintainers"
18 | git config --global user.email "mantis-oss-dev@googlegroups.com"
19 | - name: Setup jdk 8
20 | uses: actions/setup-java@v1
21 | with:
22 | java-version: 1.8
23 | - uses: actions/cache@v4
24 | id: gradle-cache
25 | with:
26 | path: ~/.gradle/caches
27 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/gradle/dependency-locks/*.lockfile') }}
28 | restore-keys: |
29 | - ${{ runner.os }}-gradle-
30 | - uses: actions/cache@v4
31 | id: gradle-wrapper-cache
32 | with:
33 | path: ~/.gradle/wrapper
34 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }}
35 | restore-keys: |
36 | - ${{ runner.os }}-gradlewrapper-
37 | - name: Publish candidate
38 | if: contains(github.ref, '-rc.')
39 | run: ./gradlew --info --stacktrace -Prelease.useLastTag=true candidate
40 | env:
41 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }}
42 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }}
43 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }}
44 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }}
45 | - name: Publish release
46 | if: (!contains(github.ref, '-rc.'))
47 | run: ./gradlew --info -Prelease.useLastTag=true final
48 | env:
49 | NETFLIX_OSS_SONATYPE_USERNAME: ${{ secrets.ORG_SONATYPE_USERNAME }}
50 | NETFLIX_OSS_SONATYPE_PASSWORD: ${{ secrets.ORG_SONATYPE_PASSWORD }}
51 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }}
52 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }}
53 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }}
54 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }}
55 | NETFLIX_OSS_SONATYPE_STAGING_PROFILE_ID: "c3547130240327"
56 |
--------------------------------------------------------------------------------
/.github/workflows/nebula-snapshot.yml:
--------------------------------------------------------------------------------
1 | name: "Publish snapshot to NetflixOSS and Maven Central"
2 |
3 | on:
4 | push:
5 | branches:
6 | - master
7 |
8 | jobs:
9 | build:
10 | runs-on: ubuntu-latest
11 | steps:
12 | - uses: actions/checkout@v2
13 | with:
14 | fetch-depth: 0
15 | - run: |
16 | git config --global user.name "Mantis OSS Maintainers"
17 | git config --global user.email "mantis-oss-dev@googlegroups.com"
18 | - name: Set up JDK
19 | uses: actions/setup-java@v1
20 | with:
21 | java-version: 8
22 | - uses: actions/cache@v4
23 | id: gradle-cache
24 | with:
25 | path: |
26 | ~/.gradle/caches
27 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
28 | - uses: actions/cache@v4
29 | id: gradle-wrapper-cache
30 | with:
31 | path: |
32 | ~/.gradle/wrapper
33 | key: ${{ runner.os }}-gradlewrapper-${{ hashFiles('gradle/wrapper/*') }}
34 | - name: Build
35 | run: ./gradlew build snapshot
36 | env:
37 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }}
38 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }}
39 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }}
40 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }}
41 |
--------------------------------------------------------------------------------
/.github/workflows/push-docker-image.yml:
--------------------------------------------------------------------------------
1 | name: "Publish Docker images to Docker Registry"
2 |
3 | on:
4 | pull_request:
5 | paths:
6 | - ./build.gradle
7 | - .github/workflows/push-docker-image.yml
8 | push:
9 | tags:
10 | - v*.*.*
11 | - v*.*.*-rc.*
12 | release:
13 | types:
14 | - published
15 |
16 | jobs:
17 | build-and-push-image:
18 | runs-on: ubuntu-latest
19 | strategy:
20 | fail-fast: false
21 | permissions:
22 | contents: read
23 | packages: write
24 |
25 | steps:
26 | - name: Checkout PR
27 | uses: actions/checkout@v3
28 | - name: Setup jdk
29 | uses: actions/setup-java@v1
30 | with:
31 | java-version: 8
32 |
33 | - name: Generate dockerfiles
34 | uses: gradle/gradle-build-action@v2
35 | env:
36 | NETFLIX_OSS_SIGNING_KEY: ${{ secrets.ORG_SIGNING_KEY }}
37 | NETFLIX_OSS_SIGNING_PASSWORD: ${{ secrets.ORG_SIGNING_PASSWORD }}
38 | NETFLIX_OSS_REPO_USERNAME: ${{ secrets.ORG_NETFLIXOSS_USERNAME }}
39 | NETFLIX_OSS_REPO_PASSWORD: ${{ secrets.ORG_NETFLIXOSS_PASSWORD }}
40 | with:
41 | arguments: --info --stacktrace :dockerCreateDockerfile
42 |
43 | - name: Fetch tags
44 | run: |
45 | git fetch --prune --unshallow --tags
46 | - name: Login to Docker Hub
47 | id: login-docker-hub
48 | uses: docker/login-action@v2
49 | with:
50 | username: ${{ secrets.ORG_NETFLIXOSS_DOCKERHUB_USERNAME }}
51 | password: ${{ secrets.ORG_NETFLIXOSS_DOCKERHUB_PASSWORD }}
52 |
53 | - name: Docker meta
54 | id: meta
55 | uses: docker/metadata-action@v4
56 | with:
57 | # list of Docker images to use as base name for tags
58 | images: "netflixoss/mantisapi"
59 | # generate Docker tags based on the following events/attributes
60 | # we generate the latest tag off the beta branch
61 | tags: |
62 | type=ref,event=branch
63 | type=ref,event=pr
64 | type=semver,pattern={{version}}
65 | type=semver,pattern={{major}}.{{minor}}
66 | type=raw,value=latest,enable=${{ github.event_name == 'release' }}
67 |
68 | - name: Set up QEMU
69 | uses: docker/setup-qemu-action@v2
70 |
71 | - name: Set up Docker Buildx
72 | uses: docker/setup-buildx-action@v2
73 |
74 | - name: Build and push Docker images
75 | uses: docker/build-push-action@v3
76 | with:
77 | context: ./build/docker
78 | file: ./build/docker/Dockerfile
79 | push: ${{ github.event_name != 'pull_request' }}
80 | tags: ${{ steps.meta.outputs.tags }}
81 | labels: ${{ steps.meta.outputs.labels }}
82 | platforms: linux/amd64,linux/arm64
83 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | # Created by https://www.gitignore.io/api/vim,java,linux,macos,gradle,intellij+all
3 | # Edit at https://www.gitignore.io/?templates=vim,java,linux,macos,gradle,intellij+all
4 |
5 | ### Intellij+all ###
6 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
7 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
8 |
9 | # User-specific stuff
10 | .idea/**/workspace.xml
11 | .idea/**/tasks.xml
12 | .idea/**/usage.statistics.xml
13 | .idea/**/dictionaries
14 | .idea/**/shelf
15 |
16 | # Generated files
17 | .idea/**/contentModel.xml
18 |
19 | # Sensitive or high-churn files
20 | .idea/**/dataSources/
21 | .idea/**/dataSources.ids
22 | .idea/**/dataSources.local.xml
23 | .idea/**/sqlDataSources.xml
24 | .idea/**/dynamic.xml
25 | .idea/**/uiDesigner.xml
26 | .idea/**/dbnavigator.xml
27 |
28 | # Gradle
29 | .idea/**/gradle.xml
30 | .idea/**/libraries
31 |
32 | # Gradle and Maven with auto-import
33 | # When using Gradle or Maven with auto-import, you should exclude module files,
34 | # since they will be recreated, and may cause churn. Uncomment if using
35 | # auto-import.
36 | # .idea/modules.xml
37 | # .idea/*.iml
38 | # .idea/modules
39 |
40 | # CMake
41 | cmake-build-*/
42 |
43 | # Mongo Explorer plugin
44 | .idea/**/mongoSettings.xml
45 |
46 | # File-based project format
47 | *.iws
48 |
49 | # IntelliJ
50 | out/
51 |
52 | # mpeltonen/sbt-idea plugin
53 | .idea_modules/
54 |
55 | # JIRA plugin
56 | atlassian-ide-plugin.xml
57 |
58 | # Cursive Clojure plugin
59 | .idea/replstate.xml
60 |
61 | # Crashlytics plugin (for Android Studio and IntelliJ)
62 | com_crashlytics_export_strings.xml
63 | crashlytics.properties
64 | crashlytics-build.properties
65 | fabric.properties
66 |
67 | # Editor-based Rest Client
68 | .idea/httpRequests
69 |
70 | # Android studio 3.1+ serialized cache file
71 | .idea/caches/build_file_checksums.ser
72 |
73 | # JetBrains templates
74 | **___jb_tmp___
75 |
76 | ### Intellij+all Patch ###
77 | # Ignores the whole .idea folder and all .iml files
78 | # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360
79 |
80 | .idea/
81 |
82 | # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023
83 |
84 | *.iml
85 | modules.xml
86 | .idea/misc.xml
87 | *.ipr
88 |
89 | # Sonarlint plugin
90 | .idea/sonarlint
91 |
92 | ### Java ###
93 | # Compiled class file
94 | *.class
95 |
96 | # Log file
97 | *.log
98 |
99 | # BlueJ files
100 | *.ctxt
101 |
102 | # Mobile Tools for Java (J2ME)
103 | .mtj.tmp/
104 |
105 | # Package Files #
106 | *.jar
107 | *.war
108 | *.nar
109 | *.ear
110 | *.zip
111 | *.tar.gz
112 | *.rar
113 |
114 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
115 | hs_err_pid*
116 |
117 | ### Linux ###
118 | *~
119 |
120 | # temporary files which can be created if a process still has a handle open of a deleted file
121 | .fuse_hidden*
122 |
123 | # KDE directory preferences
124 | .directory
125 |
126 | # Linux trash folder which might appear on any partition or disk
127 | .Trash-*
128 |
129 | # .nfs files are created when an open file is removed but is still being accessed
130 | .nfs*
131 |
132 | ### macOS ###
133 | # General
134 | .DS_Store
135 | .AppleDouble
136 | .LSOverride
137 |
138 | # Icon must end with two \r
139 | Icon
140 |
141 | # Thumbnails
142 | ._*
143 |
144 | # Files that might appear in the root of a volume
145 | .DocumentRevisions-V100
146 | .fseventsd
147 | .Spotlight-V100
148 | .TemporaryItems
149 | .Trashes
150 | .VolumeIcon.icns
151 | .com.apple.timemachine.donotpresent
152 |
153 | # Directories potentially created on remote AFP share
154 | .AppleDB
155 | .AppleDesktop
156 | Network Trash Folder
157 | Temporary Items
158 | .apdisk
159 |
160 | ### Vim ###
161 | # Swap
162 | [._]*.s[a-v][a-z]
163 | [._]*.sw[a-p]
164 | [._]s[a-rt-v][a-z]
165 | [._]ss[a-gi-z]
166 | [._]sw[a-p]
167 |
168 | # Session
169 | Session.vim
170 |
171 | # Temporary
172 | .netrwhist
173 | # Auto-generated tag files
174 | tags
175 | # Persistent undo
176 | [._]*.un~
177 |
178 | ### Gradle ###
179 | .gradle
180 | build/
181 |
182 | # Ignore Gradle GUI config
183 | gradle-app.setting
184 |
185 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored)
186 | !gradle-wrapper.jar
187 |
188 | # Cache of project
189 | .gradletasknamecache
190 |
191 | # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898
192 | # gradle/wrapper/gradle-wrapper.properties
193 |
194 | ### Gradle Patch ###
195 | **/build/
196 |
197 | # End of https://www.gitignore.io/api/vim,java,linux,macos,gradle,intellij+all
198 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing to Mantis
2 |
3 | If you would like to contribute code you can do so through GitHub by forking the repository and sending a pull request. When submitting code, please make every effort to follow existing conventions and style in order to keep the code as readable as possible.
4 |
5 | ## License
6 |
7 | By contributing your code, you agree to license your contribution under the terms of the [Apache License v2.0](http://www.apache.org/licenses/LICENSE-2.0). Your contributions should also include the following header:
8 |
9 | ```
10 | /**
11 | * Copyright 2019 Netflix, Inc.
12 | *
13 | * Licensed under the Apache License, Version 2.0 (the "License");
14 | * you may not use this file except in compliance with the License.
15 | * You may obtain a copy of the License at
16 | *
17 | * http://www.apache.org/licenses/LICENSE-2.0
18 | *
19 | * Unless required by applicable law or agreed to in writing, software
20 | * distributed under the License is distributed on an "AS IS" BASIS,
21 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22 | * See the License for the specific language governing permissions and
23 | * limitations under the License.
24 | */
25 | ```
26 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright [yyyy] [name of copyright owner]
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
203 |
--------------------------------------------------------------------------------
/NOTICE:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Netflix/mantis-api/d83dc0623ff12a7755ba9557d569411edb02656b/NOTICE
--------------------------------------------------------------------------------
/OSSMETADATA:
--------------------------------------------------------------------------------
1 | osslifecycle=active
2 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # mantis-api
2 |
3 | [](https://travis-ci.com/Netflix/mantis-api)
4 | [](https://github.com/Netflix/mantis-api)
5 | [](https://www.apache.org/licenses/LICENSE-2.0)
6 |
7 | ## Development
8 |
9 | ### Building
10 |
11 | ```sh
12 | $ ./gradlew clean build
13 | ```
14 |
15 | ### Testing
16 |
17 | ```sh
18 | $ ./gradlew clean test
19 | ```
20 |
21 | ### Building deployment into local Maven cache
22 |
23 | ```sh
24 | $ ./gradlew clean publishNebulaPublicationToMavenLocal
25 | ```
26 |
27 | ### Releasing
28 |
29 | We release by tagging which kicks off a CI build. The CI build will run tests, integration tests,
30 | static analysis, checkstyle, build, and then publish to the public Bintray repo to be synced into Maven Central.
31 |
32 | Tag format:
33 |
34 | ```
35 | vMajor.Minor.Patch
36 | ```
37 |
38 | You can tag via git or through Github's Release UI.
39 |
40 | ## Contributing
41 |
42 | Mantis is interested in building the community. We welcome any forms of contributions through discussions on any
43 | of our [mailing lists](https://netflix.github.io/mantis/community/#mailing-lists) or through patches.
44 |
45 | For more information on contribution, check out the contributions file here:
46 |
47 | - [https://github.com/Netflix/mantis/blob/master/CONTRIBUTING.md](https://github.com/Netflix/mantis/blob/master/CONTRIBUTING.md)
48 |
--------------------------------------------------------------------------------
/build.gradle:
--------------------------------------------------------------------------------
1 | import org.gradle.api.internal.project.ProjectInternal
2 |
3 | /*
4 | * Copyright 2019 Netflix, Inc.
5 | *
6 | * Licensed under the Apache License, Version 2.0 (the "License");
7 | * you may not use this file except in compliance with the License.
8 | * You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | buildscript {
20 | repositories {
21 | mavenCentral()
22 | maven {
23 | url "https://plugins.gradle.org/m2/"
24 | }
25 | gradlePluginPortal()
26 | }
27 |
28 | dependencies {
29 | classpath 'com.netflix.nebula:gradle-netflixoss-project-plugin:11.5.0'
30 | classpath 'com.bmuschko:gradle-docker-plugin:6.7.0'
31 | classpath 'com.palantir.gradle.gitversion:gradle-git-version:3.0.0'
32 | }
33 | }
34 |
35 | apply plugin: 'nebula.netflixoss'
36 | apply plugin: 'idea'
37 | apply plugin: 'java-library'
38 | apply plugin: 'pmd'
39 | apply plugin: 'application'
40 | apply plugin: 'com.bmuschko.docker-java-application'
41 | apply plugin: 'com.palantir.git-version'
42 |
43 | ext {
44 | archaiusVersion = '2.3.+'
45 | assetjCoreVersion = '3.11.1'
46 | guavaVersion = '19.+'
47 | gsonVersion = '2.8.+'
48 | guiceVersion = '4.1.0'
49 | jettyVersion = '9.4.12.v20180830'
50 | jsonVersion = '20160810'
51 | junitVersion = '4.10'
52 | lombokVersion = '1.18.10'
53 | mantisVersion = '2.0.98'
54 | mockitoVersion = '3.+'
55 | s3Version = '1.11.566'
56 | servletApiVersion = '3.1.0'
57 | spectatorVersion = '0.92.+'
58 | vavrVersion = '0.10.2'
59 | zuulVersion = '2.3.0'
60 | }
61 |
62 | group = 'io.mantisrx'
63 |
64 | sourceCompatibility = JavaVersion.VERSION_1_8
65 | targetCompatibility = JavaVersion.VERSION_1_8
66 |
67 | repositories {
68 | mavenCentral()
69 | maven {
70 | url "https://netflixoss.jfrog.io/artifactory/maven-oss-candidates"
71 | }
72 | maven {
73 | url "https://artifacts-oss.netflix.net/artifactory/maven-oss-snapshots"
74 | }
75 | }
76 |
77 | tasks.withType(Javadoc).all {
78 | enabled = false
79 | }
80 |
81 |
82 | dependencies {
83 | api("com.netflix.zuul:zuul-core:${zuulVersion}") {
84 | attributes { attribute(ProjectInternal.STATUS_ATTRIBUTE, 'release') }
85 | exclude group: 'com.google.guava', module: 'guava'
86 | exclude group: 'com.google.inject', module: 'guice'
87 | }
88 | api("com.netflix.zuul:zuul-guice:${zuulVersion}") {
89 | attributes { attribute(ProjectInternal.STATUS_ATTRIBUTE, 'release') }
90 | exclude group: 'com.google.guava', module: 'guava'
91 | exclude group: 'com.google.inject', module: 'guice'
92 | }
93 | implementation("com.netflix.zuul:zuul-groovy:${zuulVersion}") {
94 | exclude group: 'com.google.guava', module: 'guava'
95 | exclude group: 'com.google.inject', module: 'guice'
96 | }
97 | implementation 'com.netflix.blitz4j:blitz4j:1.37.2'
98 | implementation ("com.netflix.governator:governator:1.+") {
99 | attributes { attribute(ProjectInternal.STATUS_ATTRIBUTE, 'release') }
100 | }
101 | implementation "org.apache.commons:commons-lang3:3.+"
102 | implementation "commons-configuration:commons-configuration:1.8"
103 | compileOnly "com.netflix.ribbon:ribbon-loadbalancer:2.4.4"
104 | compileOnly "com.netflix.ribbon:ribbon-eureka:2.4.4"
105 | api "io.mantisrx:mantis-discovery-proto:$mantisVersion"
106 | api "io.mantisrx:mantis-client:$mantisVersion"
107 | api "io.mantisrx:mantis-runtime:$mantisVersion"
108 | api "io.mantisrx:mantis-control-plane-client:$mantisVersion"
109 | api "io.mantisrx:mantis-server-worker-client:$mantisVersion"
110 | api "io.mantisrx:mql-jvm:3.+"
111 | api "io.vavr:vavr:$vavrVersion"
112 | api "com.google.inject:guice:$guiceVersion"
113 | api "org.projectlombok:lombok:$lombokVersion"
114 | annotationProcessor "org.projectlombok:lombok:$lombokVersion"
115 |
116 | testImplementation "junit:junit:$junitVersion"
117 | testImplementation "org.mockito:mockito-core:$mockitoVersion"
118 | testImplementation"org.projectlombok:lombok:$lombokVersion"
119 | testAnnotationProcessor "org.projectlombok:lombok:$lombokVersion"
120 | }
121 |
122 |
123 | configurations.all {
124 | resolutionStrategy {
125 | force "com.google.guava:guava:$guavaVersion"
126 | force "com.google.inject:guice:$guiceVersion"
127 | force "com.google.inject.extensions:guice-multibindings:$guiceVersion"
128 | force "com.google.inject.extensions:guice-grapher:$guiceVersion"
129 | force "com.google.inject.extensions:guice-assistedinject:$guiceVersion"
130 | force "javax.servlet:javax.servlet-api:$servletApiVersion"
131 | }
132 | }
133 |
134 | /*
135 | * Run regular: ./gradlew run
136 | * Run benchmark: ./gradlew run -Pbench
137 | */
138 | run {
139 | mainClassName = "io.mantisrx.api.Bootstrap"
140 |
141 | applicationDefaultJvmArgs = ["-DTZ=GMT",
142 | "-Darchaius.deployment.environment=test",
143 | "-Dcom.sun.management.jmxremote",
144 | "-Dcom.sun.management.jmxremote.local.only=false",
145 | "-Deureka.validateInstanceId=false",
146 | "-Deureka.mt.num_retries=1"]
147 |
148 | if (project.hasProperty('bench')) {
149 | println 'Running benchmark configuration...'
150 | jvmArgs "-Darchaius.deployment.environment=benchmark"
151 | }
152 | }
153 |
154 | def installDir = file("${buildDir}/install")
155 | def resourcesDir = file("${buildDir}/resources")
156 | def ci = System.getenv('GITHUB_ACTIONS')
157 | def imageRepository = ci ? 'netflixoss' : 'localhost:5001/netflixoss'
158 | def dockerVersionTag = gitVersion()
159 |
160 | docker {
161 | dockerSyncBuildContext {
162 | from installDir
163 | from resourcesDir
164 | }
165 |
166 | dockerCreateDockerfile {
167 | // root directory to store all the files
168 | instruction 'WORKDIR /apps/mantis/mantis-api'
169 | // copy the files from the build context to the image
170 | instruction 'COPY mantis-api/bin/* /apps/mantis/mantis-api/bin/'
171 | instruction 'COPY mantis-api/lib/* /apps/mantis/mantis-api/lib/'
172 | instruction 'COPY resources/* conf/'
173 | entryPoint 'bin/mantis-api', '-p', 'api-docker.properties'
174 | }
175 |
176 | javaApplication {
177 | baseImage = 'azul/zulu-openjdk:8-latest'
178 | maintainer = 'Mantis Developers "mantis-oss-dev@netflix.com"'
179 | mainClassName = 'io.mantisrx.api.Bootstrap'
180 | images = ["$imageRepository/mantisapi:latest", "$imageRepository/mantisapi:$dockerVersionTag"]
181 | ports = [5050]
182 | }
183 | }
184 |
185 | dockerSyncBuildContext.dependsOn(installDist)
186 |
187 | pmd {
188 | ignoreFailures = true
189 | }
190 |
--------------------------------------------------------------------------------
/codequality/checkstyle.xml:
--------------------------------------------------------------------------------
1 |
2 |
17 |
18 |
21 |
22 |
23 |
24 |
25 |
26 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
152 |
158 |
159 |
160 |
163 |
164 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
203 |
--------------------------------------------------------------------------------
/codequality/netflix-intellij-code-style.xml:
--------------------------------------------------------------------------------
1 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
--------------------------------------------------------------------------------
/codequality/pmd.xml:
--------------------------------------------------------------------------------
1 |
2 |
17 |
18 |
23 |
24 | Exclude noisy rules.
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
--------------------------------------------------------------------------------
/dependencies.lock:
--------------------------------------------------------------------------------
1 | {
2 | "annotationProcessor": {
3 | "org.projectlombok:lombok": {
4 | "locked": "1.18.10"
5 | }
6 | },
7 | "compileClasspath": {
8 | "com.google.inject:guice": {
9 | "locked": "4.1.0"
10 | },
11 | "com.netflix.blitz4j:blitz4j": {
12 | "locked": "1.37.2"
13 | },
14 | "com.netflix.governator:governator": {
15 | "locked": "1.17.13"
16 | },
17 | "com.netflix.ribbon:ribbon-eureka": {
18 | "locked": "2.4.4"
19 | },
20 | "com.netflix.ribbon:ribbon-loadbalancer": {
21 | "locked": "2.4.4"
22 | },
23 | "com.netflix.zuul:zuul-core": {
24 | "locked": "2.3.0"
25 | },
26 | "com.netflix.zuul:zuul-groovy": {
27 | "locked": "2.3.0"
28 | },
29 | "com.netflix.zuul:zuul-guice": {
30 | "locked": "2.3.0"
31 | },
32 | "commons-configuration:commons-configuration": {
33 | "locked": "1.8"
34 | },
35 | "io.mantisrx:mantis-client": {
36 | "locked": "2.0.98"
37 | },
38 | "io.mantisrx:mantis-control-plane-client": {
39 | "locked": "2.0.98"
40 | },
41 | "io.mantisrx:mantis-discovery-proto": {
42 | "locked": "2.0.98"
43 | },
44 | "io.mantisrx:mantis-runtime": {
45 | "locked": "2.0.98"
46 | },
47 | "io.mantisrx:mantis-server-worker-client": {
48 | "locked": "2.0.98"
49 | },
50 | "io.mantisrx:mql-jvm": {
51 | "locked": "3.4.0"
52 | },
53 | "io.vavr:vavr": {
54 | "locked": "0.10.2"
55 | },
56 | "org.apache.commons:commons-lang3": {
57 | "locked": "3.13.0"
58 | },
59 | "org.projectlombok:lombok": {
60 | "locked": "1.18.10"
61 | }
62 | },
63 | "pmd": {
64 | "net.sourceforge.pmd:pmd-java": {
65 | "locked": "6.26.0"
66 | }
67 | },
68 | "runtimeClasspath": {
69 | "com.google.inject:guice": {
70 | "locked": "4.1.0"
71 | },
72 | "com.netflix.blitz4j:blitz4j": {
73 | "locked": "1.37.2"
74 | },
75 | "com.netflix.governator:governator": {
76 | "locked": "1.17.13"
77 | },
78 | "com.netflix.zuul:zuul-core": {
79 | "locked": "2.3.0"
80 | },
81 | "com.netflix.zuul:zuul-groovy": {
82 | "locked": "2.3.0"
83 | },
84 | "com.netflix.zuul:zuul-guice": {
85 | "locked": "2.3.0"
86 | },
87 | "commons-configuration:commons-configuration": {
88 | "locked": "1.8"
89 | },
90 | "io.mantisrx:mantis-client": {
91 | "locked": "2.0.98"
92 | },
93 | "io.mantisrx:mantis-control-plane-client": {
94 | "locked": "2.0.98"
95 | },
96 | "io.mantisrx:mantis-discovery-proto": {
97 | "locked": "2.0.98"
98 | },
99 | "io.mantisrx:mantis-runtime": {
100 | "locked": "2.0.98"
101 | },
102 | "io.mantisrx:mantis-server-worker-client": {
103 | "locked": "2.0.98"
104 | },
105 | "io.mantisrx:mql-jvm": {
106 | "locked": "3.4.0"
107 | },
108 | "io.vavr:vavr": {
109 | "locked": "0.10.2"
110 | },
111 | "org.apache.commons:commons-lang3": {
112 | "locked": "3.13.0"
113 | },
114 | "org.projectlombok:lombok": {
115 | "locked": "1.18.10"
116 | }
117 | },
118 | "testAnnotationProcessor": {
119 | "org.projectlombok:lombok": {
120 | "locked": "1.18.10"
121 | }
122 | },
123 | "testCompileClasspath": {
124 | "com.google.inject:guice": {
125 | "locked": "4.1.0"
126 | },
127 | "com.netflix.blitz4j:blitz4j": {
128 | "locked": "1.37.2"
129 | },
130 | "com.netflix.governator:governator": {
131 | "locked": "1.17.13"
132 | },
133 | "com.netflix.zuul:zuul-core": {
134 | "locked": "2.3.0"
135 | },
136 | "com.netflix.zuul:zuul-groovy": {
137 | "locked": "2.3.0"
138 | },
139 | "com.netflix.zuul:zuul-guice": {
140 | "locked": "2.3.0"
141 | },
142 | "commons-configuration:commons-configuration": {
143 | "locked": "1.8"
144 | },
145 | "io.mantisrx:mantis-client": {
146 | "locked": "2.0.98"
147 | },
148 | "io.mantisrx:mantis-control-plane-client": {
149 | "locked": "2.0.98"
150 | },
151 | "io.mantisrx:mantis-discovery-proto": {
152 | "locked": "2.0.98"
153 | },
154 | "io.mantisrx:mantis-runtime": {
155 | "locked": "2.0.98"
156 | },
157 | "io.mantisrx:mantis-server-worker-client": {
158 | "locked": "2.0.98"
159 | },
160 | "io.mantisrx:mql-jvm": {
161 | "locked": "3.4.0"
162 | },
163 | "io.vavr:vavr": {
164 | "locked": "0.10.2"
165 | },
166 | "junit:junit": {
167 | "locked": "4.13.1"
168 | },
169 | "org.apache.commons:commons-lang3": {
170 | "locked": "3.13.0"
171 | },
172 | "org.mockito:mockito-core": {
173 | "locked": "3.12.4"
174 | },
175 | "org.projectlombok:lombok": {
176 | "locked": "1.18.10"
177 | }
178 | },
179 | "testRuntimeClasspath": {
180 | "com.google.inject:guice": {
181 | "locked": "4.1.0"
182 | },
183 | "com.netflix.blitz4j:blitz4j": {
184 | "locked": "1.37.2"
185 | },
186 | "com.netflix.governator:governator": {
187 | "locked": "1.17.13"
188 | },
189 | "com.netflix.zuul:zuul-core": {
190 | "locked": "2.3.0"
191 | },
192 | "com.netflix.zuul:zuul-groovy": {
193 | "locked": "2.3.0"
194 | },
195 | "com.netflix.zuul:zuul-guice": {
196 | "locked": "2.3.0"
197 | },
198 | "commons-configuration:commons-configuration": {
199 | "locked": "1.8"
200 | },
201 | "io.mantisrx:mantis-client": {
202 | "locked": "2.0.98"
203 | },
204 | "io.mantisrx:mantis-control-plane-client": {
205 | "locked": "2.0.98"
206 | },
207 | "io.mantisrx:mantis-discovery-proto": {
208 | "locked": "2.0.98"
209 | },
210 | "io.mantisrx:mantis-runtime": {
211 | "locked": "2.0.98"
212 | },
213 | "io.mantisrx:mantis-server-worker-client": {
214 | "locked": "2.0.98"
215 | },
216 | "io.mantisrx:mql-jvm": {
217 | "locked": "3.4.0"
218 | },
219 | "io.vavr:vavr": {
220 | "locked": "0.10.2"
221 | },
222 | "junit:junit": {
223 | "locked": "4.13.1"
224 | },
225 | "org.apache.commons:commons-lang3": {
226 | "locked": "3.13.0"
227 | },
228 | "org.mockito:mockito-core": {
229 | "locked": "3.12.4"
230 | },
231 | "org.projectlombok:lombok": {
232 | "locked": "1.18.10"
233 | }
234 | }
235 | }
--------------------------------------------------------------------------------
/docs/ARCHITECTURE.md:
--------------------------------------------------------------------------------
1 | # Mantis API Architecture
2 | The primary objective of Mantis API is to provide a horizontally scalable front to the Mantis Master. The role of the API is to relieve pressure from the Master bvy reducing the total number of incoming connections.
3 |
4 | There are three primary pieces of functionality;
5 |
6 | 1. Proxying Calls to Mantis Master
7 | 1. Streaming data from Mantis Jobs on behalf of client
8 | 1. Proxying data between regions using the tunnel
9 |
10 | These pieces are wired together using Zuul's concept of a `ChannelInitializer` which is a subclass of Netty's concept of the same name. The [MantisApiServerChannelInitializer](src/main/java/io/mantisrx/api/initializers/MantisApiServerChannelInitializer.java)
11 | sets up the Netty channel and then adds handlers based on the route. Each of these pieces of functionality are discussed in turn below.
12 |
13 | ## Proxying calls to Mantis Master
14 | The API by default will proxy any unintercepted HTTP request to the Mantis Master. This is handled by the underlying [Zuul](https://github.com/netflix/zuul) implementation, we
15 | treat the Mantis Master as the Zuul Origin `api`, which treats all requests beginning with `/api` as calls to the specified origin. We always want this origin to be our current leader in the Manits Master cluster.
16 |
17 | There are three aspects of the api which are germain to proxying to Mantis Master:
18 |
19 | ### Static Server List and Leader Election
20 | Zuul typically uses Eureka to discover instances of the service to which it is proxying. This doesn't work for Mantis as we have a specific leader to which we want to proxy.
21 | To this end we use Zuul's static server list feature, which we update via the `masterClient.getMasterMonitor()` in [MantisServerStartup.java](../src/main/java/io/mantisrx/api/MantisServerStartup.java).
22 | Here we observe the Master Monitor and set the `api.ribbon.listOfServers` Note the origin name `api` at the beginning of this.
23 |
24 | ### Routes
25 | [Routes](../src/main/java/io/mantisrx/api/filters/Routes.java) affect this, and may cause a call to be routed to other "filters". Anything directed to `ZuulEndPointRunner.PROXY_ENDPOINT_FILTER_NAME` will
26 | be proxied to the master.
27 |
28 | ### Filters
29 | Zuul Filters may manipulate either incoming requests before they're proxied, or outgoinmg requests after they are proxied. We have as a policy attempted to minimize the amount of manipulation. The reason
30 | for this is simple, we want the API defined by `mantismasterv2` to define the standard, however we must support existing client expectations. Therefore minimizing differences in master responses and API responses
31 | achieves this.
32 |
33 | See [src/main/java/io/mantisrx/api/filters](../src/main/java/io/mantisrx/api/filters) for a list of all filters loaded at runtime. Note that these may be incoming, outgoing, or endpoints.
34 |
35 | ## Streaming Data
36 | The api is tasked with providing an interface to end users, via websocket/sse, to access data streaming from jobs.
37 | This is achieved through the [io.mantisrx.api.push](../src/main/java/io/mantisrx/api/push) namespace which contains a few critical components.
38 |
39 | * PushConnectionDetails is a domain object which fully specifies the target (including cross regional) of a streaming data request.
40 | * ConnectionBroker is responsible for acquiring (and potentially multiplexing) data for a PushConnectionDetails instance and returning an `Observable`
41 | * MantisSSEHandler is responsible for determining if the request is a websocket upgrade and the `Observable` serving server sent events if it is not an upgrade.
42 | * MantisWebSocketFrameHandler is responsible for serving the `Observable` over websocket.
43 |
44 | ## Cross Regional Data
45 | The ConnectionBroker also handles cross-regional data streams with the help of the contents of the [io.mantisrx.api.tunnel](../src/main/java/io/mantisrx/api/tunnel) namespace.
46 | In the open source world we do not have a tunnel provider, it is up to the user to implement `MantisCrossRegionalClient`. Internally we have a Metatron based implementation which mutually auths the servers before
47 | exchanging data over SSL.
48 |
--------------------------------------------------------------------------------
/gradle/check.gradle:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2019 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | checkstyle {
18 | toolVersion = '8.14'
19 | // TODO: Don't ignore failures.
20 | ignoreFailures = true
21 | configFile = rootProject.file('codequality/checkstyle.xml')
22 | sourceSets = [sourceSets.main]
23 | }
24 |
25 | pmd {
26 | toolVersion = '6.9.0'
27 | // TODO: Don't ignore failures.
28 | ignoreFailures = true
29 | sourceSets = [sourceSets.main]
30 | ruleSets = []
31 | ruleSetFiles = rootProject.files("codequality/pmd.xml")
32 | }
33 |
--------------------------------------------------------------------------------
/gradle/wrapper/gradle-wrapper.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Netflix/mantis-api/d83dc0623ff12a7755ba9557d569411edb02656b/gradle/wrapper/gradle-wrapper.jar
--------------------------------------------------------------------------------
/gradle/wrapper/gradle-wrapper.properties:
--------------------------------------------------------------------------------
1 | #
2 | # Copyright 2019 Netflix, Inc.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | #
16 | distributionBase=GRADLE_USER_HOME
17 | distributionPath=wrapper/dists
18 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip
19 | zipStoreBase=GRADLE_USER_HOME
20 | zipStorePath=wrapper/dists
21 |
--------------------------------------------------------------------------------
/gradlew:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env sh
2 |
3 | ##############################################################################
4 | ##
5 | ## Gradle start up script for UN*X
6 | ##
7 | ##############################################################################
8 |
9 | # Attempt to set APP_HOME
10 | # Resolve links: $0 may be a link
11 | PRG="$0"
12 | # Need this for relative symlinks.
13 | while [ -h "$PRG" ] ; do
14 | ls=`ls -ld "$PRG"`
15 | link=`expr "$ls" : '.*-> \(.*\)$'`
16 | if expr "$link" : '/.*' > /dev/null; then
17 | PRG="$link"
18 | else
19 | PRG=`dirname "$PRG"`"/$link"
20 | fi
21 | done
22 | SAVED="`pwd`"
23 | cd "`dirname \"$PRG\"`/" >/dev/null
24 | APP_HOME="`pwd -P`"
25 | cd "$SAVED" >/dev/null
26 |
27 | APP_NAME="Gradle"
28 | APP_BASE_NAME=`basename "$0"`
29 |
30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
31 | DEFAULT_JVM_OPTS=""
32 |
33 | # Use the maximum available, or set MAX_FD != -1 to use that value.
34 | MAX_FD="maximum"
35 |
36 | warn () {
37 | echo "$*"
38 | }
39 |
40 | die () {
41 | echo
42 | echo "$*"
43 | echo
44 | exit 1
45 | }
46 |
47 | # OS specific support (must be 'true' or 'false').
48 | cygwin=false
49 | msys=false
50 | darwin=false
51 | nonstop=false
52 | case "`uname`" in
53 | CYGWIN* )
54 | cygwin=true
55 | ;;
56 | Darwin* )
57 | darwin=true
58 | ;;
59 | MINGW* )
60 | msys=true
61 | ;;
62 | NONSTOP* )
63 | nonstop=true
64 | ;;
65 | esac
66 |
67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
68 |
69 | # Determine the Java command to use to start the JVM.
70 | if [ -n "$JAVA_HOME" ] ; then
71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
72 | # IBM's JDK on AIX uses strange locations for the executables
73 | JAVACMD="$JAVA_HOME/jre/sh/java"
74 | else
75 | JAVACMD="$JAVA_HOME/bin/java"
76 | fi
77 | if [ ! -x "$JAVACMD" ] ; then
78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
79 |
80 | Please set the JAVA_HOME variable in your environment to match the
81 | location of your Java installation."
82 | fi
83 | else
84 | JAVACMD="java"
85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
86 |
87 | Please set the JAVA_HOME variable in your environment to match the
88 | location of your Java installation."
89 | fi
90 |
91 | # Increase the maximum file descriptors if we can.
92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
93 | MAX_FD_LIMIT=`ulimit -H -n`
94 | if [ $? -eq 0 ] ; then
95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
96 | MAX_FD="$MAX_FD_LIMIT"
97 | fi
98 | ulimit -n $MAX_FD
99 | if [ $? -ne 0 ] ; then
100 | warn "Could not set maximum file descriptor limit: $MAX_FD"
101 | fi
102 | else
103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
104 | fi
105 | fi
106 |
107 | # For Darwin, add options to specify how the application appears in the dock
108 | if $darwin; then
109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
110 | fi
111 |
112 | # For Cygwin, switch paths to Windows format before running java
113 | if $cygwin ; then
114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"`
115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
116 | JAVACMD=`cygpath --unix "$JAVACMD"`
117 |
118 | # We build the pattern for arguments to be converted via cygpath
119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
120 | SEP=""
121 | for dir in $ROOTDIRSRAW ; do
122 | ROOTDIRS="$ROOTDIRS$SEP$dir"
123 | SEP="|"
124 | done
125 | OURCYGPATTERN="(^($ROOTDIRS))"
126 | # Add a user-defined pattern to the cygpath arguments
127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then
128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
129 | fi
130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh
131 | i=0
132 | for arg in "$@" ; do
133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
135 |
136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
138 | else
139 | eval `echo args$i`="\"$arg\""
140 | fi
141 | i=$((i+1))
142 | done
143 | case $i in
144 | (0) set -- ;;
145 | (1) set -- "$args0" ;;
146 | (2) set -- "$args0" "$args1" ;;
147 | (3) set -- "$args0" "$args1" "$args2" ;;
148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
154 | esac
155 | fi
156 |
157 | # Escape application args
158 | save () {
159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
160 | echo " "
161 | }
162 | APP_ARGS=$(save "$@")
163 |
164 | # Collect all arguments for the java command, following the shell quoting and substitution rules
165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
166 |
167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
169 | cd "$(dirname "$0")"
170 | fi
171 |
172 | exec "$JAVACMD" "$@"
173 |
--------------------------------------------------------------------------------
/gradlew.bat:
--------------------------------------------------------------------------------
1 | @if "%DEBUG%" == "" @echo off
2 | @rem ##########################################################################
3 | @rem
4 | @rem Gradle startup script for Windows
5 | @rem
6 | @rem ##########################################################################
7 |
8 | @rem Set local scope for the variables with windows NT shell
9 | if "%OS%"=="Windows_NT" setlocal
10 |
11 | set DIRNAME=%~dp0
12 | if "%DIRNAME%" == "" set DIRNAME=.
13 | set APP_BASE_NAME=%~n0
14 | set APP_HOME=%DIRNAME%
15 |
16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
17 | set DEFAULT_JVM_OPTS=
18 |
19 | @rem Find java.exe
20 | if defined JAVA_HOME goto findJavaFromJavaHome
21 |
22 | set JAVA_EXE=java.exe
23 | %JAVA_EXE% -version >NUL 2>&1
24 | if "%ERRORLEVEL%" == "0" goto init
25 |
26 | echo.
27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
28 | echo.
29 | echo Please set the JAVA_HOME variable in your environment to match the
30 | echo location of your Java installation.
31 |
32 | goto fail
33 |
34 | :findJavaFromJavaHome
35 | set JAVA_HOME=%JAVA_HOME:"=%
36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe
37 |
38 | if exist "%JAVA_EXE%" goto init
39 |
40 | echo.
41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
42 | echo.
43 | echo Please set the JAVA_HOME variable in your environment to match the
44 | echo location of your Java installation.
45 |
46 | goto fail
47 |
48 | :init
49 | @rem Get command-line arguments, handling Windows variants
50 |
51 | if not "%OS%" == "Windows_NT" goto win9xME_args
52 |
53 | :win9xME_args
54 | @rem Slurp the command line arguments.
55 | set CMD_LINE_ARGS=
56 | set _SKIP=2
57 |
58 | :win9xME_args_slurp
59 | if "x%~1" == "x" goto execute
60 |
61 | set CMD_LINE_ARGS=%*
62 |
63 | :execute
64 | @rem Setup the command line
65 |
66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
67 |
68 | @rem Execute Gradle
69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
70 |
71 | :end
72 | @rem End local scope for the variables with windows NT shell
73 | if "%ERRORLEVEL%"=="0" goto mainEnd
74 |
75 | :fail
76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
77 | rem the _cmd.exe /c_ return code!
78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
79 | exit /b 1
80 |
81 | :mainEnd
82 | if "%OS%"=="Windows_NT" endlocal
83 |
84 | :omega
85 |
--------------------------------------------------------------------------------
/settings.gradle:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2019 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | rootProject.name = 'mantis-api'
18 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/Bootstrap.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package io.mantisrx.api;
18 |
19 | import com.google.inject.Injector;
20 | import com.netflix.config.ConfigurationManager;
21 | import com.netflix.governator.InjectorBuilder;
22 | import com.netflix.zuul.netty.server.BaseServerStartup;
23 | import com.netflix.zuul.netty.server.Server;
24 |
25 | /**
26 | * Bootstrap
27 | *
28 | * Author: Arthur Gonigberg
29 | * Date: November 20, 2017
30 | */
31 | public class Bootstrap {
32 |
33 | public static void main(String[] args) {
34 | String propertiesFile = null;
35 | if (args.length >= 2 && "-p".equals(args[0])) {
36 | propertiesFile = args[1];
37 | if (propertiesFile.endsWith(".properties")) {
38 | propertiesFile = propertiesFile.substring(0, propertiesFile.length() - 11);
39 | }
40 | }
41 | new Bootstrap().start(propertiesFile);
42 | }
43 |
44 | public void start(String configName) {
45 | System.out.println("Mantis API: starting up.");
46 | long startTime = System.currentTimeMillis();
47 | int exitCode = 0;
48 |
49 | Server server = null;
50 |
51 | try {
52 | ConfigurationManager.loadCascadedPropertiesFromResources(configName);
53 | Injector injector = InjectorBuilder.fromModule(new MantisAPIModule()).createInjector();
54 | BaseServerStartup serverStartup = injector.getInstance(BaseServerStartup.class);
55 | server = serverStartup.server();
56 |
57 | long startupDuration = System.currentTimeMillis() - startTime;
58 | System.out.println("Mantis API: finished startup. Duration = " + startupDuration + " ms");
59 |
60 | server.start();
61 | server.awaitTermination();
62 | }
63 | catch (Throwable t) {
64 | t.printStackTrace();
65 | System.err.println("###############");
66 | System.err.println("Mantis API: initialization failed. Forcing shutdown now.");
67 | System.err.println("###############");
68 | exitCode = 1;
69 | }
70 | finally {
71 | // server shutdown
72 | if (server != null) server.stop();
73 |
74 | System.exit(exitCode);
75 | }
76 | }
77 | }
78 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/Constants.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api;
2 |
3 | import lombok.experimental.UtilityClass;
4 |
5 | @UtilityClass
6 | public class Constants {
7 | public static final String numMessagesCounterName = "numSinkMessages";
8 | public static final String numDroppedMessagesCounterName = "numDroppedSinkMessages";
9 | public static final String numBytesCounterName = "numSinkBytes";
10 | public static final String numDroppedBytesCounterName = "numDroppedSinkBytes";
11 | public static final String drainTriggeredCounterName = "drainTriggered";
12 | public static final String numIncomingMessagesCounterName = "numIncomingMessages";
13 |
14 | public static final String SSE_DATA_SUFFIX = "\r\n\r\n";
15 | public static final String SSE_DATA_PREFIX = "data: ";
16 |
17 | public static final long TunnelPingIntervalSecs = 12;
18 | public static final String TunnelPingMessage = "MantisApiTunnelPing";
19 | public static final String TunnelPingParamName = "MantisApiTunnelPingEnabled";
20 |
21 | public static final String OriginRegionTagName = "originRegion";
22 | public static final String ClientIdTagName = "clientId";
23 | public static final String TagsParamName = "MantisApiTag";
24 | public static final String TagNameValDelimiter = ":";
25 |
26 | public static final String metaErrorMsgHeader = "mantis.meta.error.message";
27 | public static final String metaOriginName = "mantis.meta.origin";
28 |
29 | public static final String numRemoteBytesCounterName = "numRemoteSinkBytes";
30 | public static final String numRemoteMessagesCounterName = "numRemoteMessages";
31 | public static final String numSseErrorsCounterName = "numSseErrors";
32 |
33 | public static final String DUMMY_TIMER_DATA = "DUMMY_TIMER_DATA";
34 |
35 | public static final String MANTISAPI_CACHED_HEADER = "x-nflx-mantisapi-cached";
36 | }
37 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/MantisAPIModule.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package io.mantisrx.api;
18 |
19 | import com.google.inject.Scopes;
20 | import com.google.inject.util.Modules;
21 | import com.netflix.appinfo.EurekaInstanceConfig;
22 | import com.netflix.appinfo.providers.MyDataCenterInstanceConfigProvider;
23 | import com.netflix.discovery.guice.EurekaModule;
24 | import com.netflix.zuul.*;
25 | import com.netflix.zuul.filters.FilterRegistry;
26 | import com.netflix.zuul.filters.MutableFilterRegistry;
27 | import com.netflix.zuul.groovy.GroovyCompiler;
28 | import com.netflix.zuul.groovy.GroovyFileFilter;
29 | import io.mantisrx.api.services.AppStreamDiscoveryService;
30 | import io.mantisrx.api.services.AppStreamStore;
31 | import io.mantisrx.api.services.ConfigurationBasedAppStreamStore;
32 | import io.mantisrx.server.core.Configurations;
33 | import io.mantisrx.server.core.CoreConfiguration;
34 | import io.mantisrx.server.master.client.HighAvailabilityServices;
35 | import io.mantisrx.server.master.client.HighAvailabilityServicesUtil;
36 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.DeserializationFeature;
37 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper;
38 | import com.google.inject.AbstractModule;
39 | import com.google.inject.Provides;
40 | import com.google.inject.Singleton;
41 | import com.google.inject.name.Named;
42 | import com.netflix.config.ConfigurationManager;
43 | import com.netflix.netty.common.accesslog.AccessLogPublisher;
44 | import com.netflix.netty.common.status.ServerStatusManager;
45 | import com.netflix.spectator.api.DefaultRegistry;
46 | import com.netflix.spectator.api.Registry;
47 | import com.netflix.spectator.api.patterns.ThreadPoolMonitor;
48 | import com.netflix.zuul.context.SessionContextDecorator;
49 | import com.netflix.zuul.context.ZuulSessionContextDecorator;
50 | import com.netflix.zuul.init.ZuulFiltersModule;
51 | import com.netflix.zuul.netty.server.BaseServerStartup;
52 | import com.netflix.zuul.netty.server.ClientRequestReceiver;
53 | import com.netflix.zuul.origins.BasicNettyOriginManager;
54 | import com.netflix.zuul.origins.OriginManager;
55 | import io.mantisrx.api.services.artifacts.ArtifactManager;
56 | import io.mantisrx.api.services.artifacts.InMemoryArtifactManager;
57 | import com.netflix.zuul.stats.BasicRequestMetricsPublisher;
58 | import com.netflix.zuul.stats.RequestMetricsPublisher;
59 | import io.mantisrx.api.tunnel.MantisCrossRegionalClient;
60 | import io.mantisrx.api.tunnel.NoOpCrossRegionalClient;
61 | import io.mantisrx.client.MantisClient;
62 | import io.mantisrx.server.worker.client.WorkerMetricsClient;
63 | import io.mantisrx.shaded.org.apache.curator.framework.listen.Listenable;
64 | import io.mantisrx.shaded.org.apache.curator.framework.listen.ListenerContainer;
65 | import org.apache.commons.configuration.AbstractConfiguration;
66 | import rx.Scheduler;
67 | import rx.schedulers.Schedulers;
68 |
69 | import java.io.FilenameFilter;
70 | import java.util.ArrayList;
71 | import java.util.List;
72 | import java.util.Properties;
73 | import java.util.concurrent.*;
74 |
75 | public class MantisAPIModule extends AbstractModule {
76 | @Override
77 | protected void configure() {
78 | bind(AbstractConfiguration.class).toInstance(ConfigurationManager.getConfigInstance());
79 |
80 | bind(BaseServerStartup.class).to(MantisServerStartup.class);
81 |
82 | // use provided basic netty origin manager
83 | bind(OriginManager.class).to(BasicNettyOriginManager.class);
84 |
85 | // zuul filter loading
86 | bind(DynamicCodeCompiler.class).to(GroovyCompiler.class);
87 | bind(FilenameFilter.class).to(GroovyFileFilter.class);
88 |
89 | install(Modules.override(new EurekaModule()).with(new AbstractModule() {
90 | @Override
91 | protected void configure() {
92 | bind(EurekaInstanceConfig.class).toProvider(MyDataCenterInstanceConfigProvider.class).in(Scopes.SINGLETON);
93 | }
94 | }));
95 |
96 | install(new ZuulFiltersModule());
97 | bind(FilterLoader.class).to(DynamicFilterLoader.class);
98 | bind(FilterRegistry.class).to(MutableFilterRegistry.class);
99 | bind(FilterFileManager.class).asEagerSingleton();
100 |
101 | // general server bindings
102 | bind(ServerStatusManager.class); // health/discovery status
103 | bind(SessionContextDecorator.class).to(ZuulSessionContextDecorator.class); // decorate new sessions when requests come in
104 | bind(Registry.class).to(DefaultRegistry.class); // atlas metrics registry
105 | bind(RequestCompleteHandler.class).to(BasicRequestCompleteHandler.class); // metrics post-request completion
106 | bind(RequestMetricsPublisher.class).to(BasicRequestMetricsPublisher.class); // timings publisher
107 |
108 | // access logger, including request ID generator
109 | bind(AccessLogPublisher.class).toInstance(new AccessLogPublisher("ACCESS",
110 | (channel, httpRequest) -> ClientRequestReceiver.getRequestFromChannel(channel).getContext().getUUID()));
111 |
112 | bind(ArtifactManager.class).to(InMemoryArtifactManager.class);
113 | bind(MantisCrossRegionalClient.class).to(NoOpCrossRegionalClient.class);
114 |
115 | bind(ObjectMapper.class).toInstance(new ObjectMapper()
116 | .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false));
117 | }
118 |
119 | @Provides
120 | @Singleton
121 | HighAvailabilityServices provideHighAvailabilityServices(AbstractConfiguration configuration) {
122 | Properties props = new Properties();
123 | configuration.getKeys("mantis").forEachRemaining(key -> {
124 | props.put(key, configuration.getString(key));
125 | });
126 |
127 | return HighAvailabilityServicesUtil.createHAServices(
128 | Configurations.frmProperties(props, CoreConfiguration.class));
129 | }
130 |
131 | @Provides @Singleton MantisClient provideMantisClient(AbstractConfiguration configuration) {
132 | Properties props = new Properties();
133 | configuration.getKeys("mantis").forEachRemaining(key -> {
134 | props.put(key, configuration.getString(key));
135 | });
136 |
137 | return new MantisClient(props);
138 | }
139 |
140 | @Provides
141 | @Singleton
142 | @Named("io-scheduler")
143 | Scheduler provideIoScheduler(Registry registry) {
144 | ThreadPoolExecutor executor = new ThreadPoolExecutor(16, 128, 60,
145 | TimeUnit.SECONDS, new LinkedBlockingQueue<>());
146 | ThreadPoolMonitor.attach(registry, executor, "io-thread-pool");
147 | return Schedulers.from(executor);
148 | }
149 |
150 | @Provides
151 | @Singleton
152 | ConfigurationBasedAppStreamStore.ConfigSource provideConfigSource(AbstractConfiguration configuration) {
153 | return new ConfigurationBasedAppStreamStore.ConfigSource() {
154 | @Override
155 | public Listenable getListenable() {
156 | return new ListenerContainer<>();
157 | }
158 |
159 | @Override
160 | public String get() {
161 | return String.join(",", configuration.getStringArray("mreAppJobClusterMap"));
162 | }
163 | };
164 | }
165 |
166 | @Provides
167 | @Singleton
168 | AppStreamDiscoveryService provideAppStreamDiscoveryService(MantisClient mantisClient,
169 | @Named("io-scheduler") Scheduler ioScheduler,
170 | ConfigurationBasedAppStreamStore.ConfigSource configSource) {
171 | AppStreamStore appStreamStore = new ConfigurationBasedAppStreamStore(configSource);
172 | return new AppStreamDiscoveryService(mantisClient, ioScheduler, appStreamStore);
173 | }
174 |
175 | @Provides @Singleton
176 | WorkerMetricsClient provideWorkerMetricsClient(AbstractConfiguration configuration) {
177 | Properties props = new Properties();
178 | configuration.getKeys("mantis").forEachRemaining(key -> {
179 | props.put(key, configuration.getString(key));
180 | });
181 | return new WorkerMetricsClient(props);
182 | }
183 |
184 | @Provides
185 | @Singleton
186 | @Named("push-prefixes")
187 | List providePushPrefixes() {
188 | List pushPrefixes = new ArrayList<>(20);
189 | pushPrefixes.add("/jobconnectbyid");
190 | pushPrefixes.add("/api/v1/jobconnectbyid");
191 | pushPrefixes.add("/jobconnectbyname");
192 | pushPrefixes.add("/api/v1/jobconnectbyname");
193 | pushPrefixes.add("/jobsubmitandconnect");
194 | pushPrefixes.add("/api/v1/jobsubmitandconnect");
195 | pushPrefixes.add("/jobClusters/discoveryInfoStream");
196 | pushPrefixes.add("/jobstatus");
197 | pushPrefixes.add("/api/v1/jobstatus");
198 | pushPrefixes.add("/api/v1/jobs/schedulingInfo/");
199 | pushPrefixes.add("/api/v1/metrics");
200 |
201 | return pushPrefixes;
202 | }
203 |
204 | }
205 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/MantisConfigurationBasedServerList.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2023 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package io.mantisrx.api;
17 |
18 | import com.google.common.base.Strings;
19 | import com.google.common.collect.Lists;
20 | import com.netflix.appinfo.InstanceInfo;
21 | import com.netflix.loadbalancer.ConfigurationBasedServerList;
22 | import com.netflix.loadbalancer.Server;
23 | import com.netflix.niws.loadbalancer.DiscoveryEnabledServer;
24 |
25 | import java.util.List;
26 |
27 | public class MantisConfigurationBasedServerList extends ConfigurationBasedServerList {
28 | @Override
29 | protected List derive(String value) {
30 | List list = Lists.newArrayList();
31 | if (!Strings.isNullOrEmpty(value)) {
32 | for (String s : value.split(",")) {
33 | Server server = new Server(s.trim());
34 | InstanceInfo instanceInfo =
35 | InstanceInfo.Builder.newBuilder()
36 | .setAppName("mantismasterv2")
37 | .setIPAddr(server.getHost())
38 | .setPort(server.getPort())
39 | .build();
40 | list.add(new DiscoveryEnabledServer(instanceInfo, false, true));
41 | }
42 | }
43 | return list;
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/MantisServerStartup.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package io.mantisrx.api;
18 |
19 | import com.google.inject.name.Named;
20 | import com.netflix.appinfo.ApplicationInfoManager;
21 | import com.netflix.config.DynamicIntProperty;
22 | import com.netflix.discovery.EurekaClient;
23 | import com.netflix.netty.common.accesslog.AccessLogPublisher;
24 | import com.netflix.netty.common.channel.config.ChannelConfig;
25 | import com.netflix.netty.common.channel.config.CommonChannelConfigKeys;
26 | import com.netflix.netty.common.metrics.EventLoopGroupMetrics;
27 | import com.netflix.netty.common.proxyprotocol.StripUntrustedProxyHeadersHandler;
28 | import com.netflix.netty.common.status.ServerStatusManager;
29 | import com.netflix.spectator.api.Registry;
30 | import com.netflix.zuul.FilterLoader;
31 | import com.netflix.zuul.FilterUsageNotifier;
32 | import com.netflix.zuul.RequestCompleteHandler;
33 | import com.netflix.zuul.context.SessionContextDecorator;
34 | import com.netflix.zuul.netty.server.BaseServerStartup;
35 | import com.netflix.zuul.netty.server.DirectMemoryMonitor;
36 | import com.netflix.zuul.netty.server.NamedSocketAddress;
37 |
38 | import io.mantisrx.api.initializers.MantisApiServerChannelInitializer;
39 | import io.mantisrx.api.push.ConnectionBroker;
40 | import io.mantisrx.api.tunnel.MantisCrossRegionalClient;
41 | import io.mantisrx.server.master.client.HighAvailabilityServices;
42 | import io.netty.channel.ChannelInitializer;
43 | import io.netty.channel.group.ChannelGroup;
44 | import org.apache.commons.configuration.AbstractConfiguration;
45 | import rx.Scheduler;
46 |
47 | import javax.inject.Inject;
48 | import javax.inject.Singleton;
49 | import java.net.InetSocketAddress;
50 | import java.util.*;
51 |
52 | @Singleton
53 | public class MantisServerStartup extends BaseServerStartup {
54 |
55 | private final HighAvailabilityServices highAvailabilityServices;
56 | private final MantisCrossRegionalClient mantisCrossRegionalClient;
57 | private final ConnectionBroker connectionBroker;
58 | private final Scheduler scheduler;
59 | private final List pushPrefixes;
60 |
61 | @Inject
62 | public MantisServerStartup(ServerStatusManager serverStatusManager, FilterLoader filterLoader,
63 | SessionContextDecorator sessionCtxDecorator, FilterUsageNotifier usageNotifier,
64 | RequestCompleteHandler reqCompleteHandler, Registry registry,
65 | DirectMemoryMonitor directMemoryMonitor, EventLoopGroupMetrics eventLoopGroupMetrics,
66 | EurekaClient discoveryClient, ApplicationInfoManager applicationInfoManager,
67 | AccessLogPublisher accessLogPublisher,
68 | AbstractConfiguration configurationManager,
69 | HighAvailabilityServices highAvailabilityServices,
70 | MantisCrossRegionalClient mantisCrossRegionalClient,
71 | ConnectionBroker connectionBroker,
72 | @Named("io-scheduler") Scheduler scheduler,
73 | @Named("push-prefixes") List pushPrefixes
74 | ) {
75 | super(serverStatusManager, filterLoader, sessionCtxDecorator, usageNotifier, reqCompleteHandler, registry,
76 | directMemoryMonitor, eventLoopGroupMetrics, discoveryClient, applicationInfoManager,
77 | accessLogPublisher);
78 | this.highAvailabilityServices = highAvailabilityServices;
79 | this.mantisCrossRegionalClient = mantisCrossRegionalClient;
80 | this.connectionBroker = connectionBroker;
81 | this.scheduler = scheduler;
82 | this.pushPrefixes = pushPrefixes;
83 |
84 | // Mantis Master Listener
85 | highAvailabilityServices
86 | .getMasterMonitor()
87 | .getMasterObservable()
88 | .filter(x -> x != null)
89 | .forEach(masterDescription -> {
90 | LOG.info("Received new Mantis Master: " + masterDescription);
91 | configurationManager.setProperty("api.ribbon.listOfServers",
92 | masterDescription.getHostIP() + ":" + masterDescription.getApiPort());
93 | });
94 | }
95 |
96 | @Override
97 | protected Map> chooseAddrsAndChannels(ChannelGroup clientChannels) {
98 | Map> addrsToChannels = new HashMap<>();
99 |
100 | String mainPortName = "main";
101 | int port = new DynamicIntProperty("zuul.server.port.main", 7001).get();
102 | NamedSocketAddress sockAddr = new NamedSocketAddress(mainPortName, new InetSocketAddress(port));
103 |
104 | ChannelConfig channelConfig = defaultChannelConfig(mainPortName);
105 | ChannelConfig channelDependencies = defaultChannelDependencies(mainPortName);
106 |
107 | /* These settings may need to be tweaked depending if you're running behind an ELB HTTP listener, TCP listener,
108 | * or directly on the internet.
109 | */
110 | channelConfig.set(CommonChannelConfigKeys.allowProxyHeadersWhen,
111 | StripUntrustedProxyHeadersHandler.AllowWhen.ALWAYS);
112 | channelConfig.set(CommonChannelConfigKeys.preferProxyProtocolForClientIp, false);
113 | channelConfig.set(CommonChannelConfigKeys.isSSlFromIntermediary, false);
114 | channelConfig.set(CommonChannelConfigKeys.withProxyProtocol, false);
115 |
116 | addrsToChannels.put(
117 | sockAddr,
118 | new MantisApiServerChannelInitializer(
119 | String.valueOf(port), channelConfig, channelDependencies, clientChannels, pushPrefixes,
120 | highAvailabilityServices, mantisCrossRegionalClient, connectionBroker,
121 | scheduler, false));
122 | logAddrConfigured(sockAddr);
123 |
124 | return Collections.unmodifiableMap(addrsToChannels);
125 | }
126 | }
127 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/Util.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package io.mantisrx.api;
17 |
18 | import com.google.common.base.Strings;
19 | import io.netty.handler.codec.http.QueryStringDecoder;
20 | import lombok.experimental.UtilityClass;
21 | import lombok.extern.slf4j.Slf4j;
22 | import org.apache.commons.lang3.tuple.ImmutablePair;
23 | import org.slf4j.Logger;
24 | import rx.Observable;
25 | import rx.functions.Func1;
26 |
27 | import java.util.*;
28 | import java.util.concurrent.TimeUnit;
29 |
30 | import static io.mantisrx.api.Constants.*;
31 |
32 | @UtilityClass
33 | @Slf4j
34 | public class Util {
35 | private static final int defaultNumRetries = 2;
36 |
37 | public static boolean startsWithAnyOf(final String target, List prefixes) {
38 | for (String prefix : prefixes) {
39 | if (target.startsWith(prefix)) {
40 | return true;
41 | }
42 | }
43 | return false;
44 | }
45 |
46 | //
47 | // Regions
48 | //
49 |
50 | public static String getLocalRegion() {
51 | return System.getenv("EC2_REGION");
52 | }
53 |
54 | //
55 | // Query Params
56 | //
57 |
58 | public static String[] getTaglist(String uri, String id) {
59 | return getTaglist(uri, id, null);
60 | }
61 |
62 | public static String[] getTaglist(String uri, String id, String region) {
63 | QueryStringDecoder queryStringDecoder = new QueryStringDecoder(uri);
64 | Map> queryParameters = queryStringDecoder.parameters();
65 | boolean isClientIdSet = false;
66 |
67 | final List tags = new LinkedList<>();
68 | if (queryParameters != null) {
69 | List tagVals = queryParameters.get(TagsParamName);
70 | if (tagVals != null) {
71 | for (String s : tagVals) {
72 | StringTokenizer tokenizer = new StringTokenizer(s, TagNameValDelimiter);
73 | if (tokenizer.countTokens() == 2) {
74 | String s1 = tokenizer.nextToken();
75 | String s2 = tokenizer.nextToken();
76 | if (s1 != null && !s1.isEmpty() && s2 != null && !s2.isEmpty()) {
77 | tags.add(s1);
78 | tags.add(s2);
79 | if (ClientIdTagName.equals(s1)) {
80 | isClientIdSet = true;
81 | }
82 | }
83 | }
84 | }
85 | }
86 | tagVals = queryParameters.get(ClientIdTagName);
87 | if (!isClientIdSet && tagVals != null && !tagVals.isEmpty()) {
88 | tags.add(ClientIdTagName);
89 | tags.add(tagVals.get(0));
90 | }
91 | }
92 |
93 | tags.add("SessionId");
94 | tags.add(id);
95 |
96 | tags.add("urlPath");
97 | tags.add(queryStringDecoder.path());
98 |
99 | if (!Strings.isNullOrEmpty(region)) {
100 | tags.add("region");
101 | tags.add(region);
102 | }
103 |
104 | return tags.toArray(new String[]{});
105 | }
106 |
107 | //
108 | // Retries
109 | //
110 |
111 | public static Func1, Observable>> getRetryFunc(final Logger logger, String name) {
112 | return getRetryFunc(logger, name, defaultNumRetries);
113 | }
114 |
115 | public static Func1, Observable>> getRetryFunc(final Logger logger, String name, final int retries) {
116 | final int limit = retries == Integer.MAX_VALUE ? retries : retries + 1;
117 | return attempts -> attempts
118 | .zipWith(Observable.range(1, limit), (t1, integer) -> {
119 | logger.warn("Caught exception connecting for {}.", name, t1);
120 | return new ImmutablePair(t1, integer);
121 | })
122 | .flatMap(pair -> {
123 | Throwable t = pair.left;
124 | int retryIter = pair.right;
125 | long delay = Math.round(Math.pow(2, retryIter));
126 |
127 | if (retryIter > retries) {
128 | logger.error("Exceeded maximum retries ({}) for {} with exception: {}", retries, name, t.getMessage(), t);
129 | return Observable.error(new Exception("Timeout after " + retries + " retries"));
130 | }
131 | logger.info("Retrying connection to {} after sleeping for {} seconds.", name, delay, t);
132 | return Observable.timer(delay, TimeUnit.SECONDS);
133 | });
134 | }
135 | }
136 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/AppStreamDiscovery.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package io.mantisrx.api.filters;
17 |
18 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper;
19 | import com.google.inject.Inject;
20 | import com.netflix.zuul.filters.http.HttpSyncEndpoint;
21 | import com.netflix.zuul.message.http.HttpHeaderNames;
22 | import com.netflix.zuul.message.http.HttpRequestMessage;
23 | import com.netflix.zuul.message.http.HttpResponseMessage;
24 | import com.netflix.zuul.message.http.HttpResponseMessageImpl;
25 | import io.mantisrx.api.proto.AppDiscoveryMap;
26 | import io.mantisrx.api.services.AppStreamDiscoveryService;
27 | import com.netflix.zuul.stats.status.StatusCategoryUtils;
28 | import com.netflix.zuul.stats.status.ZuulStatusCategory;
29 | import io.netty.handler.codec.http.HttpHeaderValues;
30 | import io.vavr.control.Either;
31 | import io.vavr.control.Try;
32 | import lombok.extern.slf4j.Slf4j;
33 |
34 | import java.util.List;
35 | import java.util.function.Function;
36 |
37 | @Slf4j
38 | public class AppStreamDiscovery extends HttpSyncEndpoint {
39 |
40 | private final AppStreamDiscoveryService appStreamDiscoveryService;
41 | private final ObjectMapper objectMapper;
42 | private static final String APPNAME_QUERY_PARAM = "app";
43 |
44 | @Inject
45 | public AppStreamDiscovery(AppStreamDiscoveryService appStreamDiscoveryService,
46 | ObjectMapper objectMapper) {
47 | this.appStreamDiscoveryService = appStreamDiscoveryService;
48 | this.objectMapper = objectMapper;
49 | }
50 |
51 | @Override
52 | public HttpResponseMessage apply(HttpRequestMessage request) {
53 |
54 | List apps = request.getQueryParams().get(APPNAME_QUERY_PARAM);
55 | Either result = appStreamDiscoveryService.getAppDiscoveryMap(apps);
56 |
57 | return result.bimap(errorMessage -> {
58 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 500);
59 | resp.setBodyAsText(errorMessage);
60 | StatusCategoryUtils.setStatusCategory(request.getContext(), ZuulStatusCategory.FAILURE_LOCAL);
61 | return resp;
62 | }, appDiscoveryMap -> {
63 |
64 | Try serialized = Try.of(() -> objectMapper.writeValueAsString(appDiscoveryMap));
65 |
66 | if (serialized.isSuccess()) {
67 | StatusCategoryUtils.setStatusCategory(request.getContext(), ZuulStatusCategory.SUCCESS);
68 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 200);
69 | resp.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_JSON.toString());
70 | resp.setBodyAsText(serialized.get());
71 | return resp;
72 | } else {
73 | StatusCategoryUtils.setStatusCategory(request.getContext(), ZuulStatusCategory.FAILURE_LOCAL);
74 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 500);
75 | resp.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.TEXT_PLAIN.toString());
76 | resp.setBodyAsText(serialized.getOrElseGet(Throwable::getMessage));
77 | return resp;
78 | }
79 | }).getOrElseGet(Function.identity());
80 | }
81 | }
82 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/Artifacts.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.filters;
2 |
3 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper;
4 | import com.google.common.base.Strings;
5 | import com.netflix.zuul.filters.http.HttpSyncEndpoint;
6 | import com.netflix.zuul.message.http.HttpHeaderNames;
7 | import com.netflix.zuul.message.http.HttpRequestMessage;
8 | import com.netflix.zuul.message.http.HttpResponseMessage;
9 | import com.netflix.zuul.message.http.HttpResponseMessageImpl;
10 | import io.mantisrx.api.proto.Artifact;
11 | import io.mantisrx.api.services.artifacts.ArtifactManager;
12 | import io.netty.handler.codec.http.HttpHeaderValues;
13 | import io.netty.handler.codec.http.HttpResponseStatus;
14 | import io.vavr.control.Try;
15 | import lombok.extern.slf4j.Slf4j;
16 |
17 | import javax.inject.Inject;
18 | import java.util.List;
19 | import java.util.Optional;
20 |
21 | @Slf4j
22 | public class Artifacts extends HttpSyncEndpoint {
23 |
24 | private final ArtifactManager artifactManager;
25 | private final ObjectMapper objectMapper;
26 | public static final String PATH_SPEC = "/api/v1/artifacts";
27 |
28 | @Override
29 | public boolean needsBodyBuffered(HttpRequestMessage input) {
30 | return input.getMethod().toLowerCase().equals("post");
31 | }
32 |
33 | @Inject
34 | public Artifacts(ArtifactManager artifactManager, ObjectMapper objectMapper) {
35 | this.artifactManager = artifactManager;
36 | this.objectMapper = objectMapper;
37 | artifactManager.putArtifact(new Artifact("mantis.json", 0, new byte[0]));
38 | artifactManager.putArtifact(new Artifact("mantis.zip", 0, new byte[0]));
39 | }
40 |
41 | @Override
42 | public HttpResponseMessage apply(HttpRequestMessage request) {
43 |
44 | if (request.getMethod().toLowerCase().equals("get")) {
45 |
46 | String fileName = request.getPath().replaceFirst("^" + PATH_SPEC + "/?", "");
47 | if (Strings.isNullOrEmpty(fileName)) {
48 | List files = artifactManager
49 | .getArtifacts();
50 | Try serialized = Try.of(() -> objectMapper.writeValueAsString(files));
51 |
52 | return serialized.map(body -> {
53 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request, 200);
54 | response.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_JSON.toString());
55 | response.setBodyAsText(body);
56 | return response;
57 | }).getOrElseGet(t -> {
58 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request, 500);
59 | response.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.TEXT_PLAIN.toString());
60 | response.setBodyAsText(t.getMessage());
61 | return response;
62 | });
63 | } else {
64 | Optional artifact = artifactManager.getArtifact(fileName);
65 |
66 | return artifact.map(art -> {
67 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request,
68 | HttpResponseStatus.OK.code());
69 | response.setBody(art.getContent());
70 | response.getHeaders().set(HttpHeaderNames.CONTENT_TYPE,
71 | fileName.endsWith("json")
72 | ? HttpHeaderValues.APPLICATION_JSON.toString()
73 | : HttpHeaderValues.APPLICATION_OCTET_STREAM.toString());
74 | response.getHeaders().set("Content-Disposition",
75 | String.format("attachment; filename=\"%s\"", fileName));
76 | return response;
77 | }).orElseGet(() -> {
78 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request,
79 | HttpResponseStatus.NOT_FOUND.code());
80 | response.setBody(new byte[]{});
81 | return response;
82 | });
83 |
84 | }
85 | } else if (request.getMethod().toLowerCase().equals("post")) {
86 |
87 | byte[] body = request.getBody();
88 | artifactManager.putArtifact(new Artifact("testing.json", body.length, body));
89 |
90 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request,
91 | HttpResponseStatus.OK.code());
92 | return response;
93 |
94 | }
95 |
96 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request, HttpResponseStatus.METHOD_NOT_ALLOWED.code());
97 | response.setBodyAsText(HttpResponseStatus.METHOD_NOT_ALLOWED.reasonPhrase());
98 | response.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.TEXT_PLAIN.toString());
99 | return response;
100 | }
101 | }
102 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/Favicon.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package io.mantisrx.api.filters;
17 |
18 | import com.netflix.zuul.filters.http.HttpSyncEndpoint;
19 | import com.netflix.zuul.message.http.HttpRequestMessage;
20 | import com.netflix.zuul.message.http.HttpResponseMessage;
21 | import com.netflix.zuul.message.http.HttpResponseMessageImpl;
22 | import com.netflix.zuul.stats.status.StatusCategoryUtils;
23 | import com.netflix.zuul.stats.status.ZuulStatusCategory;
24 |
25 | /**
26 | * Returns an empty 200 response to prevent 404s on Favicon.
27 | */
28 | public class Favicon extends HttpSyncEndpoint {
29 | @Override
30 | public HttpResponseMessage apply(HttpRequestMessage request) {
31 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 200);
32 | resp.setBody(new byte[0]);
33 | StatusCategoryUtils.setStatusCategory(request.getContext(), ZuulStatusCategory.SUCCESS);
34 | return resp;
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/Healthcheck.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package io.mantisrx.api.filters;
17 |
18 | import com.netflix.zuul.filters.http.HttpSyncEndpoint;
19 | import com.netflix.zuul.message.http.HttpRequestMessage;
20 | import com.netflix.zuul.message.http.HttpResponseMessage;
21 | import com.netflix.zuul.message.http.HttpResponseMessageImpl;
22 | import com.netflix.zuul.stats.status.StatusCategoryUtils;
23 | import com.netflix.zuul.stats.status.ZuulStatusCategory;
24 |
25 | public class Healthcheck extends HttpSyncEndpoint {
26 |
27 | @Override
28 | public HttpResponseMessage apply(HttpRequestMessage request) {
29 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 200);
30 | resp.setBodyAsText("mantisapi healthy");
31 | StatusCategoryUtils.setStatusCategory(request.getContext(), ZuulStatusCategory.SUCCESS);
32 | return resp;
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/JobDiscoveryCacheLoader.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package io.mantisrx.api.filters;
17 |
18 | import com.netflix.config.DynamicBooleanProperty;
19 | import com.netflix.zuul.filters.http.HttpOutboundSyncFilter;
20 | import com.netflix.zuul.message.http.HttpResponseMessage;
21 | import io.mantisrx.api.Constants;
22 | import io.mantisrx.api.services.JobDiscoveryService;
23 | import lombok.extern.slf4j.Slf4j;
24 |
25 | @Slf4j
26 | public class JobDiscoveryCacheLoader extends HttpOutboundSyncFilter {
27 |
28 | private static DynamicBooleanProperty cacheEnabled = new DynamicBooleanProperty("mantisapi.cache.enabled", false);
29 |
30 | @Override
31 | public boolean needsBodyBuffered(HttpResponseMessage message) {
32 | return true;
33 | }
34 |
35 | @Override
36 | public int filterOrder() {
37 | return 999; // Don't really care.
38 | }
39 |
40 | @Override
41 | public boolean shouldFilter(HttpResponseMessage response) {
42 | return response.getOutboundRequest().getPath().matches("^/api/v1/jobClusters/.*/latestJobDiscoveryInfo$")
43 | && response.getHeaders().getAll(Constants.MANTISAPI_CACHED_HEADER).isEmpty()
44 | && cacheEnabled.get();
45 | }
46 |
47 | @Override
48 | public HttpResponseMessage apply(HttpResponseMessage response) {
49 | String jobCluster = response.getOutboundRequest().getPath()
50 | .replaceFirst("^/api/v1/jobClusters/", "")
51 | .replaceFirst("/latestJobDiscoveryInfo$", "");
52 |
53 | String responseBody = response.getBodyAsText();
54 |
55 | if (null != responseBody) {
56 | log.info("Caching latest job discovery info for {}.", jobCluster);
57 | JobDiscoveryService.jobDiscoveryInfoCache.put(jobCluster, response.getBodyAsText());
58 | }
59 | return response;
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/JobDiscoveryInfoCacheHitChecker.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package io.mantisrx.api.filters;
17 |
18 | import com.google.common.base.Strings;
19 | import com.netflix.config.DynamicBooleanProperty;
20 | import com.netflix.zuul.filters.http.HttpInboundSyncFilter;
21 | import com.netflix.zuul.message.http.HttpHeaderNames;
22 | import com.netflix.zuul.message.http.HttpRequestMessage;
23 | import com.netflix.zuul.message.http.HttpResponseMessage;
24 | import com.netflix.zuul.message.http.HttpResponseMessageImpl;
25 | import io.mantisrx.api.Constants;
26 | import io.mantisrx.api.services.JobDiscoveryService;
27 | import io.netty.handler.codec.http.HttpHeaderValues;
28 | import lombok.extern.slf4j.Slf4j;
29 |
30 | @Slf4j
31 | public class JobDiscoveryInfoCacheHitChecker extends HttpInboundSyncFilter {
32 |
33 | public static final String PATH_SPEC = "/jobClusters/discoveryInfo";
34 | private static DynamicBooleanProperty cacheEnabled = new DynamicBooleanProperty("mantisapi.cache.enabled", false);
35 |
36 | @Override
37 | public int filterOrder() {
38 | return -1;
39 | }
40 |
41 | @Override
42 | public boolean shouldFilter(HttpRequestMessage httpRequestMessage) {
43 | String jobCluster = httpRequestMessage.getPath().replaceFirst(PATH_SPEC + "/", "");
44 | return httpRequestMessage.getPath().startsWith(PATH_SPEC)
45 | && JobDiscoveryService.jobDiscoveryInfoCache.getIfPresent(jobCluster) != null;
46 | }
47 |
48 | @Override
49 | public HttpRequestMessage apply(HttpRequestMessage request) {
50 | String jobCluster = request.getPath().replaceFirst(PATH_SPEC + "/", "");
51 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 200);
52 | String bodyText = JobDiscoveryService.jobDiscoveryInfoCache.getIfPresent(jobCluster) ;
53 | if (cacheEnabled.get() && !Strings.isNullOrEmpty(bodyText)) {
54 | log.info("Serving cached job discovery info for {}.", jobCluster);
55 | resp.setBodyAsText(bodyText);
56 | resp.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_JSON.toString());
57 | resp.getHeaders().set(Constants.MANTISAPI_CACHED_HEADER, "true");
58 | request.getContext().setStaticResponse(resp);
59 | }
60 | return request;
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/MQLParser.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package io.mantisrx.api.filters;
18 |
19 | import io.mantisrx.shaded.com.fasterxml.jackson.core.JsonProcessingException;
20 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper;
21 | import com.netflix.zuul.filters.http.HttpSyncEndpoint;
22 | import com.netflix.zuul.message.http.HttpRequestMessage;
23 | import com.netflix.zuul.message.http.HttpResponseMessage;
24 | import com.netflix.zuul.message.http.HttpResponseMessageImpl;
25 | import io.mantisrx.mql.shaded.clojure.java.api.Clojure;
26 | import io.mantisrx.mql.shaded.clojure.lang.IFn;
27 | import lombok.Value;
28 | import lombok.extern.slf4j.Slf4j;
29 |
30 | import java.nio.charset.Charset;
31 |
32 |
33 | @Slf4j
34 | public class MQLParser extends HttpSyncEndpoint {
35 |
36 | private static IFn require = Clojure.var("io.mantisrx.mql.shaded.clojure.core", "require");
37 | static {
38 | require.invoke(Clojure.read("io.mantisrx.mql.core"));
39 | require.invoke(Clojure.read("io.mantisrx.mql.jvm.interfaces.server"));
40 | require.invoke(Clojure.read("io.mantisrx.mql.jvm.interfaces.core"));
41 | }
42 | private static IFn parses = Clojure.var("io.mantisrx.mql.jvm.interfaces.core", "parses?");
43 | private static IFn getParseError = Clojure.var("io.mantisrx.mql.jvm.interfaces.core", "get-parse-error");
44 |
45 | private static final ObjectMapper objectMapper = new ObjectMapper();
46 |
47 | public @Value class MQLParseResult {
48 | private boolean success;
49 | private String criterion;
50 | private String message;
51 | }
52 |
53 | @Override
54 | public HttpResponseMessage apply(HttpRequestMessage input) {
55 | String query = input.getQueryParams().getFirst("criterion");
56 |
57 | boolean parses = parses(query);
58 | String parseError = getParseError(query);
59 | MQLParseResult result = new MQLParseResult(parses, query, parses ? "" : parseError);
60 |
61 | try {
62 | HttpResponseMessage response = new HttpResponseMessageImpl(input.getContext(), input, 200);
63 | response.setBody(objectMapper.writeValueAsBytes(result));
64 | return response;
65 |
66 | } catch (JsonProcessingException ex) {
67 | HttpResponseMessage response = new HttpResponseMessageImpl(input.getContext(), input, 500);
68 | response.setBody(getErrorResponse(ex.getMessage()).getBytes(Charset.defaultCharset()));
69 | return response;
70 | }
71 | }
72 |
73 | /**
74 | * A predicate which indicates whether or not the MQL parser considers query to be a valid query.
75 | * @param query A String representing the MQL query.
76 | * @return A boolean indicating whether or not the query successfully parses.
77 | */
78 | public static Boolean parses(String query) {
79 | return (Boolean) parses.invoke(query);
80 | }
81 |
82 | /**
83 | * A convenience function allowing a caller to determine what went wrong if a call to #parses(String query) returns
84 | * false.
85 | * @param query A String representing the MQL query.
86 | * @return A String representing the parse error for an MQL query, null if no parse error occurred.
87 | */
88 | public static String getParseError(String query) {
89 | return (String) getParseError.invoke(query);
90 | }
91 |
92 | private String getErrorResponse(String exceptionMessage) {
93 | StringBuilder sb = new StringBuilder(50);
94 | sb.append("{\"success\": false, \"messages\": \"");
95 | sb.append(exceptionMessage);
96 | sb.append("\"}");
97 | return sb.toString();
98 | }
99 | }
100 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/MREAppStreamToJobClusterMapping.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package io.mantisrx.api.filters;
17 |
18 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper;
19 | import io.mantisrx.discovery.proto.AppJobClustersMap;
20 | import com.netflix.zuul.filters.http.HttpSyncEndpoint;
21 | import com.netflix.zuul.message.http.HttpHeaderNames;
22 | import com.netflix.zuul.message.http.HttpRequestMessage;
23 | import com.netflix.zuul.message.http.HttpResponseMessage;
24 | import com.netflix.zuul.message.http.HttpResponseMessageImpl;
25 | import io.mantisrx.api.services.AppStreamDiscoveryService;
26 | import io.mantisrx.shaded.com.google.common.base.Preconditions;
27 | import io.netty.handler.codec.http.HttpHeaderValues;
28 | import io.vavr.control.Try;
29 |
30 | import java.util.List;
31 | import javax.inject.Inject;
32 |
33 | public class MREAppStreamToJobClusterMapping extends HttpSyncEndpoint {
34 |
35 | private final AppStreamDiscoveryService appStreamDiscoveryService;
36 | private final ObjectMapper objectMapper;
37 |
38 | private static final String APPNAME_QUERY_PARAM = "app";
39 | public static final String PATH_SPEC = "/api/v1/mantis/publish/streamJobClusterMap";
40 |
41 | @Inject
42 | public MREAppStreamToJobClusterMapping(AppStreamDiscoveryService appStreamDiscoveryService,
43 | ObjectMapper objectMapper) {
44 | Preconditions.checkArgument(appStreamDiscoveryService != null, "appStreamDiscoveryService cannot be null");
45 | this.appStreamDiscoveryService = appStreamDiscoveryService;
46 | Preconditions.checkArgument(objectMapper != null, "objectMapper cannot be null");
47 | this.objectMapper = objectMapper;
48 | }
49 |
50 | @Override
51 | public HttpResponseMessage apply(HttpRequestMessage request) {
52 | List apps = request.getQueryParams().get(APPNAME_QUERY_PARAM);
53 | Try payloadTry = Try.ofCallable(() -> appStreamDiscoveryService.getAppJobClustersMap(apps));
54 |
55 | Try serialized = payloadTry.flatMap(payload -> Try.of(() -> objectMapper.writeValueAsString(payload)));
56 |
57 | return serialized.map(body -> {
58 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 200);
59 | resp.setBodyAsText(body);
60 | resp.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_JSON.toString());
61 | return resp;
62 | }).getOrElseGet(t -> {
63 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, 200);
64 | resp.setBodyAsText(t.getMessage());
65 | resp.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.TEXT_PLAIN.toString());
66 | return resp;
67 | });
68 | }
69 | }
70 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/MasterCacheHitChecker.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.filters;
2 |
3 | import com.google.inject.Inject;
4 | import com.google.inject.name.Named;
5 | import com.netflix.config.DynamicBooleanProperty;
6 | import com.netflix.spectator.api.Counter;
7 | import com.netflix.zuul.filters.http.HttpInboundSyncFilter;
8 | import com.netflix.zuul.message.http.HttpHeaderNames;
9 | import com.netflix.zuul.message.http.HttpRequestMessage;
10 | import com.netflix.zuul.message.http.HttpResponseMessage;
11 | import com.netflix.zuul.message.http.HttpResponseMessageImpl;
12 | import com.netflix.zuul.netty.SpectatorUtils;
13 | import io.mantisrx.api.Constants;
14 | import io.mantisrx.api.Util;
15 | import io.netty.handler.codec.http.HttpHeaderValues;
16 | import lombok.extern.slf4j.Slf4j;
17 |
18 | import java.util.List;
19 | import java.util.concurrent.ConcurrentHashMap;
20 |
21 | @Slf4j
22 | public class MasterCacheHitChecker extends HttpInboundSyncFilter {
23 |
24 | private static DynamicBooleanProperty cacheEnabled = new DynamicBooleanProperty("mantisapi.cache.enabled", false);
25 | private static final ConcurrentHashMap cacheHitCounters = new ConcurrentHashMap<>(500);
26 | private static final ConcurrentHashMap cacheMissCounters = new ConcurrentHashMap<>(500);
27 | private static final String CACHE_HIT_COUNTER_NAME = "mantis.api.cache.count";
28 | private final List pushPrefixes;
29 |
30 | @Inject
31 | public MasterCacheHitChecker(@Named("push-prefixes") List pushPrefixes) {
32 | super();
33 | this.pushPrefixes = pushPrefixes;
34 | }
35 |
36 | @Override
37 | public HttpRequestMessage apply(HttpRequestMessage request) {
38 | if(cacheEnabled.get()) {
39 | String key = request.getPathAndQuery();
40 | String bodyText = MasterCacheLoader.masterCache.getIfPresent(key);
41 |
42 | if (bodyText != null) { // Cache Hit
43 | HttpResponseMessage response = new HttpResponseMessageImpl(request.getContext(), request, 200);
44 | response.setBodyAsText(bodyText);
45 | response.getHeaders().set(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_JSON.toString());
46 | response.getHeaders().set(Constants.MANTISAPI_CACHED_HEADER, "true");
47 | request.getContext().setStaticResponse(response);
48 |
49 | cacheHitCounters.computeIfAbsent(key,
50 | k -> SpectatorUtils.newCounter(CACHE_HIT_COUNTER_NAME, "api", "endpoint", k, "class", "hit"))
51 | .increment();
52 | } else { // Cache Miss
53 | cacheMissCounters.computeIfAbsent(key,
54 | k -> SpectatorUtils.newCounter(CACHE_HIT_COUNTER_NAME, "api", "endpoint", k, "class", "miss"))
55 | .increment();
56 | }
57 | }
58 |
59 | return request;
60 | }
61 |
62 | @Override
63 | public int filterOrder() {
64 | return 0;
65 | }
66 |
67 | @Override
68 | public boolean shouldFilter(HttpRequestMessage msg) {
69 | String key = msg.getPathAndQuery();
70 |
71 | return msg.getMethod().equalsIgnoreCase("get")
72 | && key.startsWith("/api")
73 | && !Util.startsWithAnyOf(key, pushPrefixes);
74 | }
75 | }
76 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/MasterCacheLoader.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.filters;
2 |
3 | import com.google.common.cache.Cache;
4 | import com.google.common.cache.CacheBuilder;
5 | import com.google.common.cache.CacheStats;
6 | import com.google.inject.Inject;
7 | import com.netflix.config.DynamicBooleanProperty;
8 | import com.netflix.config.DynamicIntProperty;
9 | import com.netflix.spectator.api.BasicTag;
10 | import com.netflix.spectator.api.Registry;
11 | import com.netflix.spectator.api.Spectator;
12 | import com.netflix.spectator.api.patterns.PolledMeter;
13 | import com.netflix.zuul.filters.http.HttpOutboundSyncFilter;
14 | import com.netflix.zuul.message.http.HttpResponseMessage;
15 | import io.mantisrx.api.Constants;
16 | import lombok.extern.slf4j.Slf4j;
17 |
18 | import java.util.concurrent.TimeUnit;
19 |
20 | @Slf4j
21 | public class MasterCacheLoader extends HttpOutboundSyncFilter {
22 |
23 | @Override
24 | public boolean needsBodyBuffered(HttpResponseMessage message) {
25 | return true;
26 | }
27 |
28 | private static DynamicBooleanProperty cacheEnabled = new DynamicBooleanProperty("mantisapi.cache.enabled", false);
29 | private static DynamicIntProperty cacheSize = new DynamicIntProperty("mantisapi.cache.size", 1000);
30 | private static DynamicIntProperty cacheDurationSeconds = new DynamicIntProperty("mantisapi.cache.seconds", 1);
31 |
32 | public static final Cache masterCache = CacheBuilder.newBuilder()
33 | .maximumSize(cacheSize.get())
34 | .expireAfterWrite(cacheDurationSeconds.get(), TimeUnit.SECONDS)
35 | .build();
36 |
37 | @Inject
38 | public MasterCacheLoader(Registry registry) {
39 | CacheStats stats = masterCache.stats();
40 | PolledMeter.using(registry)
41 | .withName("mantis.api.cache.size")
42 | .withTag(new BasicTag("id", "api"))
43 | .monitorMonotonicCounter(masterCache, Cache::size);
44 |
45 | PolledMeter.using(registry)
46 | .withName("mantis.api.cache.hitCount")
47 | .withTag(new BasicTag("id", "api"))
48 | .monitorMonotonicCounter(stats, CacheStats::hitCount);
49 |
50 | PolledMeter.using(registry)
51 | .withName("mantis.api.cache.missCount")
52 | .withTag(new BasicTag("id", "api"))
53 | .monitorMonotonicCounter(stats, CacheStats::missCount);
54 | }
55 |
56 | @Override
57 | public HttpResponseMessage apply(HttpResponseMessage input) {
58 | String key = input.getInboundRequest().getPathAndQuery();
59 | String responseBody = input.getBodyAsText();
60 |
61 | if (null != responseBody && cacheEnabled.get()) {
62 | masterCache.put(key, responseBody);
63 | }
64 |
65 | return input;
66 | }
67 |
68 | @Override
69 | public int filterOrder() {
70 | return 999;
71 | }
72 |
73 | @Override
74 | public boolean shouldFilter(HttpResponseMessage msg) {
75 | return msg.getOutboundRequest().getContext().getRouteVIP() != null
76 | && msg.getOutboundRequest().getContext().getRouteVIP().equalsIgnoreCase("api")
77 | && msg.getInboundRequest().getMethod().equalsIgnoreCase("get")
78 | && msg.getHeaders().getAll(Constants.MANTISAPI_CACHED_HEADER).size() == 0; // Set by the MasterCacheHitChecker, ensures we aren't re-caching.
79 | }
80 | }
81 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/MetricsReporting.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.filters;
2 |
3 | import com.netflix.spectator.api.Counter;
4 | import com.netflix.spectator.api.Timer;
5 | import com.netflix.zuul.filters.http.HttpOutboundSyncFilter;
6 | import com.netflix.zuul.message.http.HttpResponseMessage;
7 | import com.netflix.zuul.netty.SpectatorUtils;
8 | import io.vavr.Tuple;
9 | import io.vavr.Tuple2;
10 |
11 | import java.util.concurrent.ConcurrentHashMap;
12 | import java.util.concurrent.TimeUnit;
13 |
14 | public class MetricsReporting extends HttpOutboundSyncFilter {
15 |
16 | private static final ConcurrentHashMap, Timer> timerCache = new ConcurrentHashMap<>(500);
17 | private static final ConcurrentHashMap, Counter> counterCache = new ConcurrentHashMap<>(500);
18 |
19 | @Override
20 | public HttpResponseMessage apply(HttpResponseMessage input) {
21 | String path = input.getInboundRequest().getPath();
22 | String status = statusCodeToStringRepresentation(input.getStatus());
23 |
24 | // Record Latency. Zuul no longer record total request time.
25 | timerCache.computeIfAbsent(Tuple.of(path, status),
26 | tuple -> SpectatorUtils.newTimer("latency", path,"status", status))
27 | .record(input.getContext().getOriginReportedDuration(), TimeUnit.NANOSECONDS);
28 |
29 | // Record Request
30 | counterCache.computeIfAbsent(Tuple.of(path, status),
31 | tuple -> SpectatorUtils.newCounter("requests", path, "status", status))
32 | .increment();
33 |
34 | return input;
35 | }
36 |
37 | private String statusCodeToStringRepresentation(Integer statusCode) {
38 | return (statusCode / 100) + "xx";
39 | }
40 |
41 | @Override
42 | public int filterOrder() {
43 | return -100;
44 | }
45 |
46 | @Override
47 | public boolean shouldFilter(HttpResponseMessage msg) {
48 | return true;
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/Options.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.filters;
2 |
3 | import com.netflix.zuul.filters.http.HttpSyncEndpoint;
4 | import com.netflix.zuul.message.http.HttpRequestMessage;
5 | import com.netflix.zuul.message.http.HttpResponseMessage;
6 | import com.netflix.zuul.message.http.HttpResponseMessageImpl;
7 | import com.netflix.zuul.stats.status.StatusCategoryUtils;
8 | import com.netflix.zuul.stats.status.ZuulStatusCategory;
9 | import io.netty.handler.codec.http.HttpHeaderNames;
10 | import io.netty.handler.codec.http.HttpResponseStatus;
11 |
12 | public class Options extends HttpSyncEndpoint {
13 |
14 | @Override
15 | public HttpResponseMessage apply(HttpRequestMessage request) {
16 | HttpResponseMessage resp = new HttpResponseMessageImpl(request.getContext(), request, HttpResponseStatus.OK.code());
17 | resp.setBodyAsText("");
18 | StatusCategoryUtils.setStatusCategory(request.getContext(), ZuulStatusCategory.SUCCESS);
19 | return resp;
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/OutboundHeaders.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.filters;
2 |
3 | import com.netflix.zuul.filters.http.HttpOutboundSyncFilter;
4 | import com.netflix.zuul.message.HeaderName;
5 | import com.netflix.zuul.message.http.HttpResponseMessage;
6 | import io.netty.handler.codec.http.HttpHeaderNames;
7 | import io.netty.util.AsciiString;
8 |
9 | public class OutboundHeaders extends HttpOutboundSyncFilter {
10 |
11 | @Override
12 | public boolean shouldFilter(HttpResponseMessage msg) {
13 | return true;
14 | }
15 |
16 | @Override
17 | public HttpResponseMessage apply(HttpResponseMessage resp) {
18 | upsert(resp, HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN, "*");
19 | addHeaderIfMissing(resp, HttpHeaderNames.ACCESS_CONTROL_ALLOW_HEADERS,
20 | "Origin, X-Requested-With, Accept, Content-Type, Cache-Control");
21 | addHeaderIfMissing(resp, HttpHeaderNames.ACCESS_CONTROL_ALLOW_METHODS,
22 | "GET, OPTIONS, PUT, POST, DELETE, CONNECT");
23 | addHeaderIfMissing(resp, HttpHeaderNames.ACCESS_CONTROL_ALLOW_CREDENTIALS, "true");
24 | return resp;
25 | }
26 |
27 | private void upsert(HttpResponseMessage resp, AsciiString name, String value) {
28 | resp.getHeaders().remove(new HeaderName(name.toString()));
29 | resp.getHeaders().add(new HeaderName(name.toString()), value);
30 | }
31 |
32 | private void addHeaderIfMissing(HttpResponseMessage resp, AsciiString name, String value) {
33 | if (resp.getHeaders().getAll(name.toString()).size() == 0) {
34 | resp.getHeaders().add(name.toString(), value);
35 | }
36 | }
37 |
38 | @Override
39 | public int filterOrder() {
40 | return 0;
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/filters/Routes.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package io.mantisrx.api.filters;
17 |
18 | import com.netflix.zuul.context.SessionContext;
19 | import com.netflix.zuul.filters.http.HttpInboundSyncFilter;
20 | import com.netflix.zuul.message.http.HttpRequestMessage;
21 | import com.netflix.zuul.netty.filter.ZuulEndPointRunner;
22 | import lombok.extern.slf4j.Slf4j;
23 |
24 | @Slf4j
25 | public class Routes extends HttpInboundSyncFilter {
26 |
27 | @Override
28 | public int filterOrder() {
29 | return 0;
30 | }
31 |
32 | @Override
33 | public boolean shouldFilter(HttpRequestMessage httpRequestMessage) {
34 | return true;
35 | }
36 |
37 | @Override
38 | public HttpRequestMessage apply(HttpRequestMessage request) {
39 | SessionContext context = request.getContext();
40 | String path = request.getPath();
41 | String host = request.getOriginalHost();
42 |
43 | if (request.getMethod().toLowerCase().equals("options")) {
44 | context.setEndpoint(Options.class.getCanonicalName());
45 | } else if (path.equalsIgnoreCase("/healthcheck")) {
46 | context.setEndpoint(Healthcheck.class.getCanonicalName());
47 | } else if (path.equalsIgnoreCase("/favicon.ico")) {
48 | context.setEndpoint(Favicon.class.getCanonicalName());
49 | } else if (path.startsWith(Artifacts.PATH_SPEC)) {
50 | context.setEndpoint(Artifacts.class.getCanonicalName());
51 | } else if (path.equalsIgnoreCase("/api/v1/mantis/publish/streamDiscovery")) {
52 | context.setEndpoint(AppStreamDiscovery.class.getCanonicalName());
53 | } else if (path.startsWith("/jobClusters/discoveryInfo")) {
54 | String jobCluster = request.getPath().replaceFirst(JobDiscoveryInfoCacheHitChecker.PATH_SPEC + "/", "");
55 | String newUrl = "/api/v1/jobClusters/" + jobCluster + "/latestJobDiscoveryInfo";
56 | request.setPath(newUrl);
57 | context.setEndpoint(ZuulEndPointRunner.PROXY_ENDPOINT_FILTER_NAME);
58 | context.setRouteVIP("api");
59 | } else if (path.equalsIgnoreCase("/api/v1/mql/parse")) {
60 | context.setEndpoint(MQLParser.class.getCanonicalName());
61 | } else if (path.equals(MREAppStreamToJobClusterMapping.PATH_SPEC)) {
62 | context.setEndpoint(MREAppStreamToJobClusterMapping.class.getCanonicalName());
63 | } else {
64 | context.setEndpoint(ZuulEndPointRunner.PROXY_ENDPOINT_FILTER_NAME);
65 | context.setRouteVIP("api");
66 | }
67 |
68 | return request;
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/initializers/MantisApiServerChannelInitializer.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package io.mantisrx.api.initializers;
17 |
18 | import com.netflix.netty.common.HttpLifecycleChannelHandler;
19 | import com.netflix.netty.common.channel.config.ChannelConfig;
20 | import com.netflix.netty.common.channel.config.CommonChannelConfigKeys;
21 | import com.netflix.zuul.netty.server.BaseZuulChannelInitializer;
22 | import com.netflix.zuul.netty.ssl.SslContextFactory;
23 | import io.mantisrx.api.Util;
24 | import io.mantisrx.api.push.ConnectionBroker;
25 | import io.mantisrx.api.push.MantisSSEHandler;
26 | import io.mantisrx.api.push.MantisWebSocketFrameHandler;
27 | import io.mantisrx.api.tunnel.CrossRegionHandler;
28 | import io.mantisrx.api.tunnel.MantisCrossRegionalClient;
29 | import io.mantisrx.server.master.client.HighAvailabilityServices;
30 | import io.netty.channel.Channel;
31 | import io.netty.channel.ChannelHandlerContext;
32 | import io.netty.channel.ChannelInboundHandlerAdapter;
33 | import io.netty.channel.ChannelPipeline;
34 | import io.netty.channel.group.ChannelGroup;
35 | import io.netty.handler.codec.http.HttpObjectAggregator;
36 | import io.netty.handler.codec.http.websocketx.WebSocketServerProtocolHandler;
37 | import io.netty.handler.ssl.SslContext;
38 | import io.netty.handler.ssl.SslHandler;
39 | import io.netty.handler.stream.ChunkedWriteHandler;
40 | import java.util.List;
41 | import javax.net.ssl.SSLException;
42 | import rx.Scheduler;
43 |
44 |
45 | public class MantisApiServerChannelInitializer extends BaseZuulChannelInitializer
46 | {
47 | private final SslContextFactory sslContextFactory;
48 | private final SslContext sslContext;
49 | private final boolean isSSlFromIntermediary;
50 |
51 | private final ConnectionBroker connectionBroker;
52 | private final HighAvailabilityServices highAvailabilityServices;
53 | private final MantisCrossRegionalClient mantisCrossRegionalClient;
54 | private final Scheduler scheduler;
55 | private final List pushPrefixes;
56 | private final boolean sslEnabled;
57 |
58 | public MantisApiServerChannelInitializer(
59 | String metricId,
60 | ChannelConfig channelConfig,
61 | ChannelConfig channelDependencies,
62 | ChannelGroup channels,
63 | List pushPrefixes,
64 | HighAvailabilityServices highAvailabilityServices,
65 | MantisCrossRegionalClient mantisCrossRegionalClient,
66 | ConnectionBroker connectionBroker,
67 | Scheduler scheduler,
68 | boolean sslEnabled) {
69 | super(metricId, channelConfig, channelDependencies, channels);
70 |
71 | this.pushPrefixes = pushPrefixes;
72 | this.connectionBroker = connectionBroker;
73 | this.highAvailabilityServices = highAvailabilityServices;
74 | this.mantisCrossRegionalClient = mantisCrossRegionalClient;
75 | this.scheduler = scheduler;
76 | this.sslEnabled = sslEnabled;
77 |
78 | this.isSSlFromIntermediary = channelConfig.get(CommonChannelConfigKeys.isSSlFromIntermediary);
79 | this.sslContextFactory = channelConfig.get(CommonChannelConfigKeys.sslContextFactory);
80 |
81 | if (sslEnabled) {
82 | try {
83 | sslContext = sslContextFactory.createBuilderForServer().build();
84 | } catch (SSLException e) {
85 | throw new RuntimeException("Error configuring SslContext!", e);
86 | }
87 |
88 | // Enable TLS Session Tickets support.
89 | sslContextFactory.enableSessionTickets(sslContext);
90 |
91 | // Setup metrics tracking the OpenSSL stats.
92 | sslContextFactory.configureOpenSslStatsMetrics(sslContext, metricId);
93 | } else {
94 | sslContext = null;
95 | }
96 | }
97 |
98 |
99 |
100 |
101 | @Override
102 | protected void initChannel(Channel ch) throws Exception
103 | {
104 |
105 | // Configure our pipeline of ChannelHandlerS.
106 | ChannelPipeline pipeline = ch.pipeline();
107 |
108 | storeChannel(ch);
109 | addTimeoutHandlers(pipeline);
110 | addPassportHandler(pipeline);
111 | addTcpRelatedHandlers(pipeline);
112 |
113 | if (sslEnabled) {
114 | SslHandler sslHandler = sslContext.newHandler(ch.alloc());
115 | sslHandler.engine().setEnabledProtocols(sslContextFactory.getProtocols());
116 | pipeline.addLast("ssl", sslHandler);
117 | addSslInfoHandlers(pipeline, isSSlFromIntermediary);
118 | addSslClientCertChecks(pipeline);
119 | }
120 |
121 | addHttp1Handlers(pipeline);
122 | addHttpRelatedHandlers(pipeline);
123 |
124 | pipeline.addLast("mantishandler", new MantisChannelHandler(pushPrefixes));
125 | }
126 |
127 | /**
128 | * Adds a series of handlers for providing SSE/Websocket connections
129 | * to Mantis Jobs.
130 | *
131 | * @param pipeline The netty pipeline to which push handlers should be added.
132 | * @param url The url with which to initiate the websocket handler.
133 | */
134 | protected void addPushHandlers(final ChannelPipeline pipeline, String url) {
135 | pipeline.addLast(new ChunkedWriteHandler());
136 | pipeline.addLast(new HttpObjectAggregator(64 * 1024));
137 | pipeline.addLast(new MantisSSEHandler(connectionBroker, highAvailabilityServices, pushPrefixes));
138 | pipeline.addLast(new WebSocketServerProtocolHandler(url, true));
139 | pipeline.addLast(new MantisWebSocketFrameHandler(connectionBroker));
140 | }
141 |
142 | /**
143 | * Adds a series of handlers for providing SSE/Websocket connections
144 | * to Mantis Jobs.
145 | *
146 | * @param pipeline The netty pipeline to which regional handlers should be added.
147 | */
148 | protected void addRegionalHandlers(final ChannelPipeline pipeline) {
149 | pipeline.addLast(new ChunkedWriteHandler());
150 | pipeline.addLast(new HttpObjectAggregator(10 * 1024 * 1024));
151 | pipeline.addLast(new CrossRegionHandler(pushPrefixes, mantisCrossRegionalClient, connectionBroker, scheduler));
152 | }
153 |
154 | /**
155 | * The MantisChannelHandler's job is to initialize the tail end of the pipeline differently
156 | * depending on the URI of the request. This is largely to circumvent issues with endpoint responses
157 | * when the push handlers preceed the Zuul handlers.
158 | */
159 | @Sharable
160 | public class MantisChannelHandler extends ChannelInboundHandlerAdapter {
161 |
162 | private final List pushPrefixes;
163 |
164 | public MantisChannelHandler(List pushPrefixes) {
165 | this.pushPrefixes = pushPrefixes;
166 | }
167 |
168 | @Override
169 | public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
170 | if (evt instanceof HttpLifecycleChannelHandler.StartEvent) {
171 | HttpLifecycleChannelHandler.StartEvent startEvent = (HttpLifecycleChannelHandler.StartEvent) evt;
172 | String uri = startEvent.getRequest().uri();
173 | ChannelPipeline pipeline = ctx.pipeline();
174 |
175 | removeEverythingAfterThis(pipeline);
176 |
177 | if (Util.startsWithAnyOf(uri, this.pushPrefixes)) {
178 | addPushHandlers(pipeline, uri);
179 | } else if(uri.startsWith("/region/")) {
180 | addRegionalHandlers(pipeline);
181 | } else {
182 | addZuulHandlers(pipeline);
183 | }
184 | }
185 | ctx.fireUserEventTriggered(evt);
186 | }
187 | }
188 |
189 | private void removeEverythingAfterThis(ChannelPipeline pipeline) {
190 | while (pipeline.last().getClass() != MantisChannelHandler.class) {
191 | pipeline.removeLast();
192 | }
193 | }
194 | }
195 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/proto/AppDiscoveryMap.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package io.mantisrx.api.proto;
17 |
18 | import io.mantisrx.server.core.JobSchedulingInfo;
19 |
20 | import java.util.HashMap;
21 | import java.util.Map;
22 |
23 | public class AppDiscoveryMap {
24 | public final String version;
25 | public final Long timestamp;
26 | public final Map> mappings = new HashMap<>();
27 |
28 | public AppDiscoveryMap(String version, Long timestamp) {
29 | this.version = version;
30 | this.timestamp = timestamp;
31 | }
32 |
33 | public void addMapping(String app, String stream, JobSchedulingInfo schedulingInfo) {
34 | if(!mappings.containsKey(app)) {
35 | mappings.put(app, new HashMap());
36 | }
37 | mappings.get(app).put(stream, schedulingInfo);
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/proto/Artifact.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2019 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package io.mantisrx.api.proto;
18 |
19 | import java.util.Objects;
20 |
21 |
22 | public class Artifact {
23 | private long sizeInBytes;
24 | private String fileName;
25 | private byte[] content;
26 |
27 | public Artifact(String fileName, long sizeInBytes, byte[] content) {
28 | Objects.requireNonNull(fileName, "File name cannot be null");
29 | Objects.requireNonNull(content, "Content cannot be null");
30 | this.fileName = fileName;
31 | this.sizeInBytes = sizeInBytes;
32 | this.content = content;
33 | }
34 |
35 | public long getSizeInBytes() {
36 | return this.sizeInBytes;
37 | }
38 |
39 | public byte[] getContent() {
40 | return content;
41 | }
42 |
43 | public String getFileName() {
44 | return fileName;
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/push/ConnectionBroker.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.push;
2 |
3 | import io.mantisrx.shaded.com.fasterxml.jackson.databind.ObjectMapper;
4 | import com.google.inject.Inject;
5 | import com.google.inject.Singleton;
6 | import com.google.inject.name.Named;
7 | import com.netflix.spectator.api.Counter;
8 | import com.netflix.zuul.netty.SpectatorUtils;
9 | import io.mantisrx.api.Constants;
10 | import io.mantisrx.api.Util;
11 | import io.mantisrx.api.services.JobDiscoveryService;
12 | import io.mantisrx.api.tunnel.MantisCrossRegionalClient;
13 | import io.mantisrx.client.MantisClient;
14 | import io.mantisrx.client.SinkConnectionFunc;
15 | import io.mantisrx.client.SseSinkConnectionFunction;
16 | import io.mantisrx.common.MantisServerSentEvent;
17 | import io.mantisrx.runtime.parameter.SinkParameters;
18 | import io.mantisrx.server.worker.client.MetricsClient;
19 | import io.mantisrx.server.worker.client.SseWorkerConnectionFunction;
20 | import io.mantisrx.server.worker.client.WorkerConnectionsStatus;
21 | import io.mantisrx.server.worker.client.WorkerMetricsClient;
22 | import io.vavr.control.Try;
23 | import lombok.extern.slf4j.Slf4j;
24 | import mantis.io.reactivex.netty.protocol.http.client.HttpClientRequest;
25 | import mantis.io.reactivex.netty.protocol.http.client.HttpClientResponse;
26 | import mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent;
27 | import rx.Observable;
28 | import rx.Observer;
29 | import rx.Scheduler;
30 | import rx.functions.Action1;
31 | import rx.schedulers.Schedulers;
32 |
33 | import java.util.List;
34 | import java.util.Map;
35 | import java.util.WeakHashMap;
36 | import java.util.concurrent.TimeUnit;
37 | import java.util.concurrent.atomic.AtomicBoolean;
38 |
39 | import static io.mantisrx.api.Constants.TunnelPingMessage;
40 | import static io.mantisrx.api.Util.getLocalRegion;
41 |
42 | @Slf4j
43 | @Singleton
44 | public class ConnectionBroker {
45 |
46 | private final MantisClient mantisClient;
47 | private final MantisCrossRegionalClient mantisCrossRegionalClient;
48 | private final WorkerMetricsClient workerMetricsClient;
49 | private final JobDiscoveryService jobDiscoveryService;
50 | private final Scheduler scheduler;
51 | private final ObjectMapper objectMapper;
52 |
53 | private final Map> connectionCache = new WeakHashMap<>();
54 |
55 | @Inject
56 | public ConnectionBroker(MantisClient mantisClient,
57 | MantisCrossRegionalClient mantisCrossRegionalClient,
58 | WorkerMetricsClient workerMetricsClient,
59 | @Named("io-scheduler") Scheduler scheduler,
60 | ObjectMapper objectMapper) {
61 | this.mantisClient = mantisClient;
62 | this.mantisCrossRegionalClient = mantisCrossRegionalClient;
63 | this.workerMetricsClient = workerMetricsClient;
64 | this.jobDiscoveryService = JobDiscoveryService.getInstance(mantisClient, scheduler);
65 | this.scheduler = scheduler;
66 | this.objectMapper = objectMapper;
67 | }
68 |
69 | public Observable connect(PushConnectionDetails details) {
70 |
71 | if (!connectionCache.containsKey(details)) {
72 | switch (details.type) {
73 | case CONNECT_BY_NAME:
74 | return getConnectByNameFor(details)
75 | .subscribeOn(scheduler)
76 | .doOnUnsubscribe(() -> {
77 | log.info("Purging {} from cache.", details);
78 | connectionCache.remove(details);
79 | })
80 | .doOnCompleted(() -> {
81 | log.info("Purging {} from cache.", details);
82 | connectionCache.remove(details);
83 | })
84 | .share();
85 | case CONNECT_BY_ID:
86 | return getConnectByIdFor(details)
87 | .subscribeOn(scheduler)
88 | .doOnUnsubscribe(() -> {
89 | log.info("Purging {} from cache.", details);
90 | connectionCache.remove(details);
91 | })
92 | .doOnCompleted(() -> {
93 | log.info("Purging {} from cache.", details);
94 | connectionCache.remove(details);
95 | })
96 | .share();
97 |
98 | case METRICS:
99 | return getWorkerMetrics(details)
100 | .subscribeOn(scheduler)
101 | .doOnUnsubscribe(() -> {
102 | log.info("Purging {} from cache.", details);
103 | connectionCache.remove(details);
104 | })
105 | .doOnCompleted(() -> {
106 | log.info("Purging {} from cache.", details);
107 | connectionCache.remove(details);
108 | });
109 |
110 | case JOB_STATUS:
111 | connectionCache.put(details,
112 | mantisClient
113 | .getJobStatusObservable(details.target)
114 | .subscribeOn(scheduler)
115 | .doOnCompleted(() -> {
116 | log.info("Purging {} from cache.", details);
117 | connectionCache.remove(details);
118 | })
119 | .doOnUnsubscribe(() -> {
120 | log.info("Purging {} from cache.", details);
121 | connectionCache.remove(details);
122 | })
123 | .replay(25)
124 | .autoConnect());
125 | break;
126 | case JOB_SCHEDULING_INFO:
127 | connectionCache.put(details,
128 | mantisClient.getSchedulingChanges(details.target)
129 | .subscribeOn(scheduler)
130 | .map(changes -> Try.of(() -> objectMapper.writeValueAsString(changes)).getOrElse("Error"))
131 | .doOnCompleted(() -> {
132 | log.info("Purging {} from cache.", details);
133 | connectionCache.remove(details);
134 | })
135 | .doOnUnsubscribe(() -> {
136 | log.info("Purging {} from cache.", details);
137 | connectionCache.remove(details);
138 | })
139 | .replay(1)
140 | .autoConnect());
141 | break;
142 |
143 | case JOB_CLUSTER_DISCOVERY:
144 | connectionCache.put(details,
145 | jobDiscoveryService.jobDiscoveryInfoStream(jobDiscoveryService.key(JobDiscoveryService.LookupType.JOB_CLUSTER, details.target))
146 | .subscribeOn(scheduler)
147 | .map(jdi ->Try.of(() -> objectMapper.writeValueAsString(jdi)).getOrElse("Error"))
148 | .doOnCompleted(() -> {
149 | log.info("Purging {} from cache.", details);
150 | connectionCache.remove(details);
151 | })
152 | .doOnUnsubscribe(() -> {
153 | log.info("Purging {} from cache.", details);
154 | connectionCache.remove(details);
155 | })
156 | .replay(1)
157 | .autoConnect());
158 | break;
159 | }
160 | log.info("Caching connection for: {}", details);
161 | }
162 | return connectionCache.get(details);
163 | }
164 |
165 | //
166 | // Helpers
167 | //
168 |
169 | private Observable getConnectByNameFor(PushConnectionDetails details) {
170 | return details.regions.isEmpty()
171 | ? getResults(false, this.mantisClient, details.target, details.getSinkparameters())
172 | .flatMap(m -> m)
173 | .map(MantisServerSentEvent::getEventAsString)
174 | : getRemoteDataObservable(details.getUri(), details.target, details.getRegions().asJava());
175 | }
176 |
177 | private Observable getConnectByIdFor(PushConnectionDetails details) {
178 | return details.getRegions().isEmpty()
179 | ? getResults(true, this.mantisClient, details.target, details.getSinkparameters())
180 | .flatMap(m -> m)
181 | .map(MantisServerSentEvent::getEventAsString)
182 | : getRemoteDataObservable(details.getUri(), details.target, details.getRegions().asJava());
183 | }
184 |
185 |
186 | private static SinkConnectionFunc getSseConnFunc(final String target, SinkParameters sinkParameters) {
187 | return new SseSinkConnectionFunction(true,
188 | t -> log.warn("Reconnecting to sink of job " + target + " after error: " + t.getMessage()),
189 | sinkParameters);
190 | }
191 |
192 | private static Observable> getResults(boolean isJobId, MantisClient mantisClient,
193 | final String target, SinkParameters sinkParameters) {
194 | final AtomicBoolean hasError = new AtomicBoolean();
195 | return isJobId ?
196 | mantisClient.getSinkClientByJobId(target, getSseConnFunc(target, sinkParameters), null).getResults() :
197 | mantisClient.getSinkClientByJobName(target, getSseConnFunc(target, sinkParameters), null)
198 | .switchMap(serverSentEventSinkClient -> {
199 | if (serverSentEventSinkClient.hasError()) {
200 | hasError.set(true);
201 | return Observable.error(new Exception(serverSentEventSinkClient.getError()));
202 | }
203 | return serverSentEventSinkClient.getResults();
204 | })
205 | .takeWhile(o -> !hasError.get());
206 | }
207 |
208 | //
209 | // Tunnel
210 | //
211 |
212 | private Observable getRemoteDataObservable(String uri, String target, List regions) {
213 | return Observable.from(regions)
214 | .flatMap(region -> {
215 | final String originReplacement = "\\{\"" + Constants.metaOriginName + "\": \"" + region + "\", ";
216 | if (region.equalsIgnoreCase(getLocalRegion())) {
217 | return this.connect(PushConnectionDetails.from(uri))
218 | .map(datum -> datum.replaceFirst("^\\{", originReplacement));
219 | } else {
220 | log.info("Connecting to remote region {} at {}.", region, uri);
221 | return mantisCrossRegionalClient.getSecureSseClient(region)
222 | .submit(HttpClientRequest.createGet(uri))
223 | .retryWhen(Util.getRetryFunc(log, uri + " in " + region))
224 | .doOnError(throwable -> log.warn(
225 | "Error getting response from remote SSE server for uri {} in region {}: {}",
226 | uri, region, throwable.getMessage(), throwable)
227 | ).flatMap(remoteResponse -> {
228 | if (!remoteResponse.getStatus().reasonPhrase().equals("OK")) {
229 | log.warn("Unexpected response from remote sink for uri {} region {}: {}", uri, region, remoteResponse.getStatus().reasonPhrase());
230 | String err = remoteResponse.getHeaders().get(Constants.metaErrorMsgHeader);
231 | if (err == null || err.isEmpty())
232 | err = remoteResponse.getStatus().reasonPhrase();
233 | return Observable.error(new Exception(err))
234 | .map(datum -> datum.getEventAsString());
235 | }
236 | return clientResponseToObservable(remoteResponse, target, region, uri)
237 | .map(datum -> datum.replaceFirst("^\\{", originReplacement))
238 | .doOnError(t -> log.error(t.getMessage()));
239 | })
240 | .subscribeOn(scheduler)
241 | .observeOn(scheduler)
242 | .doOnError(t -> log.warn("Error streaming in remote data ({}). Will retry: {}", region, t.getMessage(), t))
243 | .doOnCompleted(() -> log.info(String.format("remote sink connection complete for uri %s, region=%s", uri, region)));
244 | }
245 | })
246 | .observeOn(scheduler)
247 | .subscribeOn(scheduler)
248 | .doOnError(t -> log.error("Error in flatMapped cross-regional observable for {}", uri, t));
249 | }
250 |
251 | private Observable clientResponseToObservable(HttpClientResponse response, String target, String
252 | region, String uri) {
253 |
254 | Counter numRemoteBytes = SpectatorUtils.newCounter(Constants.numRemoteBytesCounterName, target, "region", region);
255 | Counter numRemoteMessages = SpectatorUtils.newCounter(Constants.numRemoteMessagesCounterName, target, "region", region);
256 | Counter numSseErrors = SpectatorUtils.newCounter(Constants.numSseErrorsCounterName, target, "region", region);
257 |
258 | return response.getContent()
259 | .doOnError(t -> log.warn(t.getMessage()))
260 | .timeout(3 * Constants.TunnelPingIntervalSecs, TimeUnit.SECONDS)
261 | .doOnError(t -> log.warn("Timeout getting data from remote {} connection for {}", region, uri))
262 | .filter(sse -> !(!sse.hasEventType() || !sse.getEventTypeAsString().startsWith("error:")) ||
263 | !TunnelPingMessage.equals(sse.contentAsString()))
264 | .map(t1 -> {
265 | String data = "";
266 | if (t1.hasEventType() && t1.getEventTypeAsString().startsWith("error:")) {
267 | log.error("SSE has error, type=" + t1.getEventTypeAsString() + ", content=" + t1.contentAsString());
268 | numSseErrors.increment();
269 | throw new RuntimeException("Got error SSE event: " + t1.contentAsString());
270 | }
271 | try {
272 | data = t1.contentAsString();
273 | if (data != null) {
274 | numRemoteBytes.increment(data.length());
275 | numRemoteMessages.increment();
276 | }
277 | } catch (Exception e) {
278 | log.error("Could not extract data from SSE " + e.getMessage(), e);
279 | }
280 | return data;
281 | });
282 | }
283 |
284 | private Observable getWorkerMetrics(PushConnectionDetails details) {
285 |
286 | final String jobId = details.target;
287 |
288 | SinkParameters metricNamesFilter = details.getSinkparameters();
289 |
290 | final MetricsClient metricsClient = workerMetricsClient.getMetricsClientByJobId(jobId,
291 | new SseWorkerConnectionFunction(true, new Action1() {
292 | @Override
293 | public void call(Throwable throwable) {
294 | log.error("Metric connection error: " + throwable.getMessage());
295 | try {
296 | Thread.sleep(500);
297 | } catch (InterruptedException ie) {
298 | log.error("Interrupted waiting for retrying connection");
299 | }
300 | }
301 | }, metricNamesFilter),
302 | new Observer() {
303 | @Override
304 | public void onCompleted() {
305 | log.info("got onCompleted in WorkerConnStatus obs");
306 | }
307 |
308 | @Override
309 | public void onError(Throwable e) {
310 | log.info("got onError in WorkerConnStatus obs");
311 | }
312 |
313 | @Override
314 | public void onNext(WorkerConnectionsStatus workerConnectionsStatus) {
315 | log.info("got WorkerConnStatus {}", workerConnectionsStatus);
316 | }
317 | });
318 |
319 | return metricsClient
320 | .getResults()
321 | .flatMap(metrics -> metrics
322 | .map(MantisServerSentEvent::getEventAsString));
323 |
324 | }
325 | }
326 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/push/MantisSSEHandler.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.push;
2 |
3 | import com.netflix.config.DynamicIntProperty;
4 | import com.netflix.spectator.api.Counter;
5 | import com.netflix.zuul.netty.SpectatorUtils;
6 | import io.mantisrx.api.Constants;
7 | import io.mantisrx.api.Util;
8 | import io.mantisrx.server.core.master.MasterDescription;
9 | import io.mantisrx.server.master.client.HighAvailabilityServices;
10 | import io.mantisrx.shaded.com.google.common.util.concurrent.ThreadFactoryBuilder;
11 | import io.netty.buffer.ByteBuf;
12 | import io.netty.buffer.Unpooled;
13 | import io.netty.channel.ChannelHandlerContext;
14 | import io.netty.channel.SimpleChannelInboundHandler;
15 | import io.netty.handler.codec.http.*;
16 | import lombok.extern.slf4j.Slf4j;
17 | import mantis.io.reactivex.netty.RxNetty;
18 | import mantis.io.reactivex.netty.channel.StringTransformer;
19 | import mantis.io.reactivex.netty.pipeline.PipelineConfigurator;
20 | import mantis.io.reactivex.netty.pipeline.PipelineConfigurators;
21 | import mantis.io.reactivex.netty.protocol.http.client.HttpClient;
22 | import mantis.io.reactivex.netty.protocol.http.client.HttpClientRequest;
23 | import mantis.io.reactivex.netty.protocol.http.client.HttpClientResponse;
24 | import mantis.io.reactivex.netty.protocol.http.client.HttpResponseHeaders;
25 | import rx.Observable;
26 | import rx.Subscription;
27 |
28 | import java.nio.charset.StandardCharsets;
29 | import java.util.*;
30 | import java.util.concurrent.BlockingQueue;
31 | import java.util.concurrent.LinkedBlockingQueue;
32 | import java.util.concurrent.ScheduledExecutorService;
33 | import java.util.concurrent.ScheduledFuture;
34 | import java.util.concurrent.ScheduledThreadPoolExecutor;
35 | import java.util.concurrent.TimeUnit;
36 |
37 | /**
38 | * Http handler for the WebSocket/SSE paths.
39 | */
40 | @Slf4j
41 | public class MantisSSEHandler extends SimpleChannelInboundHandler {
42 | private final DynamicIntProperty queueCapacity = new DynamicIntProperty("io.mantisrx.api.push.queueCapacity", 1000);
43 | private final DynamicIntProperty writeIntervalMillis = new DynamicIntProperty("io.mantisrx.api.push.writeIntervalMillis", 50);
44 |
45 | private final ConnectionBroker connectionBroker;
46 | private final HighAvailabilityServices highAvailabilityServices;
47 | private final List pushPrefixes;
48 |
49 | private Subscription subscription;
50 | private ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1,
51 | new ThreadFactoryBuilder().setNameFormat("sse-handler-drainer-%d").build());
52 | private ScheduledFuture drainFuture;
53 | private String uri;
54 |
55 | public MantisSSEHandler(ConnectionBroker connectionBroker, HighAvailabilityServices highAvailabilityServices,
56 | List pushPrefixes) {
57 | super(true);
58 | this.connectionBroker = connectionBroker;
59 | this.highAvailabilityServices = highAvailabilityServices;
60 | this.pushPrefixes = pushPrefixes;
61 | }
62 |
63 | @Override
64 | protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) throws Exception {
65 | if (Util.startsWithAnyOf(request.uri(), pushPrefixes)
66 | && !isWebsocketUpgrade(request)) {
67 |
68 | if (HttpUtil.is100ContinueExpected(request)) {
69 | send100Contine(ctx);
70 | }
71 |
72 | HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1,
73 | HttpResponseStatus.OK);
74 | HttpHeaders headers = response.headers();
75 | headers.add(HttpHeaderNames.ACCESS_CONTROL_ALLOW_ORIGIN, "*");
76 | headers.add(HttpHeaderNames.ACCESS_CONTROL_ALLOW_HEADERS, "Origin, X-Requested-With, Accept, Content-Type, Cache-Control");
77 | headers.set(HttpHeaderNames.CONTENT_TYPE, "text/event-stream");
78 | headers.set(HttpHeaderNames.CACHE_CONTROL, "no-cache, no-store, max-age=0, must-revalidate");
79 | headers.set(HttpHeaderNames.PRAGMA, HttpHeaderValues.NO_CACHE);
80 | headers.set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED);
81 | response.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.KEEP_ALIVE);
82 | ctx.writeAndFlush(response);
83 |
84 | uri = request.uri();
85 | final PushConnectionDetails pcd =
86 | isSubmitAndConnect(request)
87 | ? new PushConnectionDetails(uri, jobSubmit(request), PushConnectionDetails.TARGET_TYPE.CONNECT_BY_ID, io.vavr.collection.List.empty())
88 | : PushConnectionDetails.from(uri);
89 | log.info("SSE Connecting for: {}", pcd);
90 |
91 | boolean tunnelPingsEnabled = isTunnelPingsEnabled(uri);
92 |
93 | final String[] tags = Util.getTaglist(uri, pcd.target);
94 | Counter numDroppedBytesCounter = SpectatorUtils.newCounter(Constants.numDroppedBytesCounterName, pcd.target, tags);
95 | Counter numDroppedMessagesCounter = SpectatorUtils.newCounter(Constants.numDroppedMessagesCounterName, pcd.target, tags);
96 | Counter numMessagesCounter = SpectatorUtils.newCounter(Constants.numMessagesCounterName, pcd.target, tags);
97 | Counter numBytesCounter = SpectatorUtils.newCounter(Constants.numBytesCounterName, pcd.target, tags);
98 | Counter drainTriggeredCounter = SpectatorUtils.newCounter(Constants.drainTriggeredCounterName, pcd.target, tags);
99 | Counter numIncomingMessagesCounter = SpectatorUtils.newCounter(Constants.numIncomingMessagesCounterName, pcd.target, tags);
100 |
101 | BlockingQueue queue = new LinkedBlockingQueue<>(queueCapacity.get());
102 |
103 | drainFuture = scheduledExecutorService.scheduleAtFixedRate(() -> {
104 | try {
105 | if (queue.size() > 0 && ctx.channel().isWritable()) {
106 | drainTriggeredCounter.increment();
107 | final List items = new ArrayList<>(queue.size());
108 | synchronized (queue) {
109 | queue.drainTo(items);
110 | }
111 | for (String data : items) {
112 | ctx.write(Unpooled.copiedBuffer(data, StandardCharsets.UTF_8));
113 | numMessagesCounter.increment();
114 | numBytesCounter.increment(data.length());
115 | }
116 | ctx.flush();
117 | }
118 | } catch (Exception ex) {
119 | log.error("Error writing to channel", ex);
120 | }
121 | }, writeIntervalMillis.get(), writeIntervalMillis.get(), TimeUnit.MILLISECONDS);
122 |
123 | this.subscription = this.connectionBroker.connect(pcd)
124 | .doOnNext(event -> numIncomingMessagesCounter.increment())
125 | .mergeWith(tunnelPingsEnabled
126 | ? Observable.interval(Constants.TunnelPingIntervalSecs, Constants.TunnelPingIntervalSecs,
127 | TimeUnit.SECONDS)
128 | .map(l -> Constants.TunnelPingMessage)
129 | : Observable.empty())
130 | .doOnNext(event -> {
131 | if (!Constants.DUMMY_TIMER_DATA.equals(event)) {
132 | String data = Constants.SSE_DATA_PREFIX + event + Constants.SSE_DATA_SUFFIX;
133 | boolean offer = false;
134 | synchronized (queue) {
135 | offer = queue.offer(data);
136 | }
137 | if (!offer) {
138 | numDroppedBytesCounter.increment(data.length());
139 | numDroppedMessagesCounter.increment();
140 | }
141 | }
142 | })
143 | .subscribe();
144 | } else {
145 | ctx.fireChannelRead(request.retain());
146 | }
147 | }
148 |
149 | private static void send100Contine(ChannelHandlerContext ctx) {
150 | FullHttpResponse response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1,
151 | HttpResponseStatus.CONTINUE);
152 | ctx.writeAndFlush(response);
153 | }
154 |
155 | private boolean isTunnelPingsEnabled(String uri) {
156 | QueryStringDecoder queryStringDecoder = new QueryStringDecoder(uri);
157 | return queryStringDecoder.parameters()
158 | .getOrDefault(Constants.TunnelPingParamName, Arrays.asList("false"))
159 | .get(0)
160 | .equalsIgnoreCase("true");
161 | }
162 |
163 | private boolean isWebsocketUpgrade(HttpRequest request) {
164 | HttpHeaders headers = request.headers();
165 | // Header "Connection" contains "upgrade" (case insensitive) and
166 | // Header "Upgrade" equals "websocket" (case insensitive)
167 | String connection = headers.get(HttpHeaderNames.CONNECTION);
168 | String upgrade = headers.get(HttpHeaderNames.UPGRADE);
169 | return connection != null && connection.toLowerCase().contains("upgrade") &&
170 | upgrade != null && upgrade.toLowerCase().equals("websocket");
171 | }
172 |
173 |
174 | private boolean isSubmitAndConnect(HttpRequest request) {
175 | return request.method().equals(HttpMethod.POST) && request.uri().contains("jobsubmitandconnect");
176 | }
177 |
178 | @Override
179 | public void channelUnregistered(ChannelHandlerContext ctx) throws Exception {
180 | log.info("Channel {} is unregistered. URI: {}", ctx.channel(), uri);
181 | unsubscribeIfSubscribed();
182 | super.channelUnregistered(ctx);
183 | }
184 |
185 | @Override
186 | public void channelInactive(ChannelHandlerContext ctx) throws Exception {
187 | log.info("Channel {} is inactive. URI: {}", ctx.channel(), uri);
188 | unsubscribeIfSubscribed();
189 | super.channelInactive(ctx);
190 | }
191 |
192 | @Override
193 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
194 | log.warn("Exception caught by channel {}. URI: {}", ctx.channel(), uri, cause);
195 | unsubscribeIfSubscribed();
196 | ctx.close();
197 | }
198 |
199 | /** Unsubscribe if it's subscribed. */
200 | private void unsubscribeIfSubscribed() {
201 | if (subscription != null && !subscription.isUnsubscribed()) {
202 | log.info("SSE unsubscribing subscription with URI: {}", uri);
203 | subscription.unsubscribe();
204 | }
205 | if (drainFuture != null) {
206 | drainFuture.cancel(false);
207 | }
208 | if (scheduledExecutorService != null) {
209 | scheduledExecutorService.shutdown();
210 | }
211 | }
212 |
213 | public String jobSubmit(FullHttpRequest request) {
214 | final String API_JOB_SUBMIT_PATH = "/api/submit";
215 |
216 | String content = request.content().toString(StandardCharsets.UTF_8);
217 | return callPostOnMaster(highAvailabilityServices.getMasterMonitor().getMasterObservable(), API_JOB_SUBMIT_PATH, content)
218 | .retryWhen(Util.getRetryFunc(log, API_JOB_SUBMIT_PATH))
219 | .flatMap(masterResponse -> masterResponse.getByteBuf()
220 | .take(1)
221 | .map(byteBuf -> {
222 | final String s = byteBuf.toString(StandardCharsets.UTF_8);
223 | log.info("response: " + s);
224 | return s;
225 | }))
226 | .take(1)
227 | .toBlocking()
228 | .first();
229 | }
230 |
231 | public static class MasterResponse {
232 |
233 | private final HttpResponseStatus status;
234 | private final Observable byteBuf;
235 | private final HttpResponseHeaders responseHeaders;
236 |
237 | public MasterResponse(HttpResponseStatus status, Observable byteBuf, HttpResponseHeaders responseHeaders) {
238 | this.status = status;
239 | this.byteBuf = byteBuf;
240 | this.responseHeaders = responseHeaders;
241 | }
242 |
243 | public HttpResponseStatus getStatus() {
244 | return status;
245 | }
246 |
247 | public Observable getByteBuf() {
248 | return byteBuf;
249 | }
250 |
251 | public HttpResponseHeaders getResponseHeaders() { return responseHeaders; }
252 | }
253 |
254 | public static Observable callPostOnMaster(Observable masterObservable, String uri, String content) {
255 | PipelineConfigurator, HttpClientRequest> pipelineConfigurator
256 | = PipelineConfigurators.httpClientConfigurator();
257 |
258 | return masterObservable
259 | .filter(Objects::nonNull)
260 | .flatMap(masterDesc -> {
261 | HttpClient client =
262 | RxNetty.newHttpClientBuilder(masterDesc.getHostname(), masterDesc.getApiPort())
263 | .pipelineConfigurator(pipelineConfigurator)
264 | .build();
265 | HttpClientRequest request = HttpClientRequest.create(HttpMethod.POST, uri);
266 | request = request.withHeader(HttpHeaderNames.CONTENT_TYPE.toString(), HttpHeaderValues.APPLICATION_JSON.toString());
267 | request.withRawContent(content, StringTransformer.DEFAULT_INSTANCE);
268 | return client.submit(request)
269 | .map(response -> new MasterResponse(response.getStatus(), response.getContent(), response.getHeaders()));
270 | })
271 | .take(1);
272 | }
273 | }
274 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/push/MantisWebSocketFrameHandler.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.push;
2 |
3 | import java.util.concurrent.BlockingQueue;
4 | import java.util.concurrent.LinkedBlockingQueue;
5 | import java.util.concurrent.ScheduledExecutorService;
6 | import java.util.concurrent.ScheduledFuture;
7 | import java.util.concurrent.ScheduledThreadPoolExecutor;
8 | import java.util.concurrent.TimeUnit;
9 | import java.util.ArrayList;
10 | import java.util.List;
11 |
12 | import com.netflix.config.DynamicIntProperty;
13 | import com.netflix.spectator.api.Counter;
14 | import com.netflix.zuul.netty.SpectatorUtils;
15 | import io.mantisrx.api.Constants;
16 | import io.mantisrx.api.Util;
17 | import io.mantisrx.shaded.com.google.common.util.concurrent.ThreadFactoryBuilder;
18 | import io.netty.channel.ChannelHandlerContext;
19 | import io.netty.channel.SimpleChannelInboundHandler;
20 | import io.netty.handler.codec.http.websocketx.TextWebSocketFrame;
21 | import io.netty.handler.codec.http.websocketx.WebSocketServerProtocolHandler;
22 | import io.netty.util.ReferenceCountUtil;
23 | import lombok.extern.slf4j.Slf4j;
24 | import rx.Subscription;
25 |
26 | @Slf4j
27 | public class MantisWebSocketFrameHandler extends SimpleChannelInboundHandler {
28 | private final ConnectionBroker connectionBroker;
29 | private final DynamicIntProperty queueCapacity = new DynamicIntProperty("io.mantisrx.api.push.queueCapacity", 1000);
30 | private final DynamicIntProperty writeIntervalMillis = new DynamicIntProperty("io.mantisrx.api.push.writeIntervalMillis", 50);
31 |
32 | private Subscription subscription;
33 | private String uri;
34 | private ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1,
35 | new ThreadFactoryBuilder().setNameFormat("websocket-handler-drainer-%d").build());
36 | private ScheduledFuture drainFuture;
37 |
38 | public MantisWebSocketFrameHandler(ConnectionBroker broker) {
39 | super(true);
40 | this.connectionBroker = broker;
41 | }
42 |
43 | @Override
44 | public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
45 | if (evt.getClass() == WebSocketServerProtocolHandler.HandshakeComplete.class) {
46 | WebSocketServerProtocolHandler.HandshakeComplete complete = (WebSocketServerProtocolHandler.HandshakeComplete) evt;
47 |
48 | uri = complete.requestUri();
49 | final PushConnectionDetails pcd = PushConnectionDetails.from(uri);
50 |
51 | log.info("Request to URI '{}' is a WebSSocket upgrade, removing the SSE handler", uri);
52 | if (ctx.pipeline().get(MantisSSEHandler.class) != null) {
53 | ctx.pipeline().remove(MantisSSEHandler.class);
54 | }
55 |
56 | final String[] tags = Util.getTaglist(uri, pcd.target);
57 | Counter numDroppedBytesCounter = SpectatorUtils.newCounter(Constants.numDroppedBytesCounterName, pcd.target, tags);
58 | Counter numDroppedMessagesCounter = SpectatorUtils.newCounter(Constants.numDroppedMessagesCounterName, pcd.target, tags);
59 | Counter numMessagesCounter = SpectatorUtils.newCounter(Constants.numMessagesCounterName, pcd.target, tags);
60 | Counter numBytesCounter = SpectatorUtils.newCounter(Constants.numBytesCounterName, pcd.target, tags);
61 | Counter drainTriggeredCounter = SpectatorUtils.newCounter(Constants.drainTriggeredCounterName, pcd.target, tags);
62 | Counter numIncomingMessagesCounter = SpectatorUtils.newCounter(Constants.numIncomingMessagesCounterName, pcd.target, tags);
63 |
64 | BlockingQueue queue = new LinkedBlockingQueue<>(queueCapacity.get());
65 |
66 | drainFuture = scheduledExecutorService.scheduleAtFixedRate(() -> {
67 | try {
68 | if (queue.size() > 0 && ctx.channel().isWritable()) {
69 | drainTriggeredCounter.increment();
70 | final List items = new ArrayList<>(queue.size());
71 | synchronized (queue) {
72 | queue.drainTo(items);
73 | }
74 | for (String data : items) {
75 | ctx.write(new TextWebSocketFrame(data));
76 | numMessagesCounter.increment();
77 | numBytesCounter.increment(data.length());
78 | }
79 | ctx.flush();
80 | }
81 | } catch (Exception ex) {
82 | log.error("Error writing to channel", ex);
83 | }
84 | }, writeIntervalMillis.get(), writeIntervalMillis.get(), TimeUnit.MILLISECONDS);
85 |
86 | this.subscription = this.connectionBroker.connect(pcd)
87 | .doOnNext(event -> {
88 | numIncomingMessagesCounter.increment();
89 | if (!Constants.DUMMY_TIMER_DATA.equals(event)) {
90 | boolean offer = false;
91 | synchronized (queue) {
92 | offer = queue.offer(event);
93 | }
94 | if (!offer) {
95 | numDroppedBytesCounter.increment(event.length());
96 | numDroppedMessagesCounter.increment();
97 | }
98 | }
99 | })
100 | .subscribe();
101 | } else {
102 | ReferenceCountUtil.retain(evt);
103 | super.userEventTriggered(ctx, evt);
104 | }
105 | }
106 |
107 | @Override
108 | public void channelUnregistered(ChannelHandlerContext ctx) throws Exception {
109 | log.info("Channel {} is unregistered. URI: {}", ctx.channel(), uri);
110 | unsubscribeIfSubscribed();
111 | super.channelUnregistered(ctx);
112 | }
113 |
114 | @Override
115 | public void channelInactive(ChannelHandlerContext ctx) throws Exception {
116 | log.info("Channel {} is inactive. URI: {}", ctx.channel(), uri);
117 | unsubscribeIfSubscribed();
118 | super.channelInactive(ctx);
119 | }
120 |
121 | @Override
122 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
123 | log.warn("Exception caught by channel {}. URI: {}", ctx.channel(), uri, cause);
124 | unsubscribeIfSubscribed();
125 | // This is the tail of handlers. We should close the channel between the server and the client,
126 | // essentially causing the client to disconnect and terminate.
127 | ctx.close();
128 | }
129 |
130 | @Override
131 | protected void channelRead0(ChannelHandlerContext ctx, TextWebSocketFrame msg) {
132 | // No op.
133 | }
134 |
135 | /** Unsubscribe if it's subscribed. */
136 | private void unsubscribeIfSubscribed() {
137 | if (subscription != null && !subscription.isUnsubscribed()) {
138 | log.info("WebSocket unsubscribing subscription with URI: {}", uri);
139 | subscription.unsubscribe();
140 | }
141 | if (drainFuture != null) {
142 | drainFuture.cancel(false);
143 | }
144 | if (scheduledExecutorService != null) {
145 | scheduledExecutorService.shutdown();
146 | }
147 | }
148 | }
149 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/push/PushConnectionDetails.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.push;
2 |
3 | import io.mantisrx.runtime.parameter.SinkParameter;
4 | import io.mantisrx.runtime.parameter.SinkParameters;
5 | import io.netty.handler.codec.http.QueryStringDecoder;
6 | import io.vavr.collection.List;
7 | import io.vavr.control.Try;
8 | import lombok.Value;
9 |
10 | import java.util.stream.Collectors;
11 |
12 | public @Value class PushConnectionDetails {
13 |
14 | public enum TARGET_TYPE {
15 | CONNECT_BY_NAME,
16 | CONNECT_BY_ID,
17 | JOB_STATUS,
18 | JOB_SCHEDULING_INFO,
19 | JOB_CLUSTER_DISCOVERY,
20 | METRICS
21 | }
22 |
23 | private final String uri;
24 | public final String target;
25 | public final TARGET_TYPE type;
26 | public final List regions;
27 |
28 |
29 | /**
30 | * Determines the connection type for a given push connection.
31 | *
32 | * @param uri Request URI as returned by Netty's requestUri() methods. Expects leading slash.
33 | * @return The CONNECTION_TYPE requested by the URI.
34 | */
35 | public static TARGET_TYPE determineTargetType(final String uri) {
36 | if (uri.startsWith("/jobconnectbyname") || uri.startsWith("/api/v1/jobconnectbyname")) {
37 | return TARGET_TYPE.CONNECT_BY_NAME;
38 | } else if (uri.startsWith("/jobconnectbyid") || uri.startsWith("/api/v1/jobconnectbyid")) {
39 | return TARGET_TYPE.CONNECT_BY_ID;
40 | } else if (uri.startsWith("/jobstatus/") || uri.startsWith("/api/v1/jobstatus/")) {
41 | return TARGET_TYPE.JOB_STATUS;
42 | } else if (uri.startsWith("/api/v1/jobs/schedulingInfo/")) {
43 | return TARGET_TYPE.JOB_SCHEDULING_INFO;
44 | } else if (uri.startsWith("/jobClusters/discoveryInfoStream/")) {
45 | return TARGET_TYPE.JOB_CLUSTER_DISCOVERY;
46 | } else if (uri.startsWith("/api/v1/metrics/")) {
47 | return TARGET_TYPE.METRICS;
48 | } else {
49 | throw new IllegalArgumentException("Unable to determine push connection type from URI: " + uri);
50 | }
51 | }
52 |
53 | /**
54 | * Determines the target for a push connection request. Typically a job name or id.
55 | *
56 | * @param uri Request URI as returned by Netty's requestUri() methods. Expects leading slash.
57 | * @return The target requested by the URI.
58 | */
59 | public static String determineTarget(final String uri) {
60 | String sanitized = uri.replaceFirst("^/(api/v1/)?(jobconnectbyid|jobconnectbyname|jobstatus|jobs/schedulingInfo|jobClusters/discoveryInfoStream|metrics)/", "");
61 | QueryStringDecoder queryStringDecoder = new QueryStringDecoder(sanitized);
62 | return queryStringDecoder.path();
63 | }
64 |
65 | //
66 | // Computed Properties
67 | //
68 |
69 | public SinkParameters getSinkparameters() {
70 | SinkParameters.Builder builder = new SinkParameters.Builder();
71 | QueryStringDecoder queryStringDecoder = new QueryStringDecoder(uri);
72 |
73 | builder.parameters(queryStringDecoder
74 | .parameters()
75 | .entrySet()
76 | .stream()
77 | .flatMap(entry -> entry.getValue()
78 | .stream()
79 | .map(val -> Try.of(() -> new SinkParameter(entry.getKey(), val)))
80 | .filter(Try::isSuccess)
81 | .map(Try::get))
82 | .collect(Collectors.toList())
83 | .toArray(new SinkParameter[]{}));
84 |
85 | return builder.build();
86 | }
87 |
88 | //
89 | // Static Factories
90 | //
91 |
92 | public static PushConnectionDetails from(String uri) {
93 | return from(uri, List.empty());
94 | }
95 |
96 | public static PushConnectionDetails from(String uri, List regions) {
97 | return new PushConnectionDetails(uri, determineTarget(uri), determineTargetType(uri), regions);
98 | }
99 |
100 | public static PushConnectionDetails from(String uri, java.util.List regions) {
101 | return new PushConnectionDetails(uri, determineTarget(uri), determineTargetType(uri), List.ofAll(regions));
102 | }
103 | }
104 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/services/AppStreamDiscoveryService.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2018 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package io.mantisrx.api.services;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.netflix.spectator.api.Counter;
20 | import com.netflix.zuul.netty.SpectatorUtils;
21 | import io.mantisrx.api.proto.AppDiscoveryMap;
22 | import io.mantisrx.client.MantisClient;
23 | import io.mantisrx.discovery.proto.AppJobClustersMap;
24 | import io.mantisrx.server.core.JobSchedulingInfo;
25 | import io.vavr.control.Either;
26 | import io.vavr.control.Option;
27 | import java.io.IOException;
28 | import java.util.List;
29 | import java.util.concurrent.TimeUnit;
30 | import lombok.extern.slf4j.Slf4j;
31 | import rx.Observable;
32 | import rx.Scheduler;
33 |
34 | @Slf4j
35 | public class AppStreamDiscoveryService {
36 |
37 | private final MantisClient mantisClient;
38 | private final Scheduler scheduler;
39 |
40 | private final AppStreamStore appStreamStore;
41 |
42 | public AppStreamDiscoveryService(
43 | MantisClient mantisClient,
44 | Scheduler scheduler,
45 | AppStreamStore appStreamStore) {
46 | Preconditions.checkArgument(mantisClient != null);
47 | Preconditions.checkArgument(appStreamStore != null);
48 | Preconditions.checkArgument(scheduler != null);
49 | this.mantisClient = mantisClient;
50 | this.scheduler = scheduler;
51 | this.appStreamStore = appStreamStore;
52 |
53 | Counter appJobClusterMappingNullCount = SpectatorUtils.newCounter(
54 | "appJobClusterMappingNull", "mantisapi");
55 | Counter appJobClusterMappingRequestCount = SpectatorUtils.newCounter(
56 | "appJobClusterMappingRequest", "mantisapi", "app", "unknown");
57 | Counter appJobClusterMappingFailCount = SpectatorUtils.newCounter(
58 | "appJobClusterMappingFail", "mantisapi");
59 | }
60 |
61 |
62 | public Either getAppDiscoveryMap(List appNames) {
63 | try {
64 |
65 | AppJobClustersMap appJobClusters = getAppJobClustersMap(appNames);
66 |
67 | //
68 | // Lookup discovery info per stream and build mapping
69 | //
70 |
71 | AppDiscoveryMap adm = new AppDiscoveryMap(appJobClusters.getVersion(), appJobClusters.getTimestamp());
72 |
73 | for (String app : appJobClusters.getMappings().keySet()) {
74 | for (String stream : appJobClusters.getMappings().get(app).keySet()) {
75 | String jobCluster = appJobClusters.getMappings().get(app).get(stream);
76 | Option jobSchedulingInfo = getJobDiscoveryInfo(jobCluster);
77 | jobSchedulingInfo.map(jsi -> {
78 | adm.addMapping(app, stream, jsi);
79 | return jsi;
80 | });
81 | }
82 | }
83 | return Either.right(adm);
84 | } catch (Exception ex) {
85 | log.error(ex.getMessage());
86 | return Either.left(ex.getMessage());
87 | }
88 | }
89 |
90 | public AppJobClustersMap getAppJobClustersMap(List appNames) throws IOException {
91 | return appStreamStore.getJobClusterMappings(appNames);
92 | }
93 |
94 | private Option getJobDiscoveryInfo(String jobCluster) {
95 | JobDiscoveryService jdim = JobDiscoveryService.getInstance(mantisClient, scheduler);
96 | return jdim
97 | .jobDiscoveryInfoStream(jdim.key(JobDiscoveryService.LookupType.JOB_CLUSTER, jobCluster))
98 | .map(Option::of)
99 | .take(1)
100 | .timeout(2, TimeUnit.SECONDS, Observable.just(Option.none()))
101 | .doOnError((t) -> {
102 | log.warn("Timed out looking up job discovery info for cluster: " + jobCluster + ".");
103 | })
104 | .subscribeOn(scheduler)
105 | .observeOn(scheduler)
106 | .toSingle()
107 | .toBlocking()
108 | .value();
109 | }
110 | }
111 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/services/AppStreamStore.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.services;
2 |
3 | import com.google.common.collect.ImmutableList;
4 | import io.mantisrx.discovery.proto.AppJobClustersMap;
5 | import java.io.IOException;
6 | import java.util.Collection;
7 |
8 | /**
9 | * Interface to get streams associated with a given app or set of apps
10 | */
11 | public interface AppStreamStore {
12 | default AppJobClustersMap getJobClusterMappings(String app) throws IOException {
13 | return getJobClusterMappings(ImmutableList.of(app));
14 | }
15 |
16 | AppJobClustersMap getJobClusterMappings(Collection apps) throws IOException;
17 | }
18 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/services/ConfigurationBasedAppStreamStore.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.services;
2 |
3 | import com.netflix.spectator.api.Counter;
4 | import com.netflix.zuul.netty.SpectatorUtils;
5 | import io.mantisrx.common.JsonSerializer;
6 | import io.mantisrx.discovery.proto.AppJobClustersMap;
7 | import io.mantisrx.shaded.org.apache.curator.framework.listen.Listenable;
8 | import java.io.IOException;
9 | import java.util.ArrayList;
10 | import java.util.Collection;
11 | import java.util.concurrent.atomic.AtomicReference;
12 | import java.util.function.Supplier;
13 | import javax.annotation.Nullable;
14 | import lombok.extern.slf4j.Slf4j;
15 |
16 | @SuppressWarnings("unused")
17 | @Slf4j
18 | public class ConfigurationBasedAppStreamStore implements AppStreamStore {
19 |
20 | private final JsonSerializer jsonSerializer;
21 |
22 | private final AtomicReference appJobClusterMappings = new AtomicReference<>();
23 |
24 | private final Counter appJobClusterMappingNullCount;
25 | private final Counter appJobClusterMappingFailCount;
26 | private final Counter appJobClusterMappingRequestCount;
27 |
28 | public ConfigurationBasedAppStreamStore(ConfigSource configSource) {
29 | configSource.getListenable()
30 | .addListener((newConfig) -> updateAppJobClustersMapping(newConfig));
31 | this.jsonSerializer = new JsonSerializer();
32 | updateAppJobClustersMapping(configSource.get());
33 |
34 | this.appJobClusterMappingNullCount = SpectatorUtils.newCounter(
35 | "appJobClusterMappingNull", "mantisapi");
36 | this.appJobClusterMappingRequestCount = SpectatorUtils.newCounter(
37 | "appJobClusterMappingRequest", "mantisapi", "app", "unknown");
38 | this.appJobClusterMappingFailCount = SpectatorUtils.newCounter(
39 | "appJobClusterMappingFail", "mantisapi");
40 | }
41 |
42 | @Override
43 | public AppJobClustersMap getJobClusterMappings(Collection apps) throws IOException {
44 | return getAppJobClustersMap(apps, this.appJobClusterMappings.get());
45 | }
46 |
47 | private AppJobClustersMap getAppJobClustersMap(Collection appNames,
48 | @Nullable AppJobClustersMap appJobClustersMap) throws IOException {
49 |
50 | if (appJobClustersMap != null) {
51 | final AppJobClustersMap appJobClusters;
52 | if (appNames.size() > 0) {
53 | appJobClusters = appJobClustersMap.getFilteredAppJobClustersMap(new ArrayList<>(appNames));
54 | } else {
55 | appJobClusterMappingRequestCount.increment();
56 | appJobClusters = appJobClustersMap;
57 | }
58 | return appJobClusters;
59 | } else {
60 | appJobClusterMappingNullCount.increment();
61 | throw new IOException("AppJobClustersMap is null");
62 | }
63 | }
64 |
65 | private void updateAppJobClustersMapping(String appJobClusterStr) {
66 | try {
67 | AppJobClustersMap appJobClustersMap =
68 | jsonSerializer.fromJSON(appJobClusterStr, AppJobClustersMap.class);
69 | log.info("appJobClustersMap updated to {}", appJobClustersMap);
70 | appJobClusterMappings.set(appJobClustersMap);
71 | } catch (Exception ioe) {
72 | log.error("failed to update appJobClustersMap on Property update {}", appJobClusterStr, ioe);
73 | appJobClusterMappingFailCount.increment();
74 | }
75 | }
76 |
77 | public interface ConfigSource extends Supplier {
78 |
79 | Listenable getListenable();
80 | }
81 |
82 | public interface ConfigurationChangeListener {
83 |
84 | void onConfigChange(String config);
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/services/artifacts/ArtifactManager.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.services.artifacts;
2 | /*
3 | * Copyright 2019 Netflix, Inc.
4 | *
5 | * Licensed under the Apache License, Version 2.0 (the "License");
6 | * you may not use this file except in compliance with the License.
7 | * You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | import io.mantisrx.api.proto.Artifact;
19 |
20 | import java.util.List;
21 | import java.util.Optional;
22 |
23 | public interface ArtifactManager {
24 |
25 | List getArtifacts();
26 |
27 | Optional getArtifact(String name);
28 |
29 | void deleteArtifact(String name);
30 |
31 | void putArtifact(Artifact artifact);
32 | }
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/services/artifacts/InMemoryArtifactManager.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.services.artifacts;
2 | import java.util.HashMap;
3 | import java.util.List;
4 | import java.util.Map;
5 | import java.util.Optional;
6 | import java.util.stream.Collectors;
7 |
8 | import io.mantisrx.api.proto.Artifact;
9 |
10 |
11 | public class InMemoryArtifactManager implements ArtifactManager {
12 | private Map artifacts = new HashMap<>();
13 |
14 | @Override
15 | public List getArtifacts() {
16 | return artifacts
17 | .values()
18 | .stream()
19 | .map(Artifact::getFileName)
20 | .collect(Collectors.toList());
21 | }
22 |
23 | @Override
24 | public Optional getArtifact(String name) {
25 | return artifacts
26 | .values()
27 | .stream()
28 | .filter(artifact -> artifact.getFileName().equals(name))
29 | .findFirst();
30 | }
31 |
32 | @Override
33 | public void deleteArtifact(String name) {
34 | this.artifacts.remove(name);
35 | }
36 |
37 | @Override
38 | public void putArtifact(Artifact artifact) {
39 | this.artifacts.put(artifact.getFileName(), artifact);
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/tunnel/MantisCrossRegionalClient.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package io.mantisrx.api.tunnel;
18 |
19 | import io.netty.buffer.ByteBuf;
20 | import mantis.io.reactivex.netty.protocol.http.client.HttpClient;
21 | import mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent;
22 |
23 | public interface MantisCrossRegionalClient {
24 | HttpClient getSecureSseClient(String region);
25 |
26 | HttpClient getSecureRestClient(String region);
27 | }
28 |
29 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/tunnel/NoOpCrossRegionalClient.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.tunnel;
2 |
3 | import io.netty.buffer.ByteBuf;
4 | import mantis.io.reactivex.netty.protocol.http.client.HttpClient;
5 | import mantis.io.reactivex.netty.protocol.http.sse.ServerSentEvent;
6 |
7 | public class NoOpCrossRegionalClient implements MantisCrossRegionalClient {
8 | @Override
9 | public HttpClient getSecureSseClient(String region) {
10 | throw new UnsupportedOperationException();
11 | }
12 |
13 | @Override
14 | public HttpClient getSecureRestClient(String region) {
15 | throw new UnsupportedOperationException();
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/src/main/java/io/mantisrx/api/tunnel/RegionData.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2020 Netflix, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package io.mantisrx.api.tunnel;
18 |
19 | import lombok.Value;
20 |
21 | public @Value class RegionData {
22 | private final String region;
23 | private final boolean success;
24 | private final String data;
25 | private final int responseCode;
26 | }
27 |
--------------------------------------------------------------------------------
/src/main/resources/api-docker.properties:
--------------------------------------------------------------------------------
1 | region=us-east-1
2 | mantis.localmode=false
3 |
4 | eureka.registration.enabled=false
5 | eureka.shouldFetchRegistry=false
6 |
7 | #Zookeeper is necessary for master discovery
8 |
9 | mantis.zookeeper.connectString=zookeeper:2181
10 | mantis.zookeeper.root=/mantis/master/nmahilani
11 | mantis.zookeeper.leader.announcement.path=/leader
12 |
13 | default.nfzookeeper.session-timeout-ms=120000
14 |
15 | api.ribbon.NIWSServerListClassName=io.mantisrx.api.MantisConfigurationBasedServerList
16 | zuul.filters.packages=io.mantisrx.api.filters
17 |
18 | # Controls how long inactive websocket sessions take to timeout
19 | mantisapi.connection.inactive.timeout.secs=300
20 |
21 | mantisapi.submit.instanceLimit=100
22 |
23 | mantis.sse.disablePingFiltering=true
24 |
25 | mantisapi.artifact.disk.cache.location=/apps/nfmantisapi/mantisArtifacts/
26 | mantisapi.artifact.disk.cache.enabled=true
27 |
28 | mreAppJobClusterMap={"version": "1", "timestamp": 12345, "mappings": {"__default__": {"requestEventStream": "SharedPushRequestEventSource","sentryEventStream": "SentryLogEventSource","__default__": "SharedMrePublishEventSource"},"customApp": {"logEventStream": "CustomAppEventSource","sentryEventStream": "CustomAppSentryLogSource"}}}
29 |
--------------------------------------------------------------------------------
/src/main/resources/api-local.properties:
--------------------------------------------------------------------------------
1 | region=us-east-1
2 | mantis.localmode=true
3 |
4 | eureka.registration.enabled=false
5 | eureka.shouldFetchRegistry=false
6 |
7 | #Zookeeper is necessary for master discovery
8 |
9 | mantis.zookeeper.connectString=localhost:8100
10 | mantis.zookeeper.root=/mantis/master/nmahilani
11 | mantis.zookeeper.leader.announcement.path=/leader
12 | mesos.master.location=zk://zookeeper:2181/mantis/mesos/nmahilani
13 |
14 | default.nfzookeeper.session-timeout-ms=120000
15 |
16 | api.ribbon.NIWSServerListClassName=io.mantisrx.api.MantisConfigurationBasedServerList
17 | zuul.filters.packages=io.mantisrx.api.filters
18 |
19 | # Controls how long inactive websocket sessions take to timeout
20 | mantisapi.connection.inactive.timeout.secs=300
21 |
22 | mantisapi.submit.instanceLimit=100
23 |
24 | mantis.sse.disablePingFiltering=true
25 |
26 | mantisapi.artifact.disk.cache.location=/apps/nfmantisapi/mantisArtifacts/
27 | mantisapi.artifact.disk.cache.enabled=true
28 |
29 | mreAppJobClusterMap={"version": "1", "timestamp": 12345, "mappings": {"__default__": {"requestEventStream": "SharedPushRequestEventSource","sentryEventStream": "SentryLogEventSource","__default__": "SharedPushEventSource"},"customApp": {"logEventStream": "CustomAppEventSource","sentryEventStream": "CustomAppSentryLogSource"}}}
30 |
--------------------------------------------------------------------------------
/src/main/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | #
2 | # Copyright 2015 Netflix, Inc.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | #
16 |
17 | log4j.rootLogger=INFO,stdout
18 |
19 | # raise above INFO to disable access logger
20 | log4j.logger.ACCESS=INFO
21 |
22 | # can be set to INFO for netty wire logging
23 | log4j.logger.zuul.server.nettylog=WARN
24 | log4j.logger.zuul.origin.nettylog=WARN
25 | log4j.logger.zuul.api.nettylog=WARN
26 |
27 | log4j.logger.com.netflix.loadbalancer=WARN
28 | log4j.logger.com.netflix.config=WARN
29 |
30 | # stdout
31 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender
32 | log4j.appender.stdout.layout.ConversionPattern=%d %-5p %c [%t] %m%n
33 |
34 | # filter out repeating lines for Rx
35 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
36 |
37 | # async appender
38 | batcher.com.netflix.logging.AsyncAppender.stdout.waitTimeinMillis=120000
39 | log4j.logger.asyncAppenders=INFO,stdout
40 |
--------------------------------------------------------------------------------
/src/test/java/io/mantisrx/api/UtilTest.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api;
2 |
3 | import org.junit.Test;
4 |
5 | import static org.junit.Assert.assertArrayEquals;
6 |
7 | public class UtilTest {
8 |
9 | @Test
10 | public void testGetTagList() {
11 | String[] tags = Util.getTaglist("/jobconnectbyname/rx-sps-tracker?clientId=testClientId", "testTargetId", "us-east-1");
12 | assertArrayEquals(new String[]{
13 | "clientId", "testClientId",
14 | "SessionId", "testTargetId",
15 | "urlPath", "/jobconnectbyname/rx-sps-tracker",
16 | "region", "us-east-1"}, tags);
17 |
18 | tags = Util.getTaglist("/jobconnectbyname/rx-sps-tracker?clientId=testClientId&MantisApiTag=tag1:value1", "testTargetId", "us-east-1");
19 | assertArrayEquals(new String[]{
20 | "tag1", "value1",
21 | "clientId", "testClientId",
22 | "SessionId", "testTargetId",
23 | "urlPath", "/jobconnectbyname/rx-sps-tracker",
24 | "region", "us-east-1"}, tags);
25 |
26 | tags = Util.getTaglist("/jobconnectbyname/rx-sps-tracker?clientId=testClientId&MantisApiTag=tag1:value1&MantisApiTag=clientId:testClientId2", "testTargetId", "us-east-1");
27 | assertArrayEquals(new String[]{
28 | "tag1", "value1",
29 | "clientId", "testClientId2",
30 | "SessionId", "testTargetId",
31 | "urlPath", "/jobconnectbyname/rx-sps-tracker",
32 | "region", "us-east-1"}, tags);
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/src/test/java/io/mantisrx/api/tunnel/CrossRegionHandlerTest.java:
--------------------------------------------------------------------------------
1 | package io.mantisrx.api.tunnel;
2 |
3 | import com.google.common.collect.ImmutableList;
4 | import io.mantisrx.api.push.ConnectionBroker;
5 | import junit.framework.TestCase;
6 | import org.junit.Test;
7 | import rx.Scheduler;
8 |
9 | import static org.mockito.Mockito.doReturn;
10 | import static org.mockito.Mockito.mock;
11 | import static org.mockito.Mockito.spy;
12 |
13 | public class CrossRegionHandlerTest extends TestCase {
14 |
15 | @Test
16 | public void testParseUriRegion() {
17 |
18 | CrossRegionHandler regionHandler = spy(new CrossRegionHandler(ImmutableList.of(), mock(MantisCrossRegionalClient.class), mock(ConnectionBroker.class), mock(Scheduler.class)));
19 | doReturn(ImmutableList.of("us-east-1", "eu-west-1")).when(regionHandler).getTunnelRegions();
20 |
21 | assertEquals(ImmutableList.of("us-east-1"), regionHandler.parseRegionsInUri("/region/us-east-1/foobar"));
22 | assertEquals(ImmutableList.of("us-east-2"), regionHandler.parseRegionsInUri("/region/us-east-2/foobar"));
23 | assertEquals(ImmutableList.of("us-east-1", "eu-west-1"), regionHandler.parseRegionsInUri("/region/all/foobar"));
24 | assertEquals(ImmutableList.of("us-east-1", "eu-west-1"), regionHandler.parseRegionsInUri("/region/ALL/foobar"));
25 |
26 | doReturn(ImmutableList.of("us-east-1", "eu-west-1", "us-west-2")).when(regionHandler).getTunnelRegions();
27 | assertEquals(ImmutableList.of("us-east-1", "eu-west-1", "us-west-2"), regionHandler.parseRegionsInUri("/region/ALL/foobar"));
28 |
29 | assertEquals(ImmutableList.of("us-east-1", "us-east-2"), regionHandler.parseRegionsInUri("/region/us-east-1,us-east-2/foobar"));
30 | assertEquals(ImmutableList.of("us-east-1", "us-west-2"), regionHandler.parseRegionsInUri("/region/us-east-1,us-west-2/foobar"));
31 | }
32 | }
33 |
--------------------------------------------------------------------------------