├── .credo.exs ├── .env ├── .formatter.exs ├── .github ├── FUNDING.yml ├── renovate.json5 └── workflows │ ├── code_quality.yaml │ └── integration.yaml ├── .gitignore ├── .mise.toml ├── .tool-versions ├── CHANGELOG.md ├── LICENSE.md ├── README.md ├── config └── config.exs ├── lib ├── flame_k8s_backend.ex └── flame_k8s_backend │ ├── http.ex │ ├── k8s_client.ex │ └── runner_pod_template.ex ├── mix.exs ├── mix.lock ├── test ├── flame_k8s_backend │ └── runner_pod_template_test.exs ├── integration │ ├── Dockerfile │ ├── integration_test.exs │ └── manifest.yaml └── test_helper.exs └── test_support ├── integration_test_runner.ex └── pods.ex /.credo.exs: -------------------------------------------------------------------------------- 1 | # This file contains the configuration for Credo and you are probably reading 2 | # this after creating it with `mix credo.gen.config`. 3 | # 4 | # If you find anything wrong or unclear in this file, please report an 5 | # issue on GitHub: https://github.com/rrrene/credo/issues 6 | # 7 | %{ 8 | # 9 | # You can have as many configs as you like in the `configs:` field. 10 | configs: [ 11 | %{ 12 | # 13 | # Run any config using `mix credo -C `. If no config name is given 14 | # "default" is used. 15 | # 16 | name: "default", 17 | # 18 | # These are the files included in the analysis: 19 | files: %{ 20 | # 21 | # You can give explicit globs or simply directories. 22 | # In the latter case `**/*.{ex,exs}` will be used. 23 | # 24 | included: [ 25 | "lib/", 26 | "src/", 27 | "test/", 28 | "web/", 29 | "apps/*/lib/", 30 | "apps/*/src/", 31 | "apps/*/test/", 32 | "apps/*/web/" 33 | ], 34 | excluded: [~r"/_build/", ~r"/deps/", ~r"/node_modules/"] 35 | }, 36 | # 37 | # Load and configure plugins here: 38 | # 39 | plugins: [], 40 | # 41 | # If you create your own checks, you must specify the source files for 42 | # them here, so they can be loaded by Credo before running the analysis. 43 | # 44 | requires: [], 45 | # 46 | # If you want to enforce a style guide and need a more traditional linting 47 | # experience, you can change `strict` to `true` below: 48 | # 49 | strict: false, 50 | # 51 | # To modify the timeout for parsing files, change this value: 52 | # 53 | parse_timeout: 5000, 54 | # 55 | # If you want to use uncolored output by default, you can change `color` 56 | # to `false` below: 57 | # 58 | color: true, 59 | # 60 | # You can customize the parameters of any check by adding a second element 61 | # to the tuple. 62 | # 63 | # To disable a check put `false` as second element: 64 | # 65 | # {Credo.Check.Design.DuplicatedCode, false} 66 | # 67 | checks: %{ 68 | enabled: [ 69 | # 70 | ## Consistency Checks 71 | # 72 | {Credo.Check.Consistency.ExceptionNames, []}, 73 | {Credo.Check.Consistency.LineEndings, []}, 74 | {Credo.Check.Consistency.ParameterPatternMatching, []}, 75 | {Credo.Check.Consistency.SpaceAroundOperators, []}, 76 | {Credo.Check.Consistency.SpaceInParentheses, []}, 77 | {Credo.Check.Consistency.TabsOrSpaces, []}, 78 | 79 | # 80 | ## Design Checks 81 | # 82 | # You can customize the priority of any check 83 | # Priority values are: `low, normal, high, higher` 84 | # 85 | {Credo.Check.Design.AliasUsage, 86 | [priority: :low, if_nested_deeper_than: 2, if_called_more_often_than: 0]}, 87 | {Credo.Check.Design.TagFIXME, []}, 88 | # You can also customize the exit_status of each check. 89 | # If you don't want TODO comments to cause `mix credo` to fail, just 90 | # set this value to 0 (zero). 91 | # 92 | {Credo.Check.Design.TagTODO, [exit_status: 2]}, 93 | 94 | # 95 | ## Readability Checks 96 | # 97 | {Credo.Check.Readability.AliasOrder, []}, 98 | {Credo.Check.Readability.FunctionNames, []}, 99 | {Credo.Check.Readability.LargeNumbers, []}, 100 | {Credo.Check.Readability.MaxLineLength, [priority: :low, max_length: 120]}, 101 | {Credo.Check.Readability.ModuleAttributeNames, []}, 102 | {Credo.Check.Readability.ModuleDoc, []}, 103 | {Credo.Check.Readability.ModuleNames, []}, 104 | {Credo.Check.Readability.ParenthesesInCondition, []}, 105 | {Credo.Check.Readability.ParenthesesOnZeroArityDefs, [parens: true]}, 106 | {Credo.Check.Readability.PipeIntoAnonymousFunctions, []}, 107 | {Credo.Check.Readability.PredicateFunctionNames, []}, 108 | {Credo.Check.Readability.PreferImplicitTry, []}, 109 | {Credo.Check.Readability.RedundantBlankLines, []}, 110 | {Credo.Check.Readability.Semicolons, []}, 111 | {Credo.Check.Readability.SpaceAfterCommas, []}, 112 | {Credo.Check.Readability.StringSigils, []}, 113 | {Credo.Check.Readability.TrailingBlankLine, []}, 114 | {Credo.Check.Readability.TrailingWhiteSpace, []}, 115 | {Credo.Check.Readability.UnnecessaryAliasExpansion, []}, 116 | {Credo.Check.Readability.VariableNames, []}, 117 | {Credo.Check.Readability.WithSingleClause, []}, 118 | 119 | # 120 | ## Refactoring Opportunities 121 | # 122 | {Credo.Check.Refactor.Apply, []}, 123 | {Credo.Check.Refactor.CondStatements, []}, 124 | {Credo.Check.Refactor.CyclomaticComplexity, []}, 125 | {Credo.Check.Refactor.FilterCount, []}, 126 | {Credo.Check.Refactor.FilterFilter, []}, 127 | {Credo.Check.Refactor.FunctionArity, []}, 128 | {Credo.Check.Refactor.LongQuoteBlocks, []}, 129 | {Credo.Check.Refactor.MapJoin, []}, 130 | {Credo.Check.Refactor.MatchInCondition, []}, 131 | {Credo.Check.Refactor.NegatedConditionsInUnless, []}, 132 | {Credo.Check.Refactor.NegatedConditionsWithElse, []}, 133 | {Credo.Check.Refactor.RedundantWithClauseResult, []}, 134 | {Credo.Check.Refactor.RejectReject, []}, 135 | {Credo.Check.Refactor.UnlessWithElse, []}, 136 | {Credo.Check.Refactor.WithClauses, []}, 137 | 138 | # 139 | ## Warnings 140 | # 141 | {Credo.Check.Warning.ApplicationConfigInModuleAttribute, []}, 142 | {Credo.Check.Warning.BoolOperationOnSameValues, []}, 143 | {Credo.Check.Warning.Dbg, []}, 144 | {Credo.Check.Warning.ExpensiveEmptyEnumCheck, []}, 145 | {Credo.Check.Warning.IExPry, []}, 146 | {Credo.Check.Warning.IoInspect, []}, 147 | {Credo.Check.Warning.MissedMetadataKeyInLoggerConfig, []}, 148 | {Credo.Check.Warning.OperationOnSameValues, []}, 149 | {Credo.Check.Warning.OperationWithConstantResult, []}, 150 | {Credo.Check.Warning.RaiseInsideRescue, []}, 151 | {Credo.Check.Warning.SpecWithStruct, []}, 152 | {Credo.Check.Warning.UnsafeExec, []}, 153 | {Credo.Check.Warning.UnusedEnumOperation, []}, 154 | {Credo.Check.Warning.UnusedFileOperation, []}, 155 | {Credo.Check.Warning.UnusedKeywordOperation, []}, 156 | {Credo.Check.Warning.UnusedListOperation, []}, 157 | {Credo.Check.Warning.UnusedPathOperation, []}, 158 | {Credo.Check.Warning.UnusedRegexOperation, []}, 159 | {Credo.Check.Warning.UnusedStringOperation, []}, 160 | {Credo.Check.Warning.UnusedTupleOperation, []}, 161 | {Credo.Check.Warning.WrongTestFileExtension, []} 162 | ], 163 | disabled: [ 164 | # 165 | # Checks scheduled for next check update (opt-in for now, just replace `false` with `[]`) 166 | 167 | # 168 | # Controversial and experimental checks (opt-in, just move the check to `:enabled` 169 | # and be sure to use `mix credo --strict` to see low priority checks) 170 | # 171 | {Credo.Check.Consistency.MultiAliasImportRequireUse, []}, 172 | {Credo.Check.Consistency.UnusedVariableNames, []}, 173 | {Credo.Check.Design.DuplicatedCode, []}, 174 | {Credo.Check.Design.SkipTestWithoutComment, []}, 175 | {Credo.Check.Readability.AliasAs, []}, 176 | {Credo.Check.Readability.BlockPipe, []}, 177 | {Credo.Check.Readability.ImplTrue, []}, 178 | {Credo.Check.Readability.MultiAlias, []}, 179 | {Credo.Check.Readability.NestedFunctionCalls, []}, 180 | {Credo.Check.Readability.OneArityFunctionInPipe, []}, 181 | {Credo.Check.Readability.OnePipePerLine, []}, 182 | {Credo.Check.Readability.SeparateAliasRequire, []}, 183 | {Credo.Check.Readability.SingleFunctionToBlockPipe, []}, 184 | {Credo.Check.Readability.SinglePipe, []}, 185 | {Credo.Check.Readability.Specs, []}, 186 | {Credo.Check.Readability.StrictModuleLayout, []}, 187 | {Credo.Check.Readability.WithCustomTaggedTuple, []}, 188 | {Credo.Check.Refactor.ABCSize, []}, 189 | {Credo.Check.Refactor.AppendSingleItem, []}, 190 | {Credo.Check.Refactor.DoubleBooleanNegation, []}, 191 | {Credo.Check.Refactor.FilterReject, []}, 192 | {Credo.Check.Refactor.IoPuts, []}, 193 | {Credo.Check.Refactor.MapMap, []}, 194 | {Credo.Check.Refactor.ModuleDependencies, []}, 195 | {Credo.Check.Refactor.NegatedIsNil, []}, 196 | {Credo.Check.Refactor.Nesting, []}, 197 | {Credo.Check.Refactor.PassAsyncInTestCases, []}, 198 | {Credo.Check.Refactor.PipeChainStart, []}, 199 | {Credo.Check.Refactor.RejectFilter, []}, 200 | {Credo.Check.Refactor.VariableRebinding, []}, 201 | {Credo.Check.Warning.LazyLogging, []}, 202 | {Credo.Check.Warning.LeakyEnvironment, []}, 203 | {Credo.Check.Warning.MapGetUnsafePass, []}, 204 | {Credo.Check.Warning.MixEnv, []}, 205 | {Credo.Check.Warning.UnsafeToAtom, []} 206 | 207 | # {Credo.Check.Refactor.MapInto, []}, 208 | 209 | # 210 | # Custom checks can be created using `mix credo.gen.check`. 211 | # 212 | ] 213 | } 214 | } 215 | ] 216 | } 217 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | ERL_AFLAGS = "-kernel shell_history enabled" -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test,test_support}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [mruoss] 4 | -------------------------------------------------------------------------------- /.github/renovate.json5: -------------------------------------------------------------------------------- 1 | { 2 | extends: [ 3 | 'config:recommended', 4 | ':automergeMinor', 5 | ':automergePr', 6 | ':label(renovate-update)', 7 | ':rebaseStalePrs', 8 | ':prConcurrentLimit10', 9 | ':maintainLockFilesWeekly' 10 | ], 11 | } 12 | -------------------------------------------------------------------------------- /.github/workflows/code_quality.yaml: -------------------------------------------------------------------------------- 1 | name: Code Quality 2 | on: 3 | push: 4 | branches: [main] 5 | pull_request: 6 | branches: [main] 7 | 8 | env: 9 | MIX_ENV: test 10 | 11 | jobs: 12 | code-quality: 13 | name: Code Quality 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 17 | 18 | - name: Setup elixir 19 | id: beam 20 | uses: erlef/setup-beam@v1 21 | with: 22 | version-file: .tool-versions 23 | version-type: strict 24 | install-rebar: true 25 | install-hex: true 26 | 27 | - name: Retrieve Build Cache 28 | uses: actions/cache@v4 29 | id: build-folder-cache 30 | with: 31 | path: _build/test 32 | key: ${{ runner.os }}-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-build-test-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} 33 | 34 | - name: Retrieve Mix Dependencies Cache 35 | uses: actions/cache@v4 36 | id: mix-cache 37 | with: 38 | path: deps 39 | key: ${{ runner.os }}-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} 40 | 41 | - name: Install Mix Dependencies 42 | run: mix deps.get 43 | 44 | - name: Check Formatting 45 | run: mix format --check-formatted 46 | 47 | - name: Compile 48 | run: MIX_ENV=test mix compile --warnings-as-errors 49 | 50 | - name: Unit Tests 51 | run: MIX_ENV=test mix test 52 | 53 | - name: Run Credo 54 | run: MIX_ENV=test mix credo --strict 55 | -------------------------------------------------------------------------------- /.github/workflows/integration.yaml: -------------------------------------------------------------------------------- 1 | name: Integration Tests 2 | on: 3 | push: 4 | branches: [main] 5 | pull_request: 6 | branches: [main] 7 | 8 | env: 9 | MIX_ENV: test 10 | KUBECONFIG: /home/runner/.kube/config 11 | 12 | jobs: 13 | code-quality: 14 | name: Integration Tests 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 18 | 19 | - name: Setup elixir 20 | id: beam 21 | uses: erlef/setup-beam@v1 22 | with: 23 | version-file: .tool-versions 24 | version-type: strict 25 | install-rebar: true 26 | install-hex: true 27 | 28 | - name: Retrieve Build Cache 29 | uses: actions/cache@v4 30 | id: build-folder-cache 31 | with: 32 | path: _build/test 33 | key: ${{ runner.os }}-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-build-test-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} 34 | 35 | - name: Retrieve Mix Dependencies Cache 36 | uses: actions/cache@v4 37 | id: mix-cache 38 | with: 39 | path: deps 40 | key: ${{ runner.os }}-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} 41 | 42 | - name: Install Mix Dependencies 43 | run: mix deps.get 44 | 45 | - uses: engineerd/setup-kind@v0.5.0 46 | id: kind 47 | with: 48 | version: v0.24.0 49 | name: flame-integration-test 50 | 51 | - name: Integration Tests 52 | run: MIX_ENV=test mix test --only integration 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | flame_k8s_backend-*.tar 24 | 25 | # Temporary files, for example, from tests. 26 | /tmp/ 27 | -------------------------------------------------------------------------------- /.mise.toml: -------------------------------------------------------------------------------- 1 | [env] 2 | env_file = '.env' 3 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | erlang 27.3.4 2 | elixir 1.18.4 3 | kind 0.29.0 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). 6 | 7 | ## Unreleased 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | ## [0.5.7] - 2024-12-05 16 | 17 | ### Fixed 18 | 19 | - Allow callers to set omit_owner_reference option [#59](https://github.com/mruoss/flame_k8s_backend/pull/59) 20 | 21 | ## [0.5.6] - 2024-10-09 22 | 23 | ### Fixed 24 | 25 | - Set `RELEASE_COOKIE` instead of `RELEASE_SECRET` 26 | 27 | ## [0.5.5] - 2024-10-08 28 | 29 | ### Fixed 30 | 31 | - Set `RELEASE_COOKIE`, `RELEASE_DISTRIBUTION` and `RELEASE_NODE` on runner pod if not set [#50](https://github.com/mruoss/flame_k8s_backend/issues/50), [#53](https://github.com/mruoss/flame_k8s_backend/pull/53) 32 | 33 | ## [0.5.4] - 2024-09-11 34 | 35 | ### Changed 36 | 37 | - Upgrade to FLAME 0.5.0 38 | 39 | ## [0.5.3] - 2024-08-30 40 | 41 | ### Changed 42 | 43 | - Reverted change in `0.5.2`. Users should parse the YAML if they want to. 44 | 45 | ## [0.5.2] - 2024-08-29 46 | 47 | ### Added 48 | 49 | - `FLAMEK8sBackend.RunnerPodTemplate`: Allow BYO pod template to be a binary. 50 | 51 | ## [0.5.1] - 2024-08-28 52 | 53 | ### Changed 54 | 55 | - Upgrade to FLAME 0.4.0 56 | 57 | ## [0.5.0] - 2024-08-27 58 | 59 | ### Fixed 60 | 61 | - `FLAMEK8sBackend.RunnerPodTemplate`: Only set `PHX_SERVER` if it is not passed. 62 | - `FLAMEK8sBackend.RunnerPodTemplate`: Reject `FLAME_PARENT`, not `FLAME_BACKEND` in passed env vars. 63 | 64 | ### Added 65 | 66 | - `FLAMEK8sBackend.RunnerPodTemplate`: Set `.metadata.namespace` and `.metadata.generateName` on runner pod if not set ([#43](https://github.com/mruoss/flame_k8s_backend/pull/43)) 67 | 68 | ### Changed 69 | 70 | - `FLAMEK8sBackend.RunnerPodTemplate`: Also copy `env_from` if `add_parent_env` is `true` 71 | - Improve documentation 72 | 73 | ## [0.4.3] - 2024-08-22 74 | 75 | ### Fixed 76 | 77 | - use `FLAME.Parser.JSON` instead of `Jason` 78 | 79 | ### Added 80 | 81 | - Support for BYO runner pod templates as map. 82 | 83 | ## [0.4.2] - 2024-07-28 84 | 85 | ### Fixed 86 | 87 | - SSL cert verification workaround for older OTP versions was added again - [#37](https://github.com/mruoss/flame_k8s_backend/issues/37) [#38](https://github.com/mruoss/flame_k8s_backend/pull/38) 88 | - Upgraded FLAME dependency to `0.3.0` 89 | 90 | ## [0.4.1] - 2024-07-07 91 | 92 | ### Changed 93 | 94 | - Remove `Req` dependency and use `:httpc` instead in order to be safer when run in Livebook. [#35](https://github.com/mruoss/flame_k8s_backend/pull/35) 95 | 96 | ## [0.4.0] - 2024-06-19 97 | 98 | ### Changed 99 | 100 | - Support for FLAME >= 0.2.0 and livebook integraion (requires livebook >= 0.13.0) - [#32](https://github.com/mruoss/flame_k8s_backend/pull/32) 101 | 102 | ## [0.3.3] - 2024-04-29 103 | 104 | ### Changed 105 | 106 | - With `mint` 1.6.0 out, we have no need for the temporary workaround for TLS 107 | verification anymore. 108 | 109 | ## [0.3.2] - 2024-02-25 110 | 111 | ### Changed 112 | 113 | - Dependency Updates 114 | 115 | ## [0.3.1] - 2024-01-28 116 | 117 | ### Changed 118 | 119 | - Use `:cacertfile` insead of `:cacerts` in `:transport_options` and let the OTP process the certificate - [#8](https://github.com/mruoss/flame_k8s_backend/pull/8) 120 | - Dependency Updates 121 | 122 | ## [0.3.0] - 2023-12-19 123 | 124 | ### Changed 125 | 126 | - Remove`:insecure_skip_tls_verify` option and use a custom `match_fun` instead to work around failing hostname verification for IP addresses. - [#5](https://github.com/mruoss/flame_k8s_backend/pull/5) 127 | 128 | ## [0.2.3] - 2023-12-15 129 | 130 | ### Added 131 | 132 | - `runner_pod_tpl` option for better control over the runner pod manifest - [#2](https://github.com/mruoss/flame_k8s_backend/pull/2) 133 | - Basic integration test 134 | 135 | ### Changed 136 | 137 | - Delete pod when shutting down the runner. 138 | 139 | ## [0.2.2] - 2023-12-14 140 | 141 | ### Fixed 142 | 143 | - Don't crash the runner if the `:log` option is not set (or set to `false`) 144 | 145 | ## [0.2.1] - 2023-12-11 146 | 147 | ### Changed 148 | 149 | - ENV var `DRAGONFLY_PARENT` was renamed to `FLAME_PARENT` in commit [9c2e65cc](https://github.com/phoenixframework/flame/commit/9c2e65ccd2c55514a473ad6ed986326576687064) 150 | 151 | ## [0.2.0] - 2023-12-10 152 | 153 | ### Changed 154 | 155 | - Replace `k8s` lib with a lightweight Kubernetes client implementation. 156 | 157 | ## [0.1.0] - 2023-12-09 158 | 159 | - Very early stage implementation of a Kubernetes backend. 160 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # MIT License 2 | 3 | Copyright (c) 2023 Michael Ruoss 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FLAMEK8sBackend 2 | 3 | A [FLAME](https://github.com/phoenixframework/flame/tree/main) Backend for 4 | Kubernetes. Manages pods as runners in the cluster the app is running in. 5 | 6 | [![Module Version](https://img.shields.io/hexpm/v/flame_k8s_backend.svg)](https://hex.pm/packages/flame_k8s_backend) 7 | [![Last Updated](https://img.shields.io/github/last-commit/mruoss/flame_k8s_backend.svg)](https://github.com/mruoss/flame_k8s_backend/commits/main) 8 | 9 | [![Hex Docs](https://img.shields.io/badge/hex-docs-lightgreen.svg)](https://hexdocs.pm/flame_k8s_backend/) 10 | [![Total Download](https://img.shields.io/hexpm/dt/flame_k8s_backend.svg)](https://hex.pm/packages/flame_k8s_backend) 11 | [![License](https://img.shields.io/hexpm/l/flame_k8s_backend.svg)](https://github.com/mruoss/flame_k8s_backend/blob/main/LICENSE) 12 | 13 | The current implementation is very basic and more like a proof of concept. 14 | More configuration options (resources, etc.) will follow. 15 | 16 | ## Installation 17 | 18 | ```elixir 19 | def deps do 20 | [ 21 | {:flame_k8s_backend, "~> 0.5.0"} 22 | ] 23 | end 24 | ``` 25 | 26 | ## Usage 27 | 28 | Configure the flame backend in our configuration or application setup: 29 | 30 | ```elixir 31 | # application.ex 32 | children = [ 33 | {FLAME.Pool, 34 | name: MyApp.SamplePool, 35 | backend: FLAMEK8sBackend, 36 | min: 0, 37 | max: 10, 38 | max_concurrency: 5, 39 | idle_shutdown_after: 30_000, 40 | log: :debug} 41 | ] 42 | ``` 43 | 44 | ## Prerequisites 45 | 46 | ### Env Variables 47 | 48 | In order for the runners to be able to join the cluster, you need to configure 49 | a few environment variables on your pod/deployment: 50 | 51 | The `POD_NAME` and `POD_NAMESPACE` are used by the backend to get informations 52 | from your pod and use them for the runner pods (e.g. env variables). 53 | 54 | ```yaml 55 | apiVersion: apps/v1 56 | kind: Deployment 57 | spec: 58 | selector: 59 | matchLabels: 60 | app: myapp 61 | template: 62 | spec: 63 | containers: 64 | - env: 65 | - name: POD_NAME 66 | valueFrom: 67 | fieldRef: 68 | apiVersion: v1 69 | fieldPath: metadata.name 70 | - name: POD_NAMESPACE 71 | valueFrom: 72 | fieldRef: 73 | apiVersion: v1 74 | fieldPath: metadata.namespace 75 | ``` 76 | 77 | ### RBAC 78 | 79 | Your application needs run as a service account with permissions to manage 80 | pods: 81 | 82 | ```yaml 83 | --- 84 | apiVersion: v1 85 | kind: ServiceAccount 86 | metadata: 87 | name: myapp 88 | namespace: app-namespace 89 | --- 90 | apiVersion: rbac.authorization.k8s.io/v1 91 | kind: Role 92 | metadata: 93 | namespace: app-namespace 94 | name: pod-mgr 95 | rules: 96 | - apiGroups: [""] 97 | resources: ["pods"] 98 | verbs: ["create", "get", "list", "delete", "patch"] 99 | --- 100 | apiVersion: rbac.authorization.k8s.io/v1 101 | kind: RoleBinding 102 | metadata: 103 | name: myapp-pod-mgr 104 | namespace: app-namespace 105 | subjects: 106 | - kind: ServiceAccount 107 | name: myapp 108 | namespace: app-namespace 109 | roleRef: 110 | kind: Role 111 | name: pod-mgr 112 | apiGroup: rbac.authorization.k8s.io 113 | --- 114 | apiVersion: apps/v1 115 | kind: Deployment 116 | spec: 117 | template: 118 | spec: 119 | serviceAccountName: my-app 120 | ``` 121 | 122 | ### Clustering 123 | 124 | Your application needs to be able to form a cluster with your runners. Define 125 | `POD_IP`, `RELEASE_DISTRIBUTION` and `RELEASE_NODE` environment variables on 126 | your pods as follows: 127 | 128 | ```yaml 129 | apiVersion: apps/v1 130 | kind: Deployment 131 | spec: 132 | template: 133 | spec: 134 | containers: 135 | - env: 136 | - name: POD_IP 137 | valueFrom: 138 | fieldRef: 139 | apiVersion: v1 140 | fieldPath: status.podIP 141 | - name: RELEASE_DISTRIBUTION 142 | value: name 143 | - name: RELEASE_NODE 144 | value: my_app@$(POD_IP) 145 | ``` 146 | 147 | ## How it works 148 | 149 | The FLAME Kubernetes backend first queries the Kubernetes API server to extract 150 | information from the running Pod like the container image, resource requests and 151 | limits, environment variables etc. This information is then used to build the 152 | manifest for the runner pod. The backend then sends the resulting manifest to 153 | the API server in order to spin up a runner pod. 154 | 155 | ## Troubleshooting 156 | 157 | ### My runner Pod disappears / gets killed after only a few seconds 158 | 159 | If your parent is part of a Deployment, make sure your FLAME runner Pod doesn't 160 | contain the labels you used as selectors (i.e. `.spec.selector` on your 161 | Deployment). Otherwise the replica controller sees your FLAME runner as an 162 | additional Pod to the Deployment's ReplicaSet and "downscales" the deployment to 163 | the desired replica count (i.e. snipes your runner Pod). 164 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | config :flame, :terminator, log: :debug 4 | -------------------------------------------------------------------------------- /lib/flame_k8s_backend.ex: -------------------------------------------------------------------------------- 1 | defmodule FLAMEK8sBackend do 2 | @moduledoc """ 3 | Kubernetes Backend implementation. 4 | 5 | ### Usage 6 | 7 | Configure the flame backend in our configuration or application setup: 8 | 9 | ``` 10 | # application.ex 11 | children = [ 12 | {FLAME.Pool, 13 | name: MyApp.SamplePool, 14 | backend: FLAMEK8sBackend, 15 | min: 0, 16 | max: 10, 17 | max_concurrency: 5, 18 | idle_shutdown_after: 30_000} 19 | ] 20 | ``` 21 | 22 | ### Options 23 | 24 | The following backend options are supported: 25 | 26 | * `:app_container_name` - If your application pod runs multiple containers 27 | (initContainers excluded), use this option to pass the name of the 28 | container running this application. If not given, the first container 29 | in the list of containers is used to lookup the contaienr image, env vars 30 | and resources to be used for the runner pods. 31 | 32 | * `:omit_owner_reference` - If true, no ownerReferences are configured on 33 | the runner pods. Defaults to `false` 34 | 35 | * `:runner_pod_tpl` - If given, controls how the runner pod manifest is 36 | generated. Can be a function of type 37 | `t:FLAMEK8sBackend.RunnerPodTemplate.callback/0` or a struct of type 38 | `t:FLAMEK8sBackend.RunnerPodTemplate.t/0`. 39 | A callback receives the manifest of the parent pod as a map and should 40 | return the runner pod's manifest as a map(). 41 | If a struct is given, the runner pod's manifest will be generated with 42 | values from the struct if given or from the parent pod if omitted. 43 | If this option is omitted, the parent pod's `env` and `resources` 44 | are used for the runner pod. 45 | See `FLAMEK8sBackend.RunnerPodTemplate` for more infos. 46 | 47 | * `:log` - The log level to use for verbose logging. Defaults to `false`. 48 | 49 | ### Prerequisites 50 | 51 | In order for this to work, your application needs to meet some requirements. 52 | 53 | #### Env Variables 54 | 55 | In order for the backend to be able to get informations from your pod and use 56 | them for the runner pods (e.g. env variables), you have to define `POD_NAME` 57 | and `POD_NAMESPACE` environment variables on your pod. 58 | 59 | ```yaml 60 | apiVersion: apps/v1 61 | kind: Deployment 62 | spec: 63 | selector: 64 | matchLabels: 65 | app: myapp 66 | template: 67 | spec: 68 | containers: 69 | - env: 70 | - name: POD_NAME 71 | valueFrom: 72 | fieldRef: 73 | apiVersion: v1 74 | fieldPath: metadata.name 75 | - name: POD_NAMESPACE 76 | valueFrom: 77 | fieldRef: 78 | apiVersion: v1 79 | fieldPath: metadata.namespace 80 | ``` 81 | 82 | #### RBAC 83 | 84 | Your application needs run as a service account with permissions to manage 85 | pods. This is a simple 86 | 87 | ```yaml 88 | --- 89 | apiVersion: v1 90 | kind: ServiceAccount 91 | metadata: 92 | name: myapp 93 | namespace: app-namespace 94 | --- 95 | apiVersion: rbac.authorization.k8s.io/v1 96 | kind: Role 97 | metadata: 98 | namespace: app-namespace 99 | name: pod-mgr 100 | rules: 101 | - apiGroups: [""] 102 | resources: ["pods"] 103 | verbs: ["create", "get", "list", "delete", "patch"] 104 | --- 105 | apiVersion: rbac.authorization.k8s.io/v1 106 | kind: RoleBinding 107 | metadata: 108 | name: myapp-pod-mgr 109 | namespace: app-namespace 110 | subjects: 111 | - kind: ServiceAccount 112 | name: myapp 113 | namespace: app-namespace 114 | roleRef: 115 | kind: Role 116 | name: pod-mgr 117 | apiGroup: rbac.authorization.k8s.io 118 | --- 119 | apiVersion: apps/v1 120 | kind: Deployment 121 | spec: 122 | template: 123 | spec: 124 | serviceAccountName: my-app 125 | ``` 126 | 127 | #### Clustering 128 | 129 | Your application needs to be able to form a cluster with your runners. Define 130 | `POD_IP`, `RELEASE_DISTRIBUTION` and `RELEASE_NODE` environment variables on 131 | your pods as follows: 132 | 133 | ```yaml 134 | apiVersion: apps/v1 135 | kind: Deployment 136 | spec: 137 | template: 138 | spec: 139 | containers: 140 | - env: 141 | - name: POD_IP 142 | valueFrom: 143 | fieldRef: 144 | apiVersion: v1 145 | fieldPath: status.podIP 146 | - name: RELEASE_DISTRIBUTION 147 | value: name 148 | - name: RELEASE_NODE 149 | value: my_app@$(POD_IP) 150 | ``` 151 | """ 152 | @behaviour FLAME.Backend 153 | 154 | alias FLAMEK8sBackend.K8sClient 155 | alias FLAMEK8sBackend.RunnerPodTemplate 156 | 157 | require Logger 158 | 159 | defstruct runner_pod_manifest: nil, 160 | parent_ref: nil, 161 | runner_node_name: nil, 162 | runner_pod_tpl: nil, 163 | boot_timeout: nil, 164 | remote_terminator_pid: nil, 165 | omit_owner_reference: false, 166 | log: false, 167 | http: nil 168 | 169 | @valid_opts ~w(app_container_name runner_pod_tpl terminator_sup log boot_timeout omit_owner_reference)a 170 | @required_config ~w()a 171 | 172 | @impl true 173 | def init(opts) do 174 | conf = Application.get_env(:flame, __MODULE__) || [] 175 | [_node_base | _ip] = node() |> to_string() |> String.split("@") 176 | 177 | default = %FLAMEK8sBackend{ 178 | boot_timeout: 30_000 179 | } 180 | 181 | provided_opts = 182 | conf 183 | |> Keyword.merge(opts) 184 | |> Keyword.validate!(@valid_opts) 185 | 186 | state = struct(default, provided_opts) 187 | 188 | for key <- @required_config do 189 | unless Map.get(state, key) do 190 | raise ArgumentError, "missing :#{key} config for #{inspect(__MODULE__)}" 191 | end 192 | end 193 | 194 | parent_ref = make_ref() 195 | 196 | http = K8sClient.connect() 197 | 198 | case K8sClient.get_pod(http, System.get_env("POD_NAMESPACE"), System.get_env("POD_NAME")) do 199 | {:ok, base_pod} -> 200 | new_state = 201 | struct(state, 202 | http: http, 203 | parent_ref: parent_ref, 204 | runner_pod_manifest: 205 | RunnerPodTemplate.manifest( 206 | base_pod, 207 | provided_opts[:runner_pod_tpl], 208 | parent_ref, 209 | Keyword.take(provided_opts, [:app_container_name, :omit_owner_reference]) 210 | ) 211 | ) 212 | 213 | {:ok, new_state} 214 | 215 | {:error, error} -> 216 | Logger.error(Exception.message(error)) 217 | {:error, error} 218 | end 219 | end 220 | 221 | @impl true 222 | def remote_spawn_monitor(%FLAMEK8sBackend{} = state, term) do 223 | case term do 224 | func when is_function(func, 0) -> 225 | {pid, ref} = Node.spawn_monitor(state.runner_node_name, func) 226 | {:ok, {pid, ref}} 227 | 228 | {mod, fun, args} when is_atom(mod) and is_atom(fun) and is_list(args) -> 229 | {pid, ref} = Node.spawn_monitor(state.runner_node_name, mod, fun, args) 230 | {:ok, {pid, ref}} 231 | 232 | other -> 233 | raise ArgumentError, 234 | "expected a null arity function or {mod, func, args}. Got: #{inspect(other)}" 235 | end 236 | end 237 | 238 | @impl true 239 | def system_shutdown() do 240 | # This is not very nice but I don't have the opts on the runner 241 | http = K8sClient.connect() 242 | namespace = System.get_env("POD_NAMESPACE") 243 | name = System.get_env("POD_NAME") 244 | K8sClient.delete_pod!(http, namespace, name) 245 | System.stop() 246 | end 247 | 248 | @impl true 249 | def remote_boot(%FLAMEK8sBackend{parent_ref: parent_ref} = state) do 250 | log(state, "Remote Boot") 251 | 252 | {new_state, req_connect_time} = 253 | with_elapsed_ms(fn -> 254 | created_pod = 255 | K8sClient.create_pod!(state.http, state.runner_pod_manifest, state.boot_timeout) 256 | 257 | case created_pod do 258 | {:ok, pod} -> 259 | log(state, "Runner pod created and scheduled", pod_ip: pod["status"]["podIP"]) 260 | state 261 | 262 | :error -> 263 | Logger.error("failed to schedule runner pod within #{state.boot_timeout}ms") 264 | exit(:timeout) 265 | end 266 | end) 267 | 268 | remaining_connect_window = state.boot_timeout - req_connect_time 269 | 270 | log(state, "Waiting for Remote UP.", remaining_connect_window: remaining_connect_window) 271 | 272 | remote_terminator_pid = 273 | receive do 274 | {^parent_ref, {:remote_up, remote_terminator_pid}} -> 275 | log(state, "Remote flame is Up!") 276 | remote_terminator_pid 277 | after 278 | remaining_connect_window -> 279 | Logger.error("failed to connect to runner pod within #{state.boot_timeout}ms") 280 | exit(:timeout) 281 | end 282 | 283 | new_state = 284 | struct!(new_state, 285 | remote_terminator_pid: remote_terminator_pid, 286 | runner_node_name: node(remote_terminator_pid) 287 | ) 288 | 289 | {:ok, remote_terminator_pid, new_state} 290 | end 291 | 292 | @impl true 293 | def handle_info(msg, state) do 294 | log(state, "Missed message: #{inspect(msg)}") 295 | {:noreply, state} 296 | end 297 | 298 | defp with_elapsed_ms(func) when is_function(func, 0) do 299 | {micro, result} = :timer.tc(func) 300 | {result, div(micro, 1000)} 301 | end 302 | 303 | defp log(state, msg, metadata \\ []) 304 | 305 | defp log(%FLAMEK8sBackend{log: false}, _, _), do: :ok 306 | 307 | defp log(%FLAMEK8sBackend{log: level}, msg, metadata) do 308 | Logger.log(level, msg, metadata) 309 | end 310 | end 311 | -------------------------------------------------------------------------------- /lib/flame_k8s_backend/http.ex: -------------------------------------------------------------------------------- 1 | defmodule FlameK8sBackend.HTTP do 2 | @moduledoc false 3 | alias Credo.CLI.Output.Formatter.JSON 4 | alias FLAME.Parser.JSON 5 | 6 | defstruct [:base_url, :token, :cacertfile] 7 | 8 | @type t :: %__MODULE__{ 9 | base_url: String.t(), 10 | token: String.t(), 11 | cacertfile: String.t() 12 | } 13 | 14 | @spec new(fields :: Keyword.t()) :: t() 15 | def new(fields), do: struct!(__MODULE__, fields) 16 | 17 | @spec get(t(), path :: String.t()) :: {:ok, map()} | {:error, String.t()} 18 | def get(http, path), do: request(http, :get, path) 19 | 20 | @spec get!(t(), path :: String.t()) :: map() 21 | def get!(http, path), do: request!(http, :get, path) 22 | 23 | @spec post!(t(), path :: String.t(), body :: String.t()) :: map() 24 | def post!(http, path, body), do: request!(http, :post, path, body) 25 | 26 | @spec delete!(t(), path :: String.t()) :: map() 27 | def delete!(http, path), do: request!(http, :delete, path) 28 | 29 | @spec request!(http :: t(), verb :: atom(), path :: Stringt.t()) :: map() 30 | defp request!(http, verb, path, body \\ nil) do 31 | case request(http, verb, path, body) do 32 | {:ok, response_body} -> 33 | response_body 34 | |> List.to_string() 35 | |> JSON.decode!() 36 | 37 | {:error, reason} -> 38 | raise reason 39 | end 40 | end 41 | 42 | @spec request(http :: t(), verb :: atom(), path :: String.t(), body :: String.t()) :: 43 | {:ok, String.t()} | {:error, String.t()} 44 | defp request(http, verb, path, body \\ nil) do 45 | headers = [{~c"Authorization", ~c"Bearer #{http.token}"}] 46 | 47 | http_opts = [ 48 | ssl: [ 49 | verify: :verify_peer, 50 | cacertfile: http.cacertfile, 51 | customize_hostname_check: [match_fun: &check_ips_as_dns_id/2] 52 | ] 53 | ] 54 | 55 | url = http.base_url <> path 56 | 57 | request = 58 | if is_nil(body), 59 | do: {url, headers}, 60 | else: {url, headers, ~c"application/json", body} 61 | 62 | case :httpc.request(verb, request, http_opts, []) do 63 | {:ok, {{_, status, _}, _, response_body}} when status in 200..299 -> 64 | {:ok, response_body} 65 | 66 | {:ok, {{_, status, reason}, _, resp_body}} -> 67 | {:error, 68 | "failed #{String.upcase("#{verb}")} #{url} with #{inspect(status)} (#{inspect(reason)}): #{inspect(resp_body)} #{inspect(headers)}"} 69 | 70 | {:error, reason} -> 71 | {:error, 72 | "failed #{String.upcase("#{verb}")} #{url} with #{inspect(reason)} #{inspect(http.headers)}"} 73 | end 74 | end 75 | 76 | if String.to_integer(System.otp_release()) < 27 do 77 | # Workaround for an issue in OTP<27 78 | # https://github.com/erlang/otp/issues/7968 79 | defp check_ips_as_dns_id({:dns_id, hostname}, {:iPAddress, ip}) do 80 | with {:ok, ip_tuple} <- :inet.parse_address(hostname), 81 | ^ip <- Tuple.to_list(ip_tuple) do 82 | true 83 | else 84 | _ -> :default 85 | end 86 | end 87 | end 88 | 89 | defp check_ips_as_dns_id(_, _), do: :default 90 | end 91 | -------------------------------------------------------------------------------- /lib/flame_k8s_backend/k8s_client.ex: -------------------------------------------------------------------------------- 1 | defmodule FLAMEK8sBackend.K8sClient do 2 | @moduledoc false 3 | 4 | @sa_token_path "/var/run/secrets/kubernetes.io/serviceaccount" 5 | 6 | alias FLAME.Parser.JSON 7 | alias FlameK8sBackend.HTTP 8 | 9 | def connect() do 10 | ca_cert_path = Path.join(@sa_token_path, "ca.crt") 11 | token_path = Path.join(@sa_token_path, "token") 12 | apiserver_host = System.get_env("KUBERNETES_SERVICE_HOST") 13 | apiserver_port = System.get_env("KUBERNETES_SERVICE_PORT_HTTPS") 14 | token = File.read!(token_path) 15 | 16 | HTTP.new( 17 | base_url: "https://#{apiserver_host}:#{apiserver_port}", 18 | token: token, 19 | cacertfile: String.to_charlist(ca_cert_path) 20 | ) 21 | end 22 | 23 | def get_pod!(http, namespace, name) do 24 | HTTP.get!(http, pod_path(namespace, name)) 25 | end 26 | 27 | def get_pod(http, namespace, name) do 28 | with {:ok, response_body} <- HTTP.get(http, pod_path(namespace, name)) do 29 | {:ok, 30 | response_body 31 | |> List.to_string() 32 | |> JSON.decode!()} 33 | end 34 | end 35 | 36 | def delete_pod!(http, namespace, name) do 37 | HTTP.delete!(http, pod_path(namespace, name)) 38 | end 39 | 40 | def create_pod!(http, pod, timeout) do 41 | namespace = pod["metadata"]["namespace"] 42 | created_pod = HTTP.post!(http, pod_path(namespace, ""), JSON.encode!(pod)) 43 | name = created_pod["metadata"]["name"] 44 | wait_until_scheduled(http, namespace, name, timeout) 45 | end 46 | 47 | defp wait_until_scheduled(_req, _namespace, _name, timeout) when timeout <= 0, do: :error 48 | 49 | defp wait_until_scheduled(req, namespace, name, timeout) do 50 | case get_pod!(req, namespace, name) do 51 | %{"status" => %{"podIP" => _}} = pod -> 52 | {:ok, pod} 53 | 54 | _ -> 55 | Process.sleep(1000) 56 | wait_until_scheduled(req, namespace, name, timeout - 1000) 57 | end 58 | end 59 | 60 | defp pod_path(namespace, name) do 61 | "/api/v1/namespaces/#{namespace}/pods/#{name}" 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /lib/flame_k8s_backend/runner_pod_template.ex: -------------------------------------------------------------------------------- 1 | defmodule FLAMEK8sBackend.RunnerPodTemplate do 2 | @moduledoc """ 3 | 4 | This module is responsible for generating the manifest for the runner pods. 5 | The manifest can be overridden using the `runner_pod_tpl` option on 6 | `FLAMEK8sBackend`. 7 | 8 | ### Simple Use Case 9 | 10 | By default, `resources` and `env` variables are copied from the parent pod. 11 | Using the `runner_pod_tpl` option on the `FLAMEK8sBackend`, you can add 12 | additional environment variables or set different `resources`. You would do 13 | this by setting the `runner_pod_tpl` to a struct of type 14 | `t:FLAMEK8sBackend.RunnerPodTemplate.t/0` as follows: 15 | 16 | ``` 17 | # application.ex 18 | alias FLAMEK8sBackend.RunnerPodTemplate 19 | 20 | children = [ 21 | {FLAME.Pool, 22 | name: MyApp.SamplePool, 23 | backend: {FLAMEK8sBackend, 24 | runner_pod_tpl: %RunnerPodTemplate{ 25 | env: [%{"name" => "FOO", "value" => "bar"}], 26 | resources: %{ 27 | requests: %{"memory" => "256Mi", "cpu" => "100m"}, 28 | limimts: %{"memory" => "256Mi", "cpu" => "400m"} 29 | } 30 | } 31 | }, 32 | # other opts 33 | } 34 | ] 35 | # ... 36 | ``` 37 | 38 | ### Advanced Use Cases 39 | 40 | In some cases you might need advanced control over the runner pod manifest. 41 | Maybe you want to set node affinity because you need your runners to run on 42 | nodes with GPUs or you need additional volumes etc. In this case, you can set 43 | `runner_pod_tpl` to either a map representing the Pod manifest or a callback 44 | function as described below. 45 | 46 | #### Using a Manifest Map 47 | 48 | You can set `runner_pod_tpl` to a map representing the manifest of the runner 49 | Pod: 50 | 51 | 52 | ``` 53 | # application.ex 54 | alias FLAMEK8sBackend.RunnerPodTemplate 55 | import YamlElixir.Sigil 56 | 57 | pod_template = ~y\""" 58 | apiVersion: v1 59 | kind: Pod 60 | metadata: 61 | # your metadata 62 | spec: 63 | # Pod spec 64 | \""" 65 | 66 | children = [ 67 | {FLAME.Pool, 68 | name: MyApp.SamplePool, 69 | backend: {FLAMEK8sBackend, runner_pod_tpl: pod_template}, 70 | # other opts 71 | } 72 | ] 73 | # ... 74 | ``` 75 | 76 | #### Using a Callback Function 77 | 78 | The callback has to be of type 79 | `t:FLAMEK8sBackend.RunnerPodTemplate.callback/0`. The callback will be called 80 | with the manifest of the parent pod which can be used to extract information. 81 | It should return a pod template as a map 82 | 83 | Define a callback, e.g. in a separate module: 84 | 85 | ``` 86 | defmodule MyApp.FLAMERunnerPodTemplate do 87 | def runner_pod_manifest(parent_pod_manifest) do 88 | %{ 89 | "metadata" => %{ 90 | # namespace, labels, ownerReferences,... 91 | }, 92 | "spec" => %{ 93 | "containers" => [ 94 | %{ 95 | # container definition 96 | } 97 | ] 98 | } 99 | } 100 | end 101 | end 102 | ``` 103 | 104 | Register the backend: 105 | 106 | ``` 107 | # application.ex 108 | # ... 109 | 110 | children = [ 111 | {FLAME.Pool, 112 | name: MyApp.SamplePool, 113 | backend: {FLAMEK8sBackend, runner_pod_tpl: &MyApp.FLAMERunnerPodTemplate.runner_pod_manifest/1}, 114 | # other opts 115 | } 116 | ] 117 | # ... 118 | ``` 119 | 120 | > #### Predefined Values {: .warning} 121 | > 122 | > Note that the following values are controlled by the backend and, if set by 123 | > your callback function, are going to be overwritten: 124 | > 125 | > * `apiVersion` and `Kind` of the resource (set to `v1/Pod`) 126 | > * The pod's and container's names (set to a combination of the parent 127 | > pod's name and a random id) 128 | > * The `restartPolicy` (set to `Never`) 129 | > * The container `image` (set to the image of the parent pod's app 130 | > container) 131 | 132 | > #### Automatically Defined Environment Variables {: .info} 133 | > 134 | > Some environment variables are defined automatically on the 135 | > runner pod: 136 | > 137 | > * `POD_IP` is set to the runner Pod's IP address (`.status.podIP`) - (not overridable) 138 | > * `POD_NAME` is set to the runner Pod's name (`.metadata.name`) - (not overridable) 139 | > * `POD_NAMESPACE` is set to the runner Pod's namespace (`.metadata.namespace`) - (not overridable) 140 | > * `PHX_SERVER` is set to `false` (overridable) 141 | > * `FLAME_PARENT` used internally by FLAME - (not overridable) 142 | 143 | ### Options 144 | 145 | * `:omit_owner_reference` - Omit generating and appending the parent pod as 146 | `ownerReference` to the runner pod's metadata 147 | 148 | * `:app_container_name` - name of the container running this application. By 149 | default, the first container in the list of containers is used. 150 | """ 151 | 152 | alias FLAMEK8sBackend.RunnerPodTemplate 153 | 154 | defstruct [:env, :resources, add_parent_env: true] 155 | 156 | @typedoc """ 157 | Describing the Runner Pod Template struct 158 | 159 | ### Fields 160 | 161 | * `env` - a map describing a Pod environment variable declaration 162 | `%{"name" => "MY_ENV_VAR", "value" => "my_env_var_value"}` 163 | 164 | * `resources` - Pod resource requests and limits. 165 | 166 | * `add_parent_env` - If true, all env vars of the main container 167 | including `envFrom` are copied to the runner pod. 168 | default: `true` 169 | """ 170 | @type t :: %__MODULE__{ 171 | env: map() | nil, 172 | resources: map() | nil, 173 | add_parent_env: boolean() 174 | } 175 | @type parent_pod_manifest :: map() 176 | @type callback :: (parent_pod_manifest() -> runner_pod_template :: map()) 177 | 178 | @doc """ 179 | Generates the POD manifest using information from the parent pod 180 | and the `runner_pod_tpl` option. 181 | """ 182 | @spec manifest(parent_pod_manifest(), t() | callback(), Keyword.t()) :: 183 | runner_pod_template :: map() 184 | def manifest(parent_pod_manifest, template_args_or_callback, parent_ref, opts \\ []) 185 | 186 | def manifest(parent_pod_manifest, template_callback, parent_ref, opts) 187 | when is_function(template_callback) do 188 | app_container = app_container(parent_pod_manifest, opts) 189 | 190 | parent_pod_manifest 191 | |> template_callback.() 192 | |> apply_defaults(parent_pod_manifest, app_container, parent_ref, opts) 193 | end 194 | 195 | def manifest(parent_pod_manifest, nil, parent_ref, opts) do 196 | manifest(parent_pod_manifest, %RunnerPodTemplate{}, parent_ref, opts) 197 | end 198 | 199 | def manifest(parent_pod_manifest, %RunnerPodTemplate{} = template_opts, parent_ref, opts) do 200 | app_container = app_container(parent_pod_manifest, opts) 201 | env = template_opts.env || [] 202 | 203 | parent_env = if template_opts.add_parent_env, do: app_container["env"] 204 | parent_env_from = if template_opts.add_parent_env, do: app_container["envFrom"] 205 | 206 | runner_pod_template = %{ 207 | "metadata" => %{ 208 | "namespace" => parent_pod_manifest["metadata"]["namespace"] 209 | }, 210 | "spec" => %{ 211 | "containers" => [ 212 | %{ 213 | "resources" => template_opts.resources || app_container["resources"], 214 | "env" => env ++ List.wrap(parent_env), 215 | "envFrom" => List.wrap(parent_env_from) 216 | } 217 | ] 218 | } 219 | } 220 | 221 | apply_defaults(runner_pod_template, parent_pod_manifest, app_container, parent_ref, opts) 222 | end 223 | 224 | def manifest(parent_pod_manifest, runner_pod_template, parent_ref, opts) 225 | when is_map(runner_pod_template) do 226 | app_container = app_container(parent_pod_manifest, opts) 227 | apply_defaults(runner_pod_template, parent_pod_manifest, app_container, parent_ref, opts) 228 | end 229 | 230 | defp apply_defaults( 231 | runner_pod_template, 232 | parent_pod_manifest, 233 | app_container, 234 | parent_ref, 235 | opts 236 | ) do 237 | parent_pod_manifest_name = parent_pod_manifest["metadata"]["name"] 238 | parent_pod_manifest_namespace = parent_pod_manifest["metadata"]["namespace"] 239 | 240 | object_references = 241 | if opts[:omit_owner_reference], 242 | do: [], 243 | else: object_references(parent_pod_manifest) 244 | 245 | parent = 246 | FLAME.Parent.new(parent_ref, self(), FLAMEK8sBackend, parent_pod_manifest_name, "POD_IP") 247 | 248 | parent = 249 | case System.get_env("FLAME_K8S_BACKEND_GIT_REF") do 250 | nil -> parent 251 | git_ref -> struct(parent, backend_vsn: [github: "mruoss/flame_k8s_backend", ref: git_ref]) 252 | end 253 | 254 | encoded_parent = FLAME.Parent.encode(parent) 255 | 256 | runner_pod_template 257 | |> Map.merge(%{"apiVersion" => "v1", "kind" => "Pod"}) 258 | |> update_in([Access.key("metadata", %{})], fn metadata -> 259 | metadata 260 | |> Map.delete("name") 261 | |> Map.put_new("generateName", parent_pod_manifest_name <> "-") 262 | |> Map.put_new("namespace", parent_pod_manifest_namespace) 263 | |> Map.put("ownerReferences", object_references) 264 | end) 265 | |> put_in(~w(spec restartPolicy), "Never") 266 | |> update_in(~w(spec), fn spec -> 267 | spec 268 | |> Map.put("restartPolicy", "Never") 269 | |> Map.put_new("serviceAccount", parent_pod_manifest["spec"]["serviceAccount"]) 270 | |> update_in(["containers", Access.at(0)], fn container -> 271 | container 272 | |> Map.put("image", app_container["image"]) 273 | |> Map.put("name", "runner") 274 | |> Map.put_new("env", []) 275 | |> Map.update!("env", fn env -> 276 | [ 277 | %{ 278 | "name" => "POD_NAME", 279 | "valueFrom" => %{"fieldRef" => %{"fieldPath" => "metadata.name"}} 280 | }, 281 | %{ 282 | "name" => "POD_IP", 283 | "valueFrom" => %{"fieldRef" => %{"fieldPath" => "status.podIP"}} 284 | }, 285 | %{ 286 | "name" => "POD_NAMESPACE", 287 | "valueFrom" => %{"fieldRef" => %{"fieldPath" => "metadata.namespace"}} 288 | }, 289 | %{"name" => "FLAME_PARENT", "value" => encoded_parent} 290 | | Enum.reject( 291 | env, 292 | &(&1["name"] in ["FLAME_PARENT", "POD_NAME", "POD_NAMESPACE", "POD_IP"]) 293 | ) 294 | ] 295 | |> put_new_env("PHX_SERVER", "false") 296 | |> put_new_env("RELEASE_COOKIE", Node.get_cookie()) 297 | |> put_new_env("RELEASE_DISTRIBUTION", "name") 298 | |> put_new_env("RELEASE_NODE", "flame_runner@$(POD_IP)") 299 | end) 300 | end) 301 | end) 302 | end 303 | 304 | defp put_new_env(env, _name, :nocookie), do: env 305 | 306 | defp put_new_env(env, name, value) do 307 | case get_in(env, [Access.filter(&(&1["name"] == name))]) do 308 | [] -> [%{"name" => name, "value" => value} | env] 309 | _ -> env 310 | end 311 | end 312 | 313 | defp app_container(parent_pod_manifest, opts) do 314 | container_access = 315 | case opts[:app_container_name] do 316 | nil -> [] 317 | name -> [Access.filter(&(&1["name"] == name))] 318 | end 319 | 320 | parent_pod_manifest 321 | |> get_in(["spec", "containers" | container_access]) 322 | |> List.first() 323 | end 324 | 325 | defp object_references(parent_pod_manifest) do 326 | [ 327 | %{ 328 | "apiVersion" => parent_pod_manifest["apiVersion"], 329 | "kind" => parent_pod_manifest["kind"], 330 | "name" => parent_pod_manifest["metadata"]["name"], 331 | "namespace" => parent_pod_manifest["metadata"]["namespace"], 332 | "uid" => parent_pod_manifest["metadata"]["uid"] 333 | } 334 | ] 335 | end 336 | end 337 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule FlameK8sBackend.MixProject do 2 | use Mix.Project 3 | @source_url "https://github.com/mruoss/flame_k8s_backend" 4 | @version "0.5.7" 5 | 6 | def project do 7 | [ 8 | app: :flame_k8s_backend, 9 | description: "A FLAME backend for Kubernetes", 10 | version: @version, 11 | elixir: "~> 1.15", 12 | elixirc_paths: elixirc_paths(Mix.env()), 13 | start_permanent: Mix.env() == :prod, 14 | deps: deps(), 15 | package: package(), 16 | docs: [ 17 | main: "readme", 18 | extras: ["README.md", "CHANGELOG.md"], 19 | source_ref: "v#{@version}", 20 | source_url: @source_url 21 | ] 22 | ] 23 | end 24 | 25 | # Run "mix help compile.app" to learn about applications. 26 | def application do 27 | [ 28 | extra_applications: [:logger] 29 | ] 30 | end 31 | 32 | # Run "mix help deps" to learn about dependencies. 33 | defp deps do 34 | [ 35 | {:flame, "~> 0.4.0 or ~> 0.5.0"}, 36 | {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, 37 | {:mix_test_watch, "~> 1.0", only: [:dev, :test], runtime: false}, 38 | {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, 39 | {:yaml_elixir, "~> 2.9", only: [:dev, :test], runtime: false} 40 | ] 41 | end 42 | 43 | defp package do 44 | [ 45 | name: :flame_k8s_backend, 46 | maintainers: ["Michael Ruoss"], 47 | licenses: ["MIT"], 48 | links: %{ 49 | "GitHub" => @source_url, 50 | "Sponsor" => "https://github.com/sponsors/mruoss" 51 | }, 52 | files: ["lib", "mix.exs", "README*", "LICENSE*", "CHANGELOG.md"] 53 | ] 54 | end 55 | 56 | defp elixirc_paths(:test), do: ["lib", "test_support"] 57 | defp elixirc_paths(_), do: ["lib"] 58 | end 59 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, 3 | "credo": {:hex, :credo, "1.7.12", "9e3c20463de4b5f3f23721527fcaf16722ec815e70ff6c60b86412c695d426c1", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8493d45c656c5427d9c729235b99d498bd133421f3e0a683e5c1b561471291e5"}, 4 | "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, 5 | "ex_doc": {:hex, :ex_doc, "0.38.2", "504d25eef296b4dec3b8e33e810bc8b5344d565998cd83914ffe1b8503737c02", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "732f2d972e42c116a70802f9898c51b54916e542cc50968ac6980512ec90f42b"}, 6 | "file_system": {:hex, :file_system, "1.1.0", "08d232062284546c6c34426997dd7ef6ec9f8bbd090eb91780283c9016840e8f", [:mix], [], "hexpm", "bfcf81244f416871f2a2e15c1b515287faa5db9c6bcf290222206d120b3d43f6"}, 7 | "flame": {:hex, :flame, "0.5.2", "d46c4daa19b8921b71e0e57dc69edc01ce1311b1976c160192b05d4253b336e8", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, ">= 0.0.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "82560ebef6ab3c277875493d0c93494740c930db0b1a3ff1a570eee9206cc6c0"}, 8 | "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, 9 | "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, 10 | "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, 11 | "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, 12 | "mix_test_watch": {:hex, :mix_test_watch, "1.3.0", "2ffc9f72b0d1f4ecf0ce97b044e0e3c607c3b4dc21d6228365e8bc7c2856dc77", [:mix], [{:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm", "f9e5edca976857ffac78632e635750d158df14ee2d6185a15013844af7570ffe"}, 13 | "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, 14 | "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, 15 | "yaml_elixir": {:hex, :yaml_elixir, "2.11.0", "9e9ccd134e861c66b84825a3542a1c22ba33f338d82c07282f4f1f52d847bd50", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "53cc28357ee7eb952344995787f4bb8cc3cecbf189652236e9b163e8ce1bc242"}, 16 | } 17 | -------------------------------------------------------------------------------- /test/flame_k8s_backend/runner_pod_template_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FLAMEK8sBackend.RunnerPodTemplateTest do 2 | use ExUnit.Case 3 | 4 | alias FLAMEK8sBackend.RunnerPodTemplate, as: MUT 5 | alias FLAMEK8sBackend.TestSupport.Pods 6 | 7 | import YamlElixir.Sigil 8 | 9 | defp flame_parent(pod_manifest) do 10 | pod_manifest 11 | |> get_in(env_var_access("FLAME_PARENT")) 12 | |> List.first() 13 | |> Base.decode64!() 14 | |> :erlang.binary_to_term() 15 | end 16 | 17 | defp env_var_access(name) do 18 | app_container_access(["env", Access.filter(&(&1["name"] == name)), "value"]) 19 | end 20 | 21 | defp app_container_access(field \\ []), 22 | do: ["spec", "containers", Access.at(0)] ++ List.wrap(field) 23 | 24 | setup_all do 25 | [parent_pod_manifest_full: Pods.parent_pod_manifest_full()] 26 | end 27 | 28 | describe "manifest/2 with callback" do 29 | alias FLAMEK8sBackend.TestSupport.Pods 30 | 31 | test "should pass pod manifest to callback", %{parent_pod_manifest_full: parent_pod_manifest} do 32 | callback = fn manifest -> 33 | assert manifest == parent_pod_manifest 34 | 35 | ~y""" 36 | metadata: 37 | namespace: test 38 | spec: 39 | containers: 40 | - resources: 41 | limits: 42 | memory: 500Mi 43 | cpu: 500 44 | """ 45 | |> put_in( 46 | app_container_access(~w(resources requests)), 47 | get_in(manifest, app_container_access(~w(resources requests))) 48 | ) 49 | end 50 | 51 | pod_manifest = MUT.manifest(parent_pod_manifest, callback, make_ref()) 52 | 53 | # sets defaults for required ENV vars: 54 | assert get_in(pod_manifest, env_var_access("PHX_SERVER")) == ["false"] 55 | assert get_in(pod_manifest, env_var_access("RELEASE_DISTRIBUTION")) == ["name"] 56 | assert get_in(pod_manifest, env_var_access("RELEASE_NODE")) == ["flame_runner@$(POD_IP)"] 57 | end 58 | 59 | test "should return pod manifest with data form callback", %{ 60 | parent_pod_manifest_full: parent_pod_manifest 61 | } do 62 | callback = fn parent_pod -> 63 | ~y""" 64 | metadata: 65 | namespace: test 66 | spec: 67 | containers: 68 | - resources: 69 | limits: 70 | memory: 500Mi 71 | cpu: 500 72 | env: 73 | - name: PHX_SERVER 74 | value: "true" 75 | """ 76 | |> put_in( 77 | app_container_access(~w(resources requests)), 78 | get_in(parent_pod, app_container_access(~w(resources requests))) 79 | ) 80 | end 81 | 82 | pod_manifest = MUT.manifest(parent_pod_manifest, callback, make_ref()) 83 | 84 | assert get_in(pod_manifest, env_var_access("PHX_SERVER")) == ["true"] 85 | assert get_in(pod_manifest, app_container_access(~w(resources requests memory))) == "100Mi" 86 | assert get_in(pod_manifest, app_container_access(~w(resources limits memory))) == "500Mi" 87 | end 88 | 89 | test "should add default data to pod manifest", %{ 90 | parent_pod_manifest_full: parent_pod_manifest 91 | } do 92 | callback = fn _parent_pod -> 93 | ~y""" 94 | metadata: 95 | namespace: test 96 | spec: 97 | containers: 98 | - resources: 99 | limits: 100 | memory: 500Mi 101 | cpu: 500 102 | """ 103 | end 104 | 105 | pod_manifest = MUT.manifest(parent_pod_manifest, callback, make_ref()) 106 | assert get_in(pod_manifest, app_container_access() ++ ["image"]) == "flame-test-image:0.1.0" 107 | 108 | owner_references = get_in(pod_manifest, ~w(metadata ownerReferences)) 109 | assert length(owner_references) == 1 110 | owner_reference = List.first(owner_references) 111 | assert owner_reference["kind"] == "Pod" 112 | assert owner_reference["name"] == "flame-cb76858b7-ms8nd" 113 | end 114 | 115 | test "should take data from container with given app_container_name", %{ 116 | parent_pod_manifest_full: parent_pod_manifest 117 | } do 118 | callback = fn _parent_pod -> 119 | ~y""" 120 | metadata: 121 | namespace: test 122 | spec: 123 | containers: 124 | - resources: 125 | limits: 126 | memory: 500Mi 127 | cpu: 500 128 | """ 129 | end 130 | 131 | pod_manifest = 132 | MUT.manifest(parent_pod_manifest, callback, make_ref(), 133 | app_container_name: "other-container" 134 | ) 135 | 136 | assert get_in(pod_manifest, app_container_access() ++ ["image"]) == "other-image:0.1.0" 137 | end 138 | 139 | test "should not add ownerReferences if omitted", %{ 140 | parent_pod_manifest_full: parent_pod_manifest 141 | } do 142 | callback = fn _parent_pod -> 143 | ~y""" 144 | metadata: 145 | namespace: test 146 | spec: 147 | containers: 148 | - resources: 149 | limits: 150 | memory: 500Mi 151 | cpu: 500 152 | """ 153 | end 154 | 155 | pod_manifest = 156 | MUT.manifest(parent_pod_manifest, callback, make_ref(), omit_owner_reference: true) 157 | 158 | assert [] == get_in(pod_manifest, ~w(metadata ownerReferences)) 159 | end 160 | end 161 | 162 | describe "manifest/2 with map" do 163 | test "Uses fields defined in pod template", %{ 164 | parent_pod_manifest_full: parent_pod_manifest 165 | } do 166 | pod_template = ~y""" 167 | apiVersion: v1 168 | kind: Pod 169 | metadata: 170 | namespace: default 171 | spec: 172 | containers: 173 | - name: runner 174 | resources: 175 | requests: 176 | cpu: "1" 177 | """ 178 | 179 | pod_manifest = MUT.manifest(parent_pod_manifest, pod_template, make_ref()) 180 | 181 | assert get_in(pod_manifest, app_container_access(~w(name))) == "runner" 182 | assert get_in(pod_manifest, app_container_access(~w(resources requests cpu))) == "1" 183 | end 184 | end 185 | 186 | describe "manifest/2 with empty %RunnerPodTemplate{} struct" do 187 | test "Uses parent pod's values for empty template opts", %{ 188 | parent_pod_manifest_full: parent_pod_manifest 189 | } do 190 | template_opts = %MUT{} 191 | pod_manifest = MUT.manifest(parent_pod_manifest, template_opts, make_ref()) 192 | 193 | assert get_in(pod_manifest, env_var_access("RELEASE_NODE")) == ["flame_test@$(POD_IP)"] 194 | 195 | assert get_in(pod_manifest, app_container_access("envFrom")) == [ 196 | %{"configMapRef" => %{"name" => "some-config-map"}} 197 | ] 198 | end 199 | 200 | test "Only default envs if add_parent_env is set to false", %{ 201 | parent_pod_manifest_full: parent_pod_manifest 202 | } do 203 | ref = make_ref() 204 | template_opts = %MUT{add_parent_env: false} 205 | pod_manifest = MUT.manifest(parent_pod_manifest, template_opts, ref) 206 | 207 | assert get_in(pod_manifest, app_container_access(~w(resources requests memory))) == "100Mi" 208 | assert get_in(pod_manifest, env_var_access("PHX_SERVER")) == ["false"] 209 | assert get_in(pod_manifest, app_container_access("envFrom")) == [] 210 | 211 | parent = flame_parent(pod_manifest) 212 | assert parent.ref == ref 213 | end 214 | end 215 | 216 | describe "manifest/2 with :env set in %RunnerPodTemplate{}" do 217 | test "Parent pod's vars are mergd with given vars", %{ 218 | parent_pod_manifest_full: parent_pod_manifest 219 | } do 220 | template_opts = %MUT{env: [%{"name" => "FOO", "value" => "bar"}]} 221 | pod_manifest = MUT.manifest(parent_pod_manifest, template_opts, make_ref()) 222 | 223 | assert get_in(pod_manifest, env_var_access("RELEASE_NODE")) == ["flame_test@$(POD_IP)"] 224 | assert get_in(pod_manifest, env_var_access("FOO")) == ["bar"] 225 | 226 | assert get_in(pod_manifest, app_container_access("envFrom")) == [ 227 | %{"configMapRef" => %{"name" => "some-config-map"}} 228 | ] 229 | end 230 | 231 | test "No parent envs if add_parent_env is set to false", %{ 232 | parent_pod_manifest_full: parent_pod_manifest 233 | } do 234 | template_opts = %MUT{env: [%{"name" => "FOO", "value" => "bar"}], add_parent_env: false} 235 | pod_manifest = MUT.manifest(parent_pod_manifest, template_opts, make_ref()) 236 | 237 | assert get_in(pod_manifest, env_var_access("FOO")) == ["bar"] 238 | assert get_in(pod_manifest, app_container_access("envFrom")) == [] 239 | end 240 | end 241 | 242 | describe "manifest/2 with nil as template opts" do 243 | test "Uses parent pod's values for empty template opts", %{ 244 | parent_pod_manifest_full: parent_pod_manifest 245 | } do 246 | pod_manifest = MUT.manifest(parent_pod_manifest, nil, make_ref()) 247 | 248 | assert get_in(pod_manifest, app_container_access(~w(resources requests memory))) == "100Mi" 249 | 250 | assert get_in(pod_manifest, env_var_access("RELEASE_NODE")) == ["flame_test@$(POD_IP)"] 251 | end 252 | end 253 | end 254 | -------------------------------------------------------------------------------- /test/integration/Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | FROM hexpm/elixir:1.17.2-erlang-27.0.1-debian-bullseye-20240722-slim 3 | 4 | ENV MIX_ENV=test \ 5 | MIX_HOME=/opt/mix \ 6 | HEX_HOME=/opt/hex 7 | 8 | RUN mix local.hex --force && \ 9 | mix local.rebar --force && \ 10 | apt-get update && apt-get install -y git 11 | 12 | WORKDIR /app 13 | 14 | COPY . . 15 | 16 | RUN mix deps.get --only-prod && \ 17 | mix deps.clean --unused && \ 18 | mix deps.compile 19 | 20 | CMD ["sh", "-c", "iex --name ${RELEASE_NODE} --cookie nosecret -S mix run -e 'FlameK8sBackend.IntegrationTestRunner.runner()'"] -------------------------------------------------------------------------------- /test/integration/integration_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FlameK8sBackend.IntegrationTest do 2 | use ExUnit.Case 3 | 4 | setup_all do 5 | {clusters_out, exit_code} = System.cmd("kind", ~w(get clusters)) 6 | assert 0 == exit_code, "kind is not installed. Please install kind." 7 | 8 | if not (clusters_out 9 | |> String.split("\n", trim: true) 10 | |> Enum.member?("flame-integration-test")) do 11 | exit_code = 12 | System.cmd("kind", ~w(create cluster --name flame-integration-test), 13 | stderr_to_stdout: true 14 | ) 15 | 16 | assert 0 == exit_code, "Could not create kind cluster 'flame-integration-test'" 17 | end 18 | 19 | System.cmd( 20 | "docker", 21 | ~w(build -f test/integration/Dockerfile . -t flamek8sbackend:integration), 22 | stderr_to_stdout: true 23 | ) 24 | 25 | System.cmd( 26 | "kind", 27 | ~w(load docker-image --name flame-integration-test flamek8sbackend:integration), 28 | stderr_to_stdout: true 29 | ) 30 | 31 | System.cmd("kubectl", ~w(config set-context --current kind-flame-integration-test), 32 | stderr_to_stdout: true 33 | ) 34 | 35 | System.cmd("kubectl", ~w(delete -f test/integration/manifest.yaml), stderr_to_stdout: true) 36 | System.cmd("kubectl", ~w(apply -f test/integration/manifest.yaml), stderr_to_stdout: true) 37 | 38 | on_exit(fn -> 39 | System.cmd("kubectl", ~w(delete -f test/integration/manifest.yaml), stderr_to_stdout: true) 40 | end) 41 | 42 | :ok 43 | end 44 | 45 | defp assert_logs_eventually(_pattern, timeout) when timeout < 0 do 46 | :not_found 47 | end 48 | 49 | defp assert_logs_eventually(pattern, timeout) do 50 | with {logs, 0} <- System.cmd("kubectl", ~w"-n integration logs integration"), 51 | true <- Regex.match?(pattern, logs) do 52 | :ok 53 | else 54 | _ -> 55 | Process.sleep(300) 56 | assert_logs_eventually(pattern, timeout - 300) 57 | end 58 | end 59 | 60 | # Integration test setup builds a docker image which starts the FLAME pools 61 | # defined in FlameK8sBackend.IntegrationTestRunner and starts a runner for 62 | #  each pool. It then logs the result. These checks "just" check that the logs 63 | #  statements are actually visible on the pod logs: 64 | @tag :integration 65 | test "Pod shows the log statement with the result of the first runner " do 66 | assert :ok == assert_logs_eventually(~r/Result is :flame_ok/, 30_000), 67 | "Logs were not found" 68 | end 69 | 70 | @tag :integration 71 | test "Pod shows the log statement with the result of the first runner" do 72 | assert :ok == assert_logs_eventually(~r/Result is "foobar"/, 30_000), 73 | "Logs were not found" 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /test/integration/manifest.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: integration 6 | --- 7 | apiVersion: v1 8 | kind: ServiceAccount 9 | metadata: 10 | name: integration 11 | namespace: integration 12 | --- 13 | apiVersion: rbac.authorization.k8s.io/v1 14 | kind: Role 15 | metadata: 16 | namespace: integration 17 | name: pod-mgr 18 | rules: 19 | - apiGroups: [""] 20 | resources: ["pods"] 21 | verbs: ["create", "get", "list", "delete", "patch"] 22 | --- 23 | apiVersion: rbac.authorization.k8s.io/v1 24 | kind: RoleBinding 25 | metadata: 26 | name: integration-pod-mgr 27 | namespace: integration 28 | subjects: 29 | - kind: ServiceAccount 30 | name: integration 31 | namespace: integration 32 | roleRef: 33 | kind: Role 34 | name: pod-mgr 35 | apiGroup: rbac.authorization.k8s.io 36 | --- 37 | apiVersion: v1 38 | kind: Pod 39 | metadata: 40 | name: integration 41 | namespace: integration 42 | labels: 43 | app: flame_test 44 | spec: 45 | serviceAccountName: integration 46 | containers: 47 | - name: integration 48 | image: flamek8sbackend:integration 49 | command: ["sh", "-c"] 50 | args: 51 | [ 52 | "iex --name flame_test@$(POD_IP) --cookie $(RELEASE_COOKIE) -S mix run -e FlameK8sBackend.IntegrationTestRunner.run_flame", 53 | ] 54 | resources: 55 | requests: 56 | cpu: 300m 57 | memory: 300Mi 58 | limits: 59 | cpu: 300m 60 | memory: 300Mi 61 | env: 62 | - name: POD_NAME 63 | valueFrom: 64 | fieldRef: 65 | apiVersion: v1 66 | fieldPath: metadata.name 67 | - name: POD_NAMESPACE 68 | valueFrom: 69 | fieldRef: 70 | apiVersion: v1 71 | fieldPath: metadata.namespace 72 | - name: POD_IP 73 | valueFrom: 74 | fieldRef: 75 | apiVersion: v1 76 | fieldPath: status.podIP 77 | - name: RELEASE_DISTRIBUTION 78 | value: name 79 | - name: RELEASE_NODE 80 | value: flame_test@$(POD_IP) 81 | - name: RELEASE_COOKIE 82 | value: nosecret 83 | ports: 84 | - containerPort: 80 85 | name: integration 86 | restartPolicy: Always 87 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start(exclude: [:integration]) 2 | -------------------------------------------------------------------------------- /test_support/integration_test_runner.ex: -------------------------------------------------------------------------------- 1 | defmodule FlameK8sBackend.IntegrationTestRunner do 2 | require Logger 3 | 4 | import YamlElixir.Sigil 5 | 6 | def setup() do 7 | Application.ensure_all_started(:flame) 8 | 9 | pod_template_callback = fn _ -> 10 | ~y""" 11 | spec: 12 | containers: 13 | - env: 14 | - name: "FOO" 15 | value: "foobar" 16 | """ 17 | end 18 | 19 | children = [ 20 | { 21 | FLAME.Pool, 22 | name: IntegrationTest.Runner, 23 | min: 0, 24 | max: 2, 25 | max_concurrency: 10, 26 | boot_timeout: :timer.minutes(3), 27 | idle_shutdown_after: :timer.minutes(1), 28 | timeout: :infinity, 29 | backend: FLAMEK8sBackend, 30 | track_resources: true, 31 | log: :debug 32 | }, 33 | { 34 | FLAME.Pool, 35 | name: IntegrationTest.CallbackRunner, 36 | min: 0, 37 | max: 2, 38 | max_concurrency: 10, 39 | boot_timeout: :timer.minutes(3), 40 | idle_shutdown_after: :timer.minutes(1), 41 | timeout: :infinity, 42 | backend: {FLAMEK8sBackend, runner_pod_tpl: pod_template_callback}, 43 | track_resources: true, 44 | log: :debug 45 | } 46 | ] 47 | 48 | Supervisor.start_link(children, strategy: :one_for_one) 49 | end 50 | 51 | def run_flame() do 52 | setup() 53 | 54 | [ 55 | {IntegrationTest.Runner, fn -> :flame_ok end}, 56 | {IntegrationTest.CallbackRunner, fn -> System.get_env("FOO") end} 57 | ] 58 | |> Enum.map(fn {pool, fun} -> Task.async(fn -> FLAME.call(pool, fun) end) end) 59 | |> Task.await_many(:infinity) 60 | |> Enum.each(fn result -> Logger.info("Result is #{inspect(result)}") end) 61 | end 62 | 63 | def runner() do 64 | setup() 65 | Process.sleep(60_000) 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /test_support/pods.ex: -------------------------------------------------------------------------------- 1 | defmodule FLAMEK8sBackend.TestSupport.Pods do 2 | import YamlElixir.Sigil 3 | 4 | @parent_pod_manifest_full ~y""" 5 | apiVersion: v1 6 | kind: Pod 7 | metadata: 8 | creationTimestamp: "2023-12-13T13:44:24Z" 9 | generateName: flame-cb76858b7- 10 | labels: 11 | app: flame 12 | excluster: flame 13 | pod-template-hash: cb76858b7 14 | name: flame-cb76858b7-ms8nd 15 | namespace: test-namespace 16 | ownerReferences: 17 | - apiVersion: apps/v1 18 | blockOwnerDeletion: true 19 | controller: true 20 | kind: ReplicaSet 21 | name: flame-cb76858b7 22 | uid: 403094a3-f852-45cf-bd13-e92e98748b18 23 | resourceVersion: "179501" 24 | uid: 02d47d15-c6ac-475d-9c94-7bf8f40835c7 25 | spec: 26 | containers: 27 | - env: 28 | - name: POD_NAME 29 | valueFrom: 30 | fieldRef: 31 | apiVersion: v1 32 | fieldPath: metadata.name 33 | - name: POD_NAMESPACE 34 | valueFrom: 35 | fieldRef: 36 | apiVersion: v1 37 | fieldPath: metadata.namespace 38 | - name: POD_IP 39 | valueFrom: 40 | fieldRef: 41 | apiVersion: v1 42 | fieldPath: status.podIP 43 | - name: RELEASE_DISTRIBUTION 44 | value: name 45 | - name: RELEASE_NODE 46 | value: flame_test@$(POD_IP) 47 | envFrom: 48 | - configMapRef: 49 | name: some-config-map 50 | image: flame-test-image:0.1.0 51 | imagePullPolicy: IfNotPresent 52 | name: flame 53 | resources: 54 | requests: 55 | cpu: 100m 56 | memory: 100Mi 57 | terminationMessagePath: /dev/termination-log 58 | terminationMessagePolicy: File 59 | volumeMounts: 60 | - mountPath: /var/run/secrets/kubernetes.io/serviceaccount 61 | name: kube-api-access-smcb8 62 | readOnly: true 63 | - name: other-container 64 | image: other-image:0.1.0 65 | dnsPolicy: ClusterFirst 66 | enableServiceLinks: true 67 | nodeName: flame-test-control-plane 68 | preemptionPolicy: PreemptLowerPriority 69 | priority: 0 70 | restartPolicy: Always 71 | schedulerName: default-scheduler 72 | securityContext: {} 73 | serviceAccount: flame-test 74 | serviceAccountName: flame-test 75 | terminationGracePeriodSeconds: 30 76 | tolerations: 77 | - effect: NoExecute 78 | key: node.kubernetes.io/not-ready 79 | operator: Exists 80 | tolerationSeconds: 300 81 | - effect: NoExecute 82 | key: node.kubernetes.io/unreachable 83 | operator: Exists 84 | tolerationSeconds: 300 85 | volumes: 86 | - name: kube-api-access-smcb8 87 | projected: 88 | defaultMode: 420 89 | sources: 90 | - serviceAccountToken: 91 | expirationSeconds: 3607 92 | path: token 93 | - configMap: 94 | items: 95 | - key: ca.crt 96 | path: ca.crt 97 | name: kube-root-ca.crt 98 | - downwardAPI: 99 | items: 100 | - fieldRef: 101 | apiVersion: v1 102 | fieldPath: metadata.namespace 103 | path: namespace 104 | status: 105 | conditions: 106 | - lastProbeTime: null 107 | lastTransitionTime: "2023-12-13T13:44:24Z" 108 | status: "True" 109 | type: Initialized 110 | - lastProbeTime: null 111 | lastTransitionTime: "2023-12-13T13:44:26Z" 112 | status: "True" 113 | type: Ready 114 | - lastProbeTime: null 115 | lastTransitionTime: "2023-12-13T13:44:26Z" 116 | status: "True" 117 | type: ContainersReady 118 | - lastProbeTime: null 119 | lastTransitionTime: "2023-12-13T13:44:24Z" 120 | status: "True" 121 | type: PodScheduled 122 | containerStatuses: 123 | - containerID: containerd://ebb77cb2e138daf21613342ab9553d6544a5fad793380de5f6b853581dce5a9c 124 | image: docker.io/library/flame-test:0.2.0 125 | imageID: docker.io/library/import-2023-12-12@sha256:1cdc989cd206ba27a9faa605d6898837d2b477df05a67826a8ce7d7e50517c8f 126 | lastState: {} 127 | name: flame 128 | ready: true 129 | restartCount: 0 130 | started: true 131 | state: 132 | running: 133 | startedAt: "2023-12-13T13:44:25Z" 134 | hostIP: 172.19.0.2 135 | phase: Running 136 | podIP: 10.244.0.21 137 | podIPs: 138 | - ip: 10.244.0.21 139 | qosClass: Burstable 140 | startTime: "2023-12-13T13:44:24Z" 141 | """ 142 | 143 | def parent_pod_manifest_full(), do: @parent_pod_manifest_full 144 | end 145 | --------------------------------------------------------------------------------