├── .commitlintrc.json ├── .eslintignore ├── .eslintrc.js ├── .gcloudignore ├── .github ├── dependabot.yaml ├── release-please.yml └── workflows │ ├── codehealth.yaml │ ├── codeql.yaml │ └── unit_tests.yaml ├── .gitignore ├── .husky ├── commit-msg └── pre-commit ├── .mdl.json ├── .npmrc ├── .prettierignore ├── .prettierrc.js ├── .release-please-manifest.json ├── CHANGELOG.md ├── Dockerfile-unified ├── LICENSE ├── README.md ├── autoscaler-config.schema.json ├── cloudbuild-unified.yaml ├── code-of-conduct.md ├── configeditor ├── README.md ├── build-configeditor.sh ├── index.html └── index.mjs ├── contributing.md ├── jsconfig.json ├── kubernetes └── unified │ ├── autoscaler-config │ ├── autoscaler-config.yaml.template │ └── otel-collector.yaml.template │ └── autoscaler-pkg │ ├── Kptfile │ ├── README.md │ ├── networkpolicy.yaml │ ├── otel-collector │ ├── Kptfile │ ├── README.md │ └── otel-collector.yaml │ └── scaler │ ├── Kptfile │ ├── README.md │ └── scaler.yaml ├── markdown-link-checker.json ├── package-lock.json ├── package.json ├── release-please-config.json ├── renovate.json5 ├── resources ├── architecture-abstract.png ├── architecture-centralized.png ├── architecture-distributed.png ├── architecture-forwarder.png ├── architecture-gke-unified.png ├── architecture-per-project.png └── hero-image.jpg ├── src ├── README.md ├── autoscaler-common │ ├── assert-defined.js │ ├── config-parameters.js │ ├── counters-base.js │ ├── logger.js │ ├── promiseWithResolvers.js │ └── types.js ├── forwarder │ ├── README.md │ └── index.js ├── functions.js ├── poller │ ├── README.md │ └── poller-core │ │ ├── config-validator.js │ │ ├── counters.js │ │ ├── index.js │ │ └── test │ │ ├── config-validator.test.js │ │ ├── index.test.js │ │ └── resources │ │ ├── bad-data-contents.yaml │ │ ├── bad-empty-array.json │ │ ├── bad-empty.json │ │ ├── bad-empty.yaml │ │ ├── bad-invalid-props.json │ │ ├── bad-invalid-props.yaml │ │ ├── bad-invalid-value.json │ │ ├── bad-invalid-value.yaml │ │ ├── bad-missing-props.json │ │ ├── bad-missing-props.yaml │ │ ├── bad-not-configmap.yaml │ │ ├── bad-not-yaml.yaml │ │ ├── good-config.json │ │ ├── good-config.yaml │ │ └── good-multi-config.yaml ├── scaler │ ├── README.md │ └── scaler-core │ │ ├── counters.js │ │ ├── downstream.schema.proto │ │ ├── index.js │ │ ├── scaling-methods │ │ ├── base.js │ │ ├── direct.js │ │ ├── linear.js │ │ └── stepwise.js │ │ ├── scaling-profiles │ │ ├── profiles │ │ │ ├── README.md │ │ │ ├── cpu.js │ │ │ ├── cpu_and_memory.js │ │ │ └── memory.js │ │ └── rules │ │ │ ├── README.md │ │ │ ├── cpu │ │ │ ├── README.md │ │ │ ├── cpu-high-average-utilization.js │ │ │ ├── cpu-high-maximum-utilization.js │ │ │ ├── cpu-low-average-utilization.js │ │ │ └── cpu-low-maximum-utilization.js │ │ │ └── memory │ │ │ ├── README.md │ │ │ ├── memory-high-average-utilization.js │ │ │ ├── memory-high-maximum-utilization.js │ │ │ ├── memory-low-average-utilization.js │ │ │ └── memory-low-maximum-utilization.js │ │ ├── state.js │ │ ├── test │ │ ├── counters.test.js │ │ ├── index.test.js │ │ ├── samples │ │ │ ├── custom-scaling-rules.json │ │ │ ├── downstream-msg.json │ │ │ └── parameters.json │ │ ├── scaling-methods │ │ │ ├── base.test.js │ │ │ ├── direct.test.js │ │ │ ├── linear.test.js │ │ │ └── stepwise.test.js │ │ ├── state.test.js │ │ ├── test-utils.js │ │ └── utils.test.js │ │ └── utils.js └── unified-scaler.js └── terraform ├── README.md ├── cloud-functions ├── README.md ├── centralized │ └── README.md ├── distributed │ ├── README.md │ ├── app-project │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── outputs.tf │ │ └── variables.tf │ └── autoscaler-project │ │ ├── .terraform.lock.hcl │ │ ├── main.tf │ │ ├── outputs.tf │ │ └── variables.tf └── per-project │ ├── .terraform.lock.hcl │ ├── README.md │ ├── main.tf │ ├── outputs.tf │ ├── test │ ├── go.mod │ ├── go.sum │ └── per_project_e2e_test.go │ └── variables.tf ├── gke ├── README.md └── unified │ ├── .terraform.lock.hcl │ ├── README.md │ ├── main.tf │ ├── outputs.tf │ ├── test │ ├── gke_deploy.sh │ ├── gke_e2e_test.go │ ├── go.mod │ └── go.sum │ └── variables.tf └── modules ├── autoscaler-base ├── main.tf ├── outputs.tf └── variables.tf ├── autoscaler-firestore ├── main.tf └── variables.tf ├── autoscaler-forwarder ├── main.tf ├── outputs.tf └── variables.tf ├── autoscaler-functions ├── main.tf ├── outputs.tf └── variables.tf ├── autoscaler-gke ├── main.tf ├── outputs.tf └── variables.tf ├── autoscaler-memorystore-cluster ├── main.tf ├── outputs.tf └── variables.tf ├── autoscaler-monitoring ├── dashboard.json.tftpl ├── main.tf ├── outputs.tf └── variables.tf ├── autoscaler-network ├── main.tf ├── outputs.tf └── variables.tf ├── autoscaler-scheduler ├── main.tf ├── outputs.tf └── variables.tf ├── autoscaler-spanner ├── main.tf └── variables.tf └── autoscaler-test-vm ├── main.tf ├── outputs.tf ├── scripts └── startup.sh └── variables.tf /.commitlintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "comment": "Disable some rules to match the config of conventional-commit-GCF app", 3 | "extends": ["@commitlint/config-conventional"], 4 | "rules": { 5 | "body-case": [0], 6 | "body-max-line-length": [0], 7 | "footer-max-line-length": [0], 8 | "header-max-length": [0], 9 | "subject-case": [0], 10 | "subject-full-stop": [0] 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | .next 2 | next-env.d.ts 3 | node_modules 4 | yarn.lock 5 | package-lock.json 6 | public 7 | configeditor/build 8 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module.exports = { 18 | 'env': { 19 | 'browser': true, 20 | 'commonjs': true, 21 | 'es2021': true, 22 | }, 23 | 'extends': ['google', 'plugin:prettier/recommended'], 24 | 'overrides': [ 25 | { 26 | 'env': { 27 | 'node': true, 28 | }, 29 | 'files': ['.eslintrc.{js,cjs}'], 30 | 'parserOptions': { 31 | 'sourceType': 'script', 32 | }, 33 | }, 34 | ], 35 | 'parserOptions': { 36 | 'ecmaVersion': 'latest', 37 | }, 38 | 'rules': {}, 39 | }; 40 | -------------------------------------------------------------------------------- /.gcloudignore: -------------------------------------------------------------------------------- 1 | # This file specifies files that are *not* uploaded to Google Cloud Platform 2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of 3 | # "#!include" directives (which insert the entries of the given .gitignore-style 4 | # file at that point). 5 | # 6 | # For more information, run: 7 | # $ gcloud topic gcloudignore 8 | # 9 | .gcloudignore 10 | 11 | .git 12 | .gitignore 13 | 14 | .github 15 | .nyc_output 16 | .vscode 17 | kubernetes 18 | node_modules 19 | resources 20 | terraform 21 | test/ 22 | .eslint* 23 | .husky 24 | .mdl* 25 | .prettier* 26 | *release-please* 27 | *.md 28 | configeditor/ 29 | 30 | #!include:.gitignore 31 | -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | updates: 17 | # Github-actions dependencies 18 | - directory: "/" 19 | package-ecosystem: "github-actions" 20 | schedule: 21 | interval: "weekly" 22 | commit-message: 23 | prefix: "fix" 24 | # Use Renovate for version updates, Dependabot for security updates only: 25 | open-pull-requests-limit: 0 26 | 27 | # NPM dependencies -- only prompt to update minor versions. 28 | - directory: "/" 29 | package-ecosystem: "npm" 30 | schedule: 31 | interval: "weekly" 32 | ignore: 33 | - dependency-name: "*" 34 | update-types: ["version-update:semver-major"] 35 | commit-message: 36 | prefix: "fix" 37 | # Use Renovate for version updates, Dependabot for security updates only: 38 | open-pull-requests-limit: 0 39 | 40 | # Docker dependencies 41 | - directory: "/" 42 | package-ecosystem: "docker" 43 | schedule: 44 | interval: "weekly" 45 | commit-message: 46 | prefix: "fix" 47 | # Use Renovate for version updates, Dependabot for security updates only: 48 | open-pull-requests-limit: 0 49 | 50 | # Terraform dependencies 51 | - directory: "/terraform" 52 | package-ecosystem: "terraform" 53 | schedule: 54 | interval: "weekly" 55 | commit-message: 56 | prefix: "fix" 57 | # Use Renovate for version updates, Dependabot for security updates only: 58 | open-pull-requests-limit: 0 59 | 60 | version: 2 61 | -------------------------------------------------------------------------------- /.github/release-please.yml: -------------------------------------------------------------------------------- 1 | handleGHRelease: true 2 | manifest: true 3 | -------------------------------------------------------------------------------- /.github/workflows/codehealth.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: "Code health checks (npm audit, eslint, tscompiler, ...)" 16 | 17 | on: 18 | push: 19 | pull_request: 20 | 21 | jobs: 22 | analyze: 23 | name: "Analyze" 24 | runs-on: ubuntu-latest 25 | permissions: 26 | actions: read 27 | contents: read 28 | strategy: 29 | fail-fast: false 30 | 31 | steps: 32 | - name: Checkout repository 33 | uses: actions/checkout@v4 34 | 35 | - name: Use Node.js 36 | uses: actions/setup-node@v4 37 | with: 38 | node-version: 20 39 | check-latest: true 40 | 41 | - name: Use terraform 42 | uses: hashicorp/setup-terraform@v3 43 | 44 | - name: Install node modules 45 | run: npm install 46 | 47 | - name: Execute "npm run typecheck" 48 | run: npm run typecheck 49 | 50 | - name: Execute "npm run eslint" 51 | run: npm run eslint 52 | 53 | - name: Execute "npm run check-format" 54 | run: npm run check-format 55 | 56 | - name: Execute "npm run mdlint" 57 | run: npm run mdlint 58 | 59 | - name: Execute "npm audit" 60 | run: npm audit 61 | 62 | - name: Execute "npm run markdown-link-check" 63 | run: npm run markdown-link-check 64 | 65 | - name: terraform validate deployments 66 | run: npm run terraform-validate 67 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: "CodeQL" 16 | 17 | on: 18 | push: 19 | pull_request: 20 | schedule: 21 | - cron: "27 22 * * 5" 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: ["javascript"] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v4 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v3 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). 56 | # If this step fails, then you should remove it and run the build manually (see below) 57 | - name: Autobuild 58 | uses: github/codeql-action/autobuild@v3 59 | 60 | # ℹ️ Command-line programs to run using the OS shell. 61 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 62 | 63 | # If the Autobuild fails above, remove it and uncomment the following three lines. 64 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 65 | 66 | # - run: | 67 | # echo "Run, Build Application using script" 68 | # ./location_of_script_within_repo/buildscript.sh 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@v3 72 | with: 73 | category: "/language:${{matrix.language}}" 74 | -------------------------------------------------------------------------------- /.github/workflows/unit_tests.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: Node.js unit tests 16 | 17 | on: [push, pull_request] 18 | 19 | jobs: 20 | unit-tests: 21 | runs-on: ubuntu-latest 22 | 23 | defaults: 24 | run: 25 | working-directory: src/poller/poller-core 26 | 27 | strategy: 28 | matrix: 29 | node-version: [20.x, 22.x] 30 | 31 | steps: 32 | - uses: actions/checkout@v4 33 | 34 | - name: Use Node.js ${{ matrix.node-version }} 35 | uses: actions/setup-node@v4 36 | with: 37 | node-version: ${{ matrix.node-version }} 38 | 39 | - name: npm install 40 | run: npm install 41 | 42 | - name: Unit tests 43 | run: npm test 44 | env: 45 | CI: true 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Auto-generated when installing Node packages, e.g. CF emulator 2 | node_modules 3 | 4 | # General 5 | tmp 6 | *.swp 7 | *.swo 8 | .DS_Store 9 | 10 | # https://www.gitignore.io/api/visualstudiocode 11 | .vscode 12 | .vscode/* 13 | !.vscode/settings.json 14 | !.vscode/tasks.json 15 | !.vscode/launch.json 16 | !.vscode/extensions.json 17 | 18 | ### VisualStudioCode Patch ### 19 | # Ignore all local history of files 20 | .history 21 | 22 | # Misc 23 | setenv.sh 24 | out 25 | 26 | # Terraform 27 | *.tfstate 28 | *.tfstate.backup 29 | *.tfstate.lock.info 30 | *.tfplan 31 | .terraform 32 | terraform/*/build 33 | terraform/*/*.json 34 | terraform/*/*/build 35 | terraform/*/*/*/build 36 | terraform/*/*/*.json 37 | !dashboard.json 38 | 39 | # Code coverage report 40 | .nyc_output 41 | 42 | # Kubernetes manifests generated from templates 43 | kubernetes/**/autoscaler-config/*.yaml 44 | kubernetes/**/resourcegroup.yaml 45 | 46 | # Terratest 47 | .test-data 48 | 49 | configeditor/build 50 | -------------------------------------------------------------------------------- /.husky/commit-msg: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # From Gerrit Code Review 3.9.2-695-gc36e51bbb2 3 | # 4 | # Part of Gerrit Code Review (https://www.gerritcodereview.com/) 5 | # 6 | # Copyright (C) 2009 The Android Open Source Project 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | 20 | set -u 21 | set +e 22 | 23 | echo "" 24 | echo "running .husky/commit-msg checks" 25 | echo "" 26 | 27 | # avoid [[ which is not POSIX sh. 28 | if test "$#" != 1 ; then 29 | echo "$0 requires an argument." 30 | exit 1 31 | fi 32 | 33 | if test ! -f "$1" ; then 34 | echo "file does not exist: $1" 35 | exit 1 36 | fi 37 | 38 | # Run conventional-commit checks 39 | # 40 | if ! npx @commitlint/cli -e $1 ; then 41 | echo "Conventional commit message checks failed" 42 | exit 1 43 | fi 44 | 45 | # Do not create a change id if requested 46 | case "$(git config --get gerrit.createChangeId)" in 47 | false) 48 | exit 0 49 | ;; 50 | always) 51 | ;; 52 | *) 53 | # Do not create a change id for squash/fixup commits. 54 | if head -n1 "$1" | LC_ALL=C grep -q '^[a-z][a-z]*! '; then 55 | exit 0 56 | fi 57 | ;; 58 | esac 59 | 60 | 61 | if git rev-parse --verify HEAD >/dev/null 2>&1; then 62 | refhash="$(git rev-parse HEAD)" 63 | else 64 | refhash="$(git hash-object -t tree /dev/null)" 65 | fi 66 | 67 | random=$({ git var GIT_COMMITTER_IDENT ; echo "$refhash" ; cat "$1"; } | git hash-object --stdin) 68 | dest="$1.tmp.${random}" 69 | 70 | trap 'rm -f "$dest" "$dest-2"' EXIT 71 | 72 | if ! cat "$1" | sed -e '/>8/q' | git stripspace --strip-comments > "${dest}" ; then 73 | echo "cannot strip comments from $1" 74 | exit 1 75 | fi 76 | 77 | if test ! -s "${dest}" ; then 78 | echo "file is empty: $1" 79 | exit 1 80 | fi 81 | 82 | reviewurl="$(git config --get gerrit.reviewUrl)" 83 | if test -n "${reviewurl}" ; then 84 | token="Link" 85 | value="${reviewurl%/}/id/I$random" 86 | pattern=".*/id/I[0-9a-f]\{40\}" 87 | else 88 | token="Change-Id" 89 | value="I$random" 90 | pattern=".*" 91 | fi 92 | 93 | if git interpret-trailers --parse < "$1" | grep -q "^$token: $pattern$" ; then 94 | exit 0 95 | fi 96 | 97 | # There must be a Signed-off-by trailer for the code below to work. Insert a 98 | # sentinel at the end to make sure there is one. 99 | # Avoid the --in-place option which only appeared in Git 2.8 100 | if ! git interpret-trailers \ 101 | --trailer "Signed-off-by: SENTINEL" < "$1" > "$dest-2" ; then 102 | echo "cannot insert Signed-off-by sentinel line in $1" 103 | exit 1 104 | fi 105 | 106 | # Make sure the trailer appears before any Signed-off-by trailers by inserting 107 | # it as if it was a Signed-off-by trailer and then use sed to remove the 108 | # Signed-off-by prefix and the Signed-off-by sentinel line. 109 | # Avoid the --in-place option which only appeared in Git 2.8 110 | # Avoid the --where option which only appeared in Git 2.15 111 | if ! git -c trailer.where=before interpret-trailers \ 112 | --trailer "Signed-off-by: $token: $value" < "$dest-2" | 113 | sed -e "s/^Signed-off-by: \($token: \)/\1/" \ 114 | -e "/^Signed-off-by: SENTINEL/d" > "$dest" ; then 115 | echo "cannot insert $token line in $1" 116 | exit 1 117 | fi 118 | 119 | if ! mv "${dest}" "$1" ; then 120 | echo "cannot mv ${dest} to $1" 121 | exit 1 122 | fi 123 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Running .husky/pre-commit checks. Use -n/--no-verify to skip" 4 | echo "------------------------------------------------------------" 5 | 6 | npm run prettier-check 7 | npm run eslint 8 | 9 | function hasModifiedMatching() { 10 | [[ -z "$1" ]] && echo "hasModifiedMatching needs arg" && return 1 11 | git status --short --untracked-files=all --column=never | grep -q "$1" 12 | return $? 13 | } 14 | 15 | # check for modified markdown? 16 | if hasModifiedMatching '\.md$' ; then 17 | echo "Markdown files modified... running checks" 18 | npm run markdown-link-check 19 | npm run mdlint 20 | fi 21 | 22 | if hasModifiedMatching ' src/' ; then 23 | echo "src files modified... running checks" 24 | npm run typecheck 25 | npm test 26 | fi 27 | 28 | 29 | if hasModifiedMatching '\.tf$' ; then 30 | echo "Terraform files modified... running checks" 31 | npm run terraform-fmt-check 32 | npm run terraform-validate 33 | fi 34 | -------------------------------------------------------------------------------- /.mdl.json: -------------------------------------------------------------------------------- 1 | { 2 | "default": true, 3 | "MD033": false, 4 | "MD041": false, 5 | "MD002": false, 6 | "MD004": { "style": "asterisk" }, 7 | "MD007": { "indent": 4 }, 8 | "MD013": { 9 | "ignore_code_blocks": true, 10 | "code_blocks": false, 11 | "tables": false 12 | }, 13 | "MD029": { "style": "ordered" }, 14 | "MD030": { "ul_single": 3, "ul_multi": 3, "ol_single": 2, "ol_multi": 2 } 15 | } 16 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | engine-strict=true 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | .next 2 | next-env.d.ts 3 | node_modules 4 | yarn.lock 5 | package-lock.json 6 | public 7 | *.md 8 | configeditor/build 9 | -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | const shared = { 17 | printWidth: 80, 18 | tabWidth: 2, 19 | useTabs: false, 20 | semi: true, 21 | singleQuote: true, 22 | quoteProps: 'preserve', 23 | bracketSpacing: false, 24 | trailingComma: 'all', 25 | arrowParens: 'always', 26 | embeddedLanguageFormatting: 'off', 27 | bracketSameLine: true, 28 | singleAttributePerLine: false, 29 | jsxSingleQuote: false, 30 | htmlWhitespaceSensitivity: 'strict', 31 | }; 32 | 33 | module.exports = { 34 | overrides: [ 35 | { 36 | /** TSX/TS/JS-specific configuration. */ 37 | files: '*.tsx', 38 | options: shared, 39 | }, 40 | { 41 | files: '*.ts', 42 | options: shared, 43 | }, 44 | { 45 | files: '*.js', 46 | options: shared, 47 | }, 48 | { 49 | /** Sass-specific configuration. */ 50 | files: '*.scss', 51 | options: { 52 | singleQuote: true, 53 | }, 54 | }, 55 | { 56 | files: '*.html', 57 | options: { 58 | printWidth: 100, 59 | }, 60 | }, 61 | { 62 | files: '*.acx.html', 63 | options: { 64 | parser: 'angular', 65 | singleQuote: true, 66 | }, 67 | }, 68 | { 69 | files: '*.ng.html', 70 | options: { 71 | parser: 'angular', 72 | singleQuote: true, 73 | printWidth: 100, 74 | }, 75 | }, 76 | ], 77 | }; 78 | -------------------------------------------------------------------------------- /.release-please-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | ".": "3.0.0" 3 | } 4 | -------------------------------------------------------------------------------- /Dockerfile-unified: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ARG NODE_VERSION=20 16 | FROM node:${NODE_VERSION}-alpine AS build-env 17 | 18 | WORKDIR /usr/src/app 19 | COPY src/autoscaler-common/ src/autoscaler-common/ 20 | COPY src/scaler/scaler-core/ src/scaler/scaler-core/ 21 | COPY src/poller/poller-core/ src/poller/poller-core/ 22 | COPY src/unified-scaler.js src/ 23 | COPY package*.json ./ 24 | COPY autoscaler-config.schema.json ./ 25 | RUN npm config set update-notifier false 26 | RUN npm install --omit=dev 27 | RUN find /usr/src/app/ -type d -exec chmod a+x '{}' ';' 28 | RUN find /usr/src/app/ -type f -name '*.js*' -exec chmod a+r '{}' ';' 29 | 30 | FROM gcr.io/distroless/nodejs${NODE_VERSION}:nonroot 31 | COPY --from=build-env /usr/src/app /usr/src/app 32 | WORKDIR /usr/src/app/ 33 | 34 | CMD ["-e", "require('./src/unified-scaler').main()"] 35 | -------------------------------------------------------------------------------- /cloudbuild-unified.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | steps: 16 | - name: "gcr.io/cloud-builders/docker" 17 | args: 18 | [ 19 | "build", 20 | "--tag=$LOCATION-docker.pkg.dev/$PROJECT_ID/memorystore-cluster-autoscaler/scaler", 21 | "-f", 22 | "Dockerfile-unified", 23 | ".", 24 | ] 25 | images: 26 | ["$LOCATION-docker.pkg.dev/$PROJECT_ID/memorystore-cluster-autoscaler/scaler"] 27 | options: 28 | logging: "CLOUD_LOGGING_ONLY" 29 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Google Open Source Community Guidelines 2 | 3 | At Google, we recognize and celebrate the creativity and collaboration of open 4 | source contributors and the diversity of skills, experiences, cultures, and 5 | opinions they bring to the projects and communities they participate in. 6 | 7 | Every one of Google's open source projects and communities are inclusive 8 | environments, based on treating all individuals respectfully, regardless of 9 | gender identity and expression, sexual orientation, disabilities, 10 | neurodiversity, physical appearance, body size, ethnicity, nationality, race, 11 | age, religion, or similar personal characteristic. 12 | 13 | We value diverse opinions, but we value respectful behavior more. 14 | 15 | Respectful behavior includes: 16 | 17 | * Being considerate, kind, constructive, and helpful. 18 | * Not engaging in demeaning, discriminatory, harassing, hateful, sexualized, or 19 | physically threatening behavior, speech, and imagery. 20 | * Not engaging in unwanted physical contact. 21 | 22 | Some Google open source projects [may adopt][] an explicit project code of 23 | conduct, which may have additional detailed expectations for participants. Most 24 | of those projects will use our [modified Contributor Covenant][]. 25 | 26 | [may adopt]: https://opensource.google/docs/releasing/preparing/#conduct 27 | [modified Contributor Covenant]: https://opensource.google/docs/releasing/template/CODE_OF_CONDUCT/ 28 | 29 | ## Resolve peacefully 30 | 31 | We do not believe that all conflict is necessarily bad; healthy debate and 32 | disagreement often yields positive results. However, it is never okay to be 33 | disrespectful. 34 | 35 | If you see someone behaving disrespectfully, you are encouraged to address the 36 | behavior directly with those involved. Many issues can be resolved quickly and 37 | easily, and this gives people more control over the outcome of their dispute. 38 | If you are unable to resolve the matter for any reason, or if the behavior is 39 | threatening or harassing, report it. We are dedicated to providing an 40 | environment where participants feel welcome and safe. 41 | 42 | ## Reporting problems 43 | 44 | Some Google open source projects may adopt a project-specific code of conduct. 45 | In those cases, a Google employee will be identified as the Project Steward, 46 | who will receive and handle reports of code of conduct violations. In the event 47 | that a project hasn’t identified a Project Steward, you can report problems by 48 | emailing opensource@google.com. 49 | 50 | We will investigate every complaint, but you may not receive a direct response. 51 | We will use our discretion in determining when and how to follow up on reported 52 | incidents, which may range from not taking action to permanent expulsion from 53 | the project and project-sponsored spaces. We will notify the accused of the 54 | report and provide them an opportunity to discuss it before any action is 55 | taken. The identity of the reporter will be omitted from the details of the 56 | report supplied to the accused. In potentially harmful situations, such as 57 | ongoing harassment or threats to anyone's safety, we may take action without 58 | notice. 59 | 60 | *This document was adapted from the [IndieWeb Code of Conduct][] and can also 61 | be found at .* 62 | 63 | [IndieWeb Code of Conduct]: https://indieweb.org/code-of-conduct 64 | -------------------------------------------------------------------------------- /configeditor/README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 |

OSS Memorystore Cluster Autoscaler

4 | Autoscaler 5 | 6 |

7 | Validating editor for Autoscaler configuration. 8 |
9 | Home 10 | · 11 | Scaler component 12 | · 13 | Poller component 14 | · 15 | Forwarder component 16 | · 17 | Terraform configuration 18 | · 19 | Monitoring 20 |

21 | 22 | ## Overview 23 | 24 | This directory contains a simple web-based autoscaler config file editor that 25 | validates that the JSON config is correct - both for JSON syntax errors and that 26 | the config has the correct set of parameters and values. 27 | 28 | For GKE configurations, a YAML ConfigMap equivalent is displayed below. 29 | 30 | While directly editing the YAML configMap for GKE is not supported in this 31 | editor, you can paste the configmap into the JSON editor, and it will be 32 | converted to JSON for editing and validation, with the equivalent YAML shown 33 | below. 34 | 35 | ## Usage 36 | 37 | Build the editor and start the HTTP server on port `8080`: 38 | 39 | ```sh 40 | npm run start-configeditor-server -- --port 8080 41 | ``` 42 | 43 | Then browse to `http://127.0.0.1:8080/` 44 | 45 | ## Command line config validation 46 | 47 | The JSON and YAML configurations can also be validated using the command line: 48 | 49 | ```sh 50 | npm install 51 | npm run validate-config-file -- path/to/config_file 52 | ``` 53 | -------------------------------------------------------------------------------- /configeditor/build-configeditor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2024 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | # 18 | set -e 19 | 20 | SCRIPTDIR=$(dirname "$0") 21 | cd "$SCRIPTDIR" 22 | 23 | npm install --quiet 24 | mkdir -p build/vanilla-jsoneditor 25 | JSONEDITOR_JS=build/vanilla-jsoneditor/standalone.js 26 | # renovate: datasource=npm packageName=vanilla-jsoneditor 27 | JSONEDITOR_VERSION=0.23.8 28 | JSONEDITOR_JS_URL="https://cdn.jsdelivr.net/npm/vanilla-jsoneditor@${JSONEDITOR_VERSION}/standalone.js" 29 | # sha256sum of file at $JSONEDITOR_JS_URL 30 | JSONEDITOR_JS_HASH="91886177f9cab8541f73e02aa195fcea27089acfdf5be48b20ed60f65543f6cf" 31 | if [[ ! -e "$JSONEDITOR_JS" ]]; then 32 | echo "Downloading npm/vanilla-jsoneditor@${JSONEDITOR_VERSION}/standalone.js" 33 | curl -s -o "$JSONEDITOR_JS" "$JSONEDITOR_JS_URL" 34 | 35 | # Check sha256sum hash 36 | if ! echo "$JSONEDITOR_JS_HASH $JSONEDITOR_JS" \ 37 | | sha256sum --check --quiet ; then 38 | echo "" 39 | echo "FAILED $JSONEDITOR_JS Checksum does not match expected value" 40 | rm "$JSONEDITOR_JS" 41 | exit 1 42 | fi 43 | fi 44 | 45 | cp -r ../node_modules/js-yaml ../autoscaler-config.schema.json build/ 46 | 47 | [[ "$1" == "--quiet" ]] || cat < 2 | 3 | 18 | 19 | Autoscaler config file editor 20 | 21 | 22 | 23 | 24 | 25 | 26 |

JSON Autoscaler config file editor

27 |

Copy/Paste your YAML or JSON autoscaler config in the editor below.

28 |

The configuration will automatically be validated and any errors shown

29 |
30 | Loading... 31 |
32 | If this Loading message does not disappear, check that you have run 33 | ./build-configeditor.sh 34 |
35 |
36 |
37 |

Equivalent GKE configmap YAML

38 | 43 |

44 | Powered by jsoneditoronline.org 45 |

46 | 47 | 48 | -------------------------------------------------------------------------------- /contributing.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows [Google's Open Source Community 28 | Guidelines](https://opensource.google/conduct/). 29 | -------------------------------------------------------------------------------- /jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "ES6", 5 | "checkJs": true, 6 | "allowJs": true, 7 | "noEmit": true, 8 | "strict": true, 9 | "resolveJsonModule": true 10 | }, 11 | "include": ["src/**/*"] 12 | } 13 | -------------------------------------------------------------------------------- /kubernetes/unified/autoscaler-config/autoscaler-config.yaml.template: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: ConfigMap 17 | metadata: 18 | name: autoscaler-config 19 | namespace: memorystore-cluster-autoscaler 20 | data: 21 | autoscaler-config.yaml: | 22 | --- 23 | - projectId: ${PROJECT_ID} 24 | regionId: ${REGION} 25 | clusterId: autoscaler-target-memorystore-cluster 26 | # Delete this stanza if using Firestore for state 27 | stateDatabase: 28 | name: spanner 29 | instanceId: memorystore-autoscaler-state 30 | databaseId: memorystore-autoscaler-state 31 | scalingProfile: CPU_AND_MEMORY 32 | scalingMethod: STEPWISE 33 | units: SHARDS 34 | minSize: 1 35 | maxSize: 10 36 | -------------------------------------------------------------------------------- /kubernetes/unified/autoscaler-config/otel-collector.yaml.template: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: ConfigMap 17 | metadata: 18 | name: otel-config 19 | namespace: memorystore-cluster-autoscaler 20 | data: 21 | config.yaml: | 22 | --- 23 | receivers: 24 | otlp: 25 | protocols: 26 | grpc: 27 | endpoint: 0.0.0.0:4317 28 | 29 | processors: 30 | resourcedetection: 31 | detectors: [gcp] 32 | timeout: 10s 33 | override: false 34 | 35 | batch: 36 | # batch metrics before sending to reduce API usage 37 | send_batch_max_size: 200 38 | send_batch_size: 200 39 | # NOTE: If batching timeout is greater than the frequency of which 40 | # metrics from long running processes are pushed to the OTEL collector, 41 | # Duplicate TimeSeries errors can occur as muliple metrics pushes 42 | # are exported. 43 | # NOTE: If using Google Cloud Monitoring exporter, then the minimum 44 | # batching time is 5 seconds. 45 | timeout: 10s 46 | 47 | memory_limiter: 48 | # drop metrics if memory usage gets too high 49 | check_interval: 1s 50 | limit_percentage: 65 51 | spike_limit_percentage: 20 52 | 53 | exporters: 54 | googlecloud: 55 | timeout: 45s 56 | # Enable the debug exporter, and add to expoters pipeline to see the metrics being delivered 57 | # debug: 58 | # verbosity: detailed 59 | 60 | service: 61 | pipelines: 62 | metrics: 63 | receivers: [otlp] 64 | processors: [resourcedetection, batch, memory_limiter] 65 | # If using the debug exporter, add it to the following list 66 | exporters: [googlecloud] 67 | telemetry: 68 | logs: 69 | # Change log level from "info" to "debug" to view detailed logs 70 | level: "info" 71 | -------------------------------------------------------------------------------- /kubernetes/unified/autoscaler-pkg/Kptfile: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: kpt.dev/v1 15 | kind: Kptfile 16 | metadata: 17 | name: autoscaler-pkg 18 | annotations: 19 | config.kubernetes.io/local-config: "true" 20 | info: 21 | description: Config for Memorystore Cluster autoscaler 22 | -------------------------------------------------------------------------------- /kubernetes/unified/autoscaler-pkg/README.md: -------------------------------------------------------------------------------- 1 | # autoscaler-pkg 2 | 3 | ## Description 4 | 5 | Config for Memorystore Cluster Autoscaler 6 | 7 | ### View package content 8 | 9 | `kpt pkg tree autoscaler-pkg` 10 | [Details](https://kpt.dev/reference/cli/pkg/tree/) 11 | 12 | ## Installation 13 | 14 | See [documentation][docs] for installation and configuration instructions. 15 | 16 | [docs]: ../../../terraform/gke/README.md 17 | -------------------------------------------------------------------------------- /kubernetes/unified/autoscaler-pkg/networkpolicy.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | apiVersion: networking.k8s.io/v1 16 | kind: NetworkPolicy 17 | metadata: 18 | name: default-deny-all 19 | namespace: memorystore-cluster-autoscaler # kpt-set: ${namespace} 20 | spec: 21 | podSelector: {} 22 | policyTypes: 23 | - Ingress 24 | --- 25 | apiVersion: networking.k8s.io/v1 26 | kind: NetworkPolicy 27 | metadata: 28 | name: allow-otel-submitter-to-collector 29 | namespace: memorystore-cluster-autoscaler # kpt-set: ${namespace} 30 | spec: 31 | podSelector: 32 | matchLabels: 33 | app: otel-collector 34 | policyTypes: 35 | - Ingress 36 | ingress: 37 | - from: 38 | - podSelector: 39 | matchLabels: 40 | otel-submitter: "true" 41 | ports: 42 | - protocol: TCP 43 | port: 4317 44 | -------------------------------------------------------------------------------- /kubernetes/unified/autoscaler-pkg/otel-collector/Kptfile: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: kpt.dev/v1 15 | kind: Kptfile 16 | metadata: 17 | name: otel-collector 18 | annotations: 19 | config.kubernetes.io/local-config: "true" 20 | info: 21 | description: Config for OpenTelemetry Collector component of Memorystore Cluster autoscaler 22 | -------------------------------------------------------------------------------- /kubernetes/unified/autoscaler-pkg/otel-collector/README.md: -------------------------------------------------------------------------------- 1 | # Open Telemtry Collector 2 | 3 | ## Description 4 | 5 | Pod config for [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) 6 | component of Memorystore Cluster Autoscaler 7 | 8 | ### View package content 9 | 10 | `kpt pkg tree otel-collector` 11 | [Details](https://kpt.dev/reference/cli/pkg/tree/) 12 | 13 | ## Installation 14 | 15 | See [documentation][docs] for installation and configuration instructions. 16 | 17 | [docs]: ../../../../terraform/gke/unified/README.md 18 | -------------------------------------------------------------------------------- /kubernetes/unified/autoscaler-pkg/otel-collector/otel-collector.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: apps/v1 15 | kind: Deployment 16 | metadata: 17 | name: otel-collector 18 | namespace: memorystore-cluster-autoscaler # kpt-set: ${namespace} 19 | labels: 20 | app: otel-collector 21 | spec: 22 | replicas: 1 23 | selector: 24 | matchLabels: 25 | app: otel-collector 26 | template: 27 | metadata: 28 | labels: 29 | app: otel-collector 30 | spec: 31 | containers: 32 | - name: otel-collector 33 | image: otel/opentelemetry-collector-contrib:0.93.0 34 | resources: 35 | requests: 36 | memory: "128Mi" 37 | cpu: "250m" 38 | limits: 39 | memory: "256Mi" 40 | args: 41 | - --config 42 | - /etc/otel/config.yaml 43 | securityContext: 44 | allowPrivilegeEscalation: false 45 | readOnlyRootFilesystem: true 46 | runAsNonRoot: true 47 | capabilities: 48 | drop: 49 | - all 50 | volumeMounts: 51 | - mountPath: /etc/otel/ 52 | name: otel-config 53 | volumes: 54 | - name: otel-config 55 | configMap: 56 | name: otel-config 57 | nodeSelector: 58 | iam.gke.io/gke-metadata-server-enabled: "true" 59 | serviceAccountName: otel-collector-sa 60 | automountServiceAccountToken: true 61 | --- 62 | apiVersion: v1 63 | kind: Service 64 | metadata: 65 | name: otel-collector 66 | namespace: memorystore-cluster-autoscaler # kpt-set: ${namespace} 67 | spec: 68 | type: ClusterIP 69 | selector: 70 | app: otel-collector 71 | ports: 72 | - protocol: TCP 73 | port: 4317 74 | targetPort: 4317 75 | -------------------------------------------------------------------------------- /kubernetes/unified/autoscaler-pkg/scaler/Kptfile: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: kpt.dev/v1 15 | kind: Kptfile 16 | metadata: 17 | name: scaler 18 | annotations: 19 | config.kubernetes.io/local-config: "true" 20 | info: 21 | description: Config for Memorystore Cluster autoscaler 22 | -------------------------------------------------------------------------------- /kubernetes/unified/autoscaler-pkg/scaler/README.md: -------------------------------------------------------------------------------- 1 | # scaler 2 | 3 | ## Description 4 | 5 | Config for Memorystore Cluster Autoscaler 6 | 7 | ### View package content 8 | 9 | `kpt pkg tree scaler` 10 | [Details](https://kpt.dev/reference/cli/pkg/tree/) 11 | 12 | ## Installation 13 | 14 | See [documentation][docs] for installation and configuration instructions. 15 | 16 | [docs]: ../../../../terraform/gke/unified/README.md 17 | -------------------------------------------------------------------------------- /kubernetes/unified/autoscaler-pkg/scaler/scaler.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: batch/v1 15 | kind: CronJob 16 | metadata: 17 | name: scaler 18 | namespace: memorystore-cluster-autoscaler # kpt-set: ${namespace} 19 | spec: 20 | concurrencyPolicy: Forbid 21 | schedule: "*/2 * * * *" 22 | jobTemplate: 23 | spec: 24 | template: 25 | metadata: 26 | labels: 27 | app: scaler 28 | otel-submitter: "true" 29 | spec: 30 | containers: 31 | - name: scaler 32 | image: scaler-image # kpt-set: ${scaler_image} 33 | resources: 34 | requests: 35 | memory: "256Mi" 36 | cpu: "250m" 37 | limits: 38 | memory: "256Mi" 39 | env: 40 | - name: K8S_POD_NAME 41 | valueFrom: 42 | fieldRef: 43 | fieldPath: metadata.name 44 | - name: OTEL_COLLECTOR_URL 45 | value: "http://otel-collector:4317/" 46 | - name: OTEL_IS_LONG_RUNNING_PROCESS 47 | value: "false" 48 | securityContext: 49 | allowPrivilegeEscalation: false 50 | readOnlyRootFilesystem: true 51 | runAsNonRoot: true 52 | capabilities: 53 | drop: 54 | - all 55 | volumeMounts: 56 | - name: config-volume 57 | mountPath: /etc/autoscaler-config 58 | volumes: 59 | - name: config-volume 60 | configMap: 61 | name: autoscaler-config 62 | nodeSelector: 63 | iam.gke.io/gke-metadata-server-enabled: "true" 64 | restartPolicy: Never 65 | serviceAccountName: scaler-sa 66 | -------------------------------------------------------------------------------- /markdown-link-checker.json: -------------------------------------------------------------------------------- 1 | { 2 | "ignorePatterns": [ 3 | { 4 | "pattern": "^https://console.cloud.google.com/" 5 | }, 6 | { 7 | "pattern": "^https://example.org" 8 | } 9 | ], 10 | "replacementPatterns": [ 11 | { 12 | "pattern": "^([./].*)/(#.*)?$", 13 | "replacement": "$1/__LOCAL_URL_MUST_END_IN_FILENAME_NOT_RAW_PATH__$2" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /release-please-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": { 3 | ".": { 4 | "changelog-path": "CHANGELOG.md", 5 | "include-component-in-tag": false, 6 | "release-type": "node", 7 | "bump-minor-pre-major": false, 8 | "bump-patch-for-minor-pre-major": false, 9 | "draft": false, 10 | "prerelease": false, 11 | "extra-files": [ 12 | "terraform/modules/autoscaler-functions/main.tf", 13 | "terraform/modules/autoscaler-gke/main.tf" 14 | ] 15 | } 16 | }, 17 | "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json" 18 | } 19 | -------------------------------------------------------------------------------- /renovate.json5: -------------------------------------------------------------------------------- 1 | { 2 | $schema: "https://docs.renovatebot.com/renovate-schema.json", 3 | extends: [ 4 | "config:recommended", 5 | ":semanticCommits", 6 | ":semanticCommitTypeAll(fix)", 7 | ":enableVulnerabilityAlertsWithLabel(security)", 8 | ], 9 | ignorePaths: [ 10 | // override default ingorePaths which would ignore files in test directories 11 | "**/node_modules/**", 12 | "**/bower_components/**", 13 | ], 14 | packageRules: [ 15 | { 16 | description: "Do not create PRs for nodejs/npm engine updates", 17 | matchPackageNames: ["node", "npm"], 18 | matchDepTypes: ["engines"], 19 | dependencyDashboardApproval: true, 20 | groupName: "npm-engine-versions", 21 | }, 22 | { 23 | description: "Group minor/patch updates for all NPM packages except googleapis", 24 | matchDatasources: ["npm"], 25 | matchUpdateTypes: ["minor", "patch"], 26 | groupName: "npm-packages", 27 | matchPackageNames: ["!googleapis", "!npm", "!node"], 28 | }, 29 | { 30 | description: "Allow all update types for googleapis", 31 | matchDatasources: ["npm"], 32 | matchPackageNames: ["googleapis"], 33 | groupName: "npm-packages", 34 | }, 35 | { 36 | description: "Group for non-googleapis major NPM updates, that does not create PRs", 37 | matchDatasources: ["npm"], 38 | matchUpdateTypes: ["major"], 39 | groupName: "npm-major-packages", 40 | dependencyDashboardApproval: true, 41 | matchPackageNames: ["!googleapis", "!npm", "!node"], 42 | }, 43 | { 44 | // Temporarily put opentelemetry into its own group... 45 | matchPackageNames: ["@opentelemetry/**"], 46 | matchUpdateTypes: ["major", "minor", "patch"], 47 | matchDatasources: ["npm"], 48 | dependencyDashboardApproval: false, 49 | groupName: "opentelemetry", 50 | }, 51 | { 52 | matchDatasources: ["terraform-module", "terraform-provider"], 53 | groupName: "terraform", 54 | }, 55 | { 56 | matchDatasources: ["docker"], 57 | groupName: "docker-containers", 58 | }, 59 | { 60 | matchDatasources: ["go"], 61 | groupName: "golang-modules", 62 | }, 63 | ], 64 | customManagers: [ 65 | { 66 | customType: "regex", 67 | description: "Update _VERSION variables in Dockerfiles, shell scripts", 68 | fileMatch: [ 69 | "(^|/|\\.)([Dd]ocker|[Cc]ontainer)file$", 70 | "(^|/)([Dd]ocker|[Cc]ontainer)file[^/]*$", 71 | "(^|/)*.sh", 72 | ], 73 | matchStrings: [ 74 | '# renovate: datasource=(?[a-z-]+?)(?: depName=(?.+?))? packageName=(?.+?)(?: versioning=(?[a-z-]+?))?\\s(?:ENV|ARG)?\\s*.+?_VERSION="?(?.+?)"?\\s', 75 | ], 76 | }, 77 | ], 78 | rangeStrategy: "bump", 79 | } 80 | -------------------------------------------------------------------------------- /resources/architecture-abstract.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/memorystore-cluster-autoscaler/68c386b128a1b527b4209876df2c6dd2a318225a/resources/architecture-abstract.png -------------------------------------------------------------------------------- /resources/architecture-centralized.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/memorystore-cluster-autoscaler/68c386b128a1b527b4209876df2c6dd2a318225a/resources/architecture-centralized.png -------------------------------------------------------------------------------- /resources/architecture-distributed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/memorystore-cluster-autoscaler/68c386b128a1b527b4209876df2c6dd2a318225a/resources/architecture-distributed.png -------------------------------------------------------------------------------- /resources/architecture-forwarder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/memorystore-cluster-autoscaler/68c386b128a1b527b4209876df2c6dd2a318225a/resources/architecture-forwarder.png -------------------------------------------------------------------------------- /resources/architecture-gke-unified.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/memorystore-cluster-autoscaler/68c386b128a1b527b4209876df2c6dd2a318225a/resources/architecture-gke-unified.png -------------------------------------------------------------------------------- /resources/architecture-per-project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/memorystore-cluster-autoscaler/68c386b128a1b527b4209876df2c6dd2a318225a/resources/architecture-per-project.png -------------------------------------------------------------------------------- /resources/hero-image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/memorystore-cluster-autoscaler/68c386b128a1b527b4209876df2c6dd2a318225a/resources/hero-image.jpg -------------------------------------------------------------------------------- /src/README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 |

OSS Memorystore Cluster Autoscaler

4 | Autoscaler 5 | 6 |

7 | 8 | Automatically increase or reduce the size of Memorystore clusters. 9 |
10 | Home 11 | · 12 | Poller component 13 | · 14 | Scaler component 15 |

16 |

17 | 18 | ## Table of Contents 19 | 20 | * [Table of Contents](#table-of-contents) 21 | * [Overview](#overview) 22 | 23 | ## Overview 24 | 25 | This directory contains the source code for the two main components of the 26 | autoscaler: the Poller and the Scaler: 27 | 28 | * [Poller](poller/README.md) 29 | * [Scaler](scaler/README.md) 30 | 31 | As well as the Forwarder, which is used in the 32 | [distributed deployment model][distributed-docs]: 33 | 34 | * [Forwarder](forwarder/README.md) 35 | 36 | [distributed-docs]: ../terraform/cloud-functions/distributed/README.md 37 | -------------------------------------------------------------------------------- /src/autoscaler-common/assert-defined.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | /** 17 | * Asserts that given value is not null or undefined 18 | * 19 | * @template T 20 | * @param {T|null|undefined} value 21 | * @param {string} [valueName=''] 22 | * @return {!T} 23 | */ 24 | function assertDefined(value, valueName = '') { 25 | if (value == null) { 26 | throw new Error( 27 | `Fatal error: value ${valueName} must not be null/undefined.`, 28 | ); 29 | } 30 | return value; 31 | } 32 | 33 | module.exports = assertDefined; 34 | -------------------------------------------------------------------------------- /src/autoscaler-common/config-parameters.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | /** @fileoverview Provides common constants regarding Autoscaler setup. */ 17 | 18 | const CLUSTER_SIZE_MIN = 1; 19 | 20 | module.exports = { 21 | CLUSTER_SIZE_MIN, 22 | }; 23 | -------------------------------------------------------------------------------- /src/autoscaler-common/promiseWithResolvers.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | /** @typedef {{ 17 | * promise: Promise; 18 | * resolve: (value: any) => void; 19 | * reject: (reason: any) => void; 20 | * }} PromiseWithResolvers */ 21 | 22 | /** 23 | * Node version of ECMA262's Promise.withResolvers() 24 | * @see https://tc39.es/proposal-promise-with-resolvers/#sec-promise.withResolvers 25 | * 26 | * @return {PromiseWithResolvers} 27 | */ 28 | function promiseWithResolvers() { 29 | /** @type { (value: any) => void} */ 30 | let resolve; 31 | /** @type { (reason: any) => void} */ 32 | let reject; 33 | const promise = new Promise(function (res, rej) { 34 | resolve = res; 35 | reject = rej; 36 | }); 37 | // @ts-ignore used-before-assigned 38 | return {promise, resolve, reject}; 39 | } 40 | 41 | module.exports = { 42 | create: promiseWithResolvers, 43 | }; 44 | -------------------------------------------------------------------------------- /src/autoscaler-common/types.js: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /** 16 | * @fileoverview Common types for the autoscaler. 17 | * 18 | * Any changes to the AutoscalerMemorystoreCluster types also need to be 19 | * reflected in autoscaler-config.schema.json, and in 20 | * autoscaler-common/types.js. 21 | */ 22 | 23 | /** 24 | * @enum {string} 25 | */ 26 | const AutoscalerUnits = { 27 | SHARDS: 'SHARDS', 28 | }; 29 | 30 | /** 31 | * @enum {string} 32 | */ 33 | const AutoscalerDirection = { 34 | IN: 'IN', 35 | OUT: 'OUT', 36 | NONE: 'NONE', 37 | }; 38 | 39 | /** 40 | * @enum {string} 41 | */ 42 | const AutoscalerEngine = { 43 | REDIS: 'REDIS', 44 | VALKEY: 'VALKEY', 45 | }; 46 | 47 | /** 48 | * @typedef {{ 49 | * currentSize: number, 50 | * }} MemorystoreClusterMetadata 51 | */ 52 | 53 | /** 54 | * @typedef {{ 55 | * name: string, 56 | * filter: string, 57 | * reducer: string, 58 | * aligner: string, 59 | * period: number, 60 | * }} MemorystoreClusterMetric 61 | */ 62 | 63 | /** 64 | * @typedef {{ 65 | * name: string, 66 | * value: number, 67 | * threshold?: number, 68 | * }} MemorystoreClusterMetricValue 69 | */ 70 | 71 | /** 72 | * @typedef {{ 73 | * name: string, 74 | * instanceId?: string, 75 | * databaseId?: string 76 | * }} StateDatabaseConfig 77 | */ 78 | 79 | /** 80 | * @typedef {import('json-rules-engine').RuleProperties} Rule 81 | */ 82 | 83 | /** 84 | * @typedef {{ 85 | * projectId: string, 86 | * regionId: string, 87 | * clusterId: string, 88 | * engine: AutoscalerEngine, 89 | * units: AutoscalerUnits, 90 | * minSize: number, 91 | * maxSize: number, 92 | * scalingProfile: string, 93 | * scalingMethod: string, 94 | * stepSize: number, 95 | * scaleInLimit?: number, 96 | * scaleOutLimit?: number, 97 | * minFreeMemoryPercent: number, 98 | * scaleOutCoolingMinutes: number, 99 | * scaleInCoolingMinutes: number, 100 | * stateProjectId?: string, 101 | * stateDatabase?: StateDatabaseConfig, 102 | * scalerPubSubTopic?: string, 103 | * downstreamPubSubTopic?: string, 104 | * metrics: (MemorystoreClusterMetric | MemorystoreClusterMetricValue)[], 105 | * scalingRules?: Rule[] 106 | * }} MemorystoreClusterConfig; 107 | */ 108 | 109 | /** 110 | * @typedef {MemorystoreClusterConfig & MemorystoreClusterMetadata 111 | * } AutoscalerMemorystoreCluster; 112 | */ 113 | 114 | /** 115 | * @typedef {MemorystoreClusterMetricValue[]} ScalingMetricList 116 | */ 117 | 118 | /** 119 | * @typedef {{[x:string]: import('json-rules-engine').RuleProperties}} RuleSet 120 | */ 121 | 122 | /** 123 | * @typedef {import('json-rules-engine').ConditionProperties} 124 | * ConditionProperties 125 | */ 126 | 127 | /** 128 | * Extends ConditionProperty with the facts and fact results. 129 | * Workaround because json-rules-engine typing does not match the actual 130 | * signature nor exports the Condition class directly. 131 | * @link https://github.com/CacheControl/json-rules-engine/issues/253 132 | * @typedef {{ 133 | * factResult?: number, 134 | * result?: boolean, 135 | * }} AdditionalConditionProperties 136 | */ 137 | 138 | /** 139 | * @typedef {ConditionProperties & AdditionalConditionProperties} Condition 140 | */ 141 | 142 | /** 143 | * @typedef {{ 144 | * firingRuleCount: !Object, 145 | * matchedConditions: !Object, 146 | * scalingMetrics: !Object> 147 | * }} RuleEngineAnalysis 148 | */ 149 | 150 | module.exports = { 151 | AutoscalerUnits, 152 | AutoscalerDirection, 153 | AutoscalerEngine, 154 | }; 155 | -------------------------------------------------------------------------------- /src/forwarder/README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 |

OSS Memorystore Cluster Autoscaler

4 | Autoscaler 5 | 6 |

7 | 8 | Forward messages from Cloud Scheduler to the Poller function topic. 9 |
10 | Home 11 | · 12 | Poller component 13 | · 14 | Scaler component 15 | · 16 | Forwarder component 17 | · 18 | Terraform configuration 19 | · 20 | Monitoring 21 |

22 |

23 | 24 | ## Table of Contents 25 | 26 | * [Table of Contents](#table-of-contents) 27 | * [Overview](#overview) 28 | * [Architecture](#architecture) 29 | * [Configuration parameters](#configuration-parameters) 30 | * [Required](#required) 31 | 32 | ## Overview 33 | 34 | The Forwarder function takes messages published to PubSub from Cloud Scheduler, 35 | checks their JSON syntax and forwards them to the Poller PubSub topic. The topic 36 | can belong to a different project from the Scheduler. 37 | 38 | ## Architecture 39 | 40 | ![architecture-forwarder](../../resources/architecture-forwarder.png) 41 | 42 | The Memorystore Cluster instances reside in a given application project. 43 | 44 | 1. Cloud Scheduler lives in the same project as the Memorystore Cluster 45 | instances. 46 | 47 | 2. Cloud Scheduler publishes its messages to the Forwarder topic in the same project. 48 | 49 | 3. The Forwarder Cloud Function reads messages from the Forwarder topic, and 50 | 51 | 4. Forwards them to the Polling topic. The Polling topic resides in a 52 | different project. 53 | 54 | 5. The Poller function reads the messages from the polling topic and 55 | further continues the process as described in 56 | the [main architecture section](../../terraform/cloud-functions/README.md#architecture). 57 | 58 | It is important to note that Autoscaler infrastructure is now distributed across 59 | several projects. *The core components reside in the Autoscaler project* An 60 | instance of Cloud Scheduler, the Forwarder topic and the Forwarder Function 61 | reside in each of the application projects. 62 | 63 | ## Configuration parameters 64 | 65 | Using the Forward function forwards to the PubSub specified in the environment 66 | variable `POLLER_TOPIC`. 67 | 68 | ### Required 69 | 70 | | Key | Description | 71 | | -------------- | ------------------------------------------- | 72 | | `POLLER_TOPIC` | PubSub topic the Poller function listens on | 73 | -------------------------------------------------------------------------------- /src/forwarder/index.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | /* 17 | * Autoscaler Forwarder function 18 | * 19 | * * Forwards PubSub messages from the Scheduler topic to the Poller topic. 20 | */ 21 | // eslint-disable-next-line no-unused-vars -- for type checking only. 22 | const express = require('express'); 23 | const {PubSub} = require('@google-cloud/pubsub'); 24 | const {logger} = require('../autoscaler-common/logger'); 25 | const assertDefined = require('../autoscaler-common/assert-defined'); 26 | 27 | // GCP service clients 28 | const pubSub = new PubSub(); 29 | 30 | /** 31 | * Handle the forwarder request from HTTP 32 | * 33 | * For testing purposes - uses a fixed message. 34 | * 35 | * @param {express.Request} req 36 | * @param {express.Response} res 37 | */ 38 | async function forwardFromHTTP(req, res) { 39 | const payloadString = 40 | '[{ ' + 41 | ' "projectId": "memorystore-cluster-autoscaler", ' + 42 | ' "instanceId": "my-memorystore-cluster", ' + 43 | ' "scalerPubSubTopic": ' + 44 | '"projects/memorystore-cluster-autoscaler/topics/my-scaling-topic", ' + 45 | ' "minSize": 1, ' + 46 | ' "maxSize": 3, ' + 47 | ' "stateProjectId" : "memorystore-cluster-autoscaler" ' + 48 | '}]'; 49 | try { 50 | const payload = Buffer.from(payloadString, 'utf8'); 51 | 52 | JSON.parse(payload.toString()); // Log exception in App project if payload 53 | // cannot be parsed 54 | 55 | const pollerTopicName = assertDefined( 56 | process.env.POLLER_TOPIC, 57 | 'POLLER_TOPIC environment variable', 58 | ); 59 | 60 | const pollerTopic = pubSub.topic(pollerTopicName); 61 | pollerTopic.publishMessage({data: payload}); 62 | logger.debug({ 63 | message: `Poll request forwarded to PubSub Topic ${pollerTopicName}`, 64 | }); 65 | res.status(200).end(); 66 | } catch (err) { 67 | logger.error({ 68 | message: `An error occurred in the Autoscaler forwarder (HTTP): ${err}`, 69 | err: err, 70 | payload: payloadString, 71 | }); 72 | res.status(500).end('An exception occurred'); 73 | } 74 | } 75 | 76 | /** 77 | * Handle the Forwarder request from PubSub 78 | * 79 | * @param {any} pubSubEvent 80 | * @param {*} context 81 | */ 82 | async function forwardFromPubSub(pubSubEvent, context) { 83 | let payload; 84 | try { 85 | payload = Buffer.from(pubSubEvent.data, 'base64'); 86 | JSON.parse(payload.toString()); // Log exception in App project if payload 87 | // cannot be parsed 88 | 89 | const pollerTopicName = assertDefined( 90 | process.env.POLLER_TOPIC, 91 | 'POLLER_TOPIC environment variable', 92 | ); 93 | const pollerTopic = pubSub.topic(pollerTopicName); 94 | pollerTopic.publishMessage({data: payload}); 95 | logger.debug({ 96 | message: `Poll request forwarded to PubSub Topic ${pollerTopicName}`, 97 | }); 98 | } catch (err) { 99 | logger.error({ 100 | message: `An error occurred in the Autoscaler forwarder (PubSub): ${err}`, 101 | err: err, 102 | payload: payload, 103 | }); 104 | } 105 | } 106 | 107 | module.exports = { 108 | forwardFromHTTP, 109 | forwardFromPubSub, 110 | }; 111 | -------------------------------------------------------------------------------- /src/functions.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | /** 17 | * @fileoverview 18 | * Cloud Memorystore Cluster Autoscaler. 19 | * 20 | * Entry points for Cloud Run functions invocations. 21 | */ 22 | 23 | const poller = require('./poller/poller-core'); 24 | const scaler = require('./scaler/scaler-core'); 25 | const forwarder = require('./forwarder'); 26 | const {logger} = require('./autoscaler-common/logger'); 27 | const {version: packageVersion} = require('../package.json'); 28 | 29 | logger.info(`Cloud Memorystore Cluster autoscaler v${packageVersion} started`); 30 | 31 | module.exports = { 32 | checkMemorystoreClusterScaleMetricsPubSub: 33 | poller.checkMemorystoreClusterScaleMetricsPubSub, 34 | checkMemorystoreClusterScaleMetricsHTTP: 35 | poller.checkMemorystoreClusterScaleMetricsHTTP, 36 | 37 | scaleMemorystoreClusterPubSub: scaler.scaleMemorystoreClusterPubSub, 38 | scaleMemorystoreClusterHTTP: scaler.scaleMemorystoreClusterHTTP, 39 | 40 | forwardFromPubSub: forwarder.forwardFromPubSub, 41 | forwardFromHTTP: forwarder.forwardFromHTTP, 42 | }; 43 | -------------------------------------------------------------------------------- /src/poller/poller-core/counters.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | /* 17 | * Autoscaler Counters package 18 | * 19 | * Publishes Counters to Cloud Monitoring 20 | * 21 | */ 22 | const CountersBase = require('../../autoscaler-common/counters-base.js'); 23 | 24 | const COUNTERS_PREFIX = 'poller/'; 25 | 26 | const COUNTER_NAMES = { 27 | POLLING_SUCCESS: COUNTERS_PREFIX + 'polling-success', 28 | POLLING_FAILED: COUNTERS_PREFIX + 'polling-failed', 29 | REQUESTS_SUCCESS: COUNTERS_PREFIX + 'requests-success', 30 | REQUESTS_FAILED: COUNTERS_PREFIX + 'requests-failed', 31 | }; 32 | 33 | /** 34 | * @typedef {import('../../autoscaler-common/types.js') 35 | * .AutoscalerMemorystoreCluster} AutoscalerMemorystoreCluster 36 | */ 37 | /** 38 | * @typedef {import('@opentelemetry/api').Attributes} Attributes 39 | */ 40 | 41 | /** 42 | * @type {import('../../autoscaler-common/counters-base.js') 43 | * .CounterDefinition[]} 44 | */ 45 | const COUNTERS = [ 46 | { 47 | counterName: COUNTER_NAMES.POLLING_SUCCESS, 48 | counterDesc: 49 | 'The number of Memorystore Cluster polling events that succeeded', 50 | }, 51 | { 52 | counterName: COUNTER_NAMES.POLLING_FAILED, 53 | counterDesc: 'The number of Memorystore Cluster polling events that failed', 54 | }, 55 | { 56 | counterName: COUNTER_NAMES.REQUESTS_SUCCESS, 57 | counterDesc: 'The number of polling request messages handled successfully', 58 | }, 59 | { 60 | counterName: COUNTER_NAMES.REQUESTS_FAILED, 61 | counterDesc: 'The number of polling request messages that failed', 62 | }, 63 | ]; 64 | 65 | const pendingInit = CountersBase.createCounters(COUNTERS); 66 | 67 | /** 68 | * Build an attribute object for the counter 69 | * 70 | * @private 71 | * @param {AutoscalerMemorystoreCluster} cluster config object 72 | * @return {Attributes} 73 | */ 74 | function _getCounterAttributes(cluster) { 75 | return { 76 | [CountersBase.COUNTER_ATTRIBUTE_NAMES.CLUSTER_PROJECT_ID]: 77 | cluster.projectId, 78 | [CountersBase.COUNTER_ATTRIBUTE_NAMES.CLUSTER_INSTANCE_ID]: 79 | cluster.clusterId, 80 | }; 81 | } 82 | 83 | /** 84 | * Increment polling success counter 85 | * 86 | * @param {AutoscalerMemorystoreCluster} cluster config object 87 | */ 88 | async function incPollingSuccessCounter(cluster) { 89 | await pendingInit; 90 | CountersBase.incCounter( 91 | COUNTER_NAMES.POLLING_SUCCESS, 92 | _getCounterAttributes(cluster), 93 | ); 94 | } 95 | 96 | /** 97 | * Increment polling failed counter 98 | * 99 | * @param {AutoscalerMemorystoreCluster} cluster config object 100 | */ 101 | async function incPollingFailedCounter(cluster) { 102 | await pendingInit; 103 | CountersBase.incCounter( 104 | COUNTER_NAMES.POLLING_FAILED, 105 | _getCounterAttributes(cluster), 106 | ); 107 | } 108 | 109 | /** 110 | * Increment messages success counter 111 | */ 112 | async function incRequestsSuccessCounter() { 113 | await pendingInit; 114 | CountersBase.incCounter(COUNTER_NAMES.REQUESTS_SUCCESS); 115 | } 116 | 117 | /** 118 | * Increment messages failed counter 119 | */ 120 | async function incRequestsFailedCounter() { 121 | await pendingInit; 122 | CountersBase.incCounter(COUNTER_NAMES.REQUESTS_FAILED); 123 | } 124 | 125 | module.exports = { 126 | incPollingSuccessCounter, 127 | incPollingFailedCounter, 128 | incRequestsSuccessCounter, 129 | incRequestsFailedCounter, 130 | tryFlush: CountersBase.tryFlush, 131 | }; 132 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/bad-data-contents.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: dsdssfdfdsa 17 | metadata: 18 | name: autoscaler-config 19 | namespace: memorystore-autoscaler 20 | data: 21 | autoscaler-config.yaml: | 22 | hello world 23 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/bad-empty-array.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/bad-empty.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/memorystore-cluster-autoscaler/68c386b128a1b527b4209876df2c6dd2a318225a/src/poller/poller-core/test/resources/bad-empty.json -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/bad-empty.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: dsdssfdfdsa 17 | metadata: 18 | name: autoscaler-config 19 | namespace: memorystore-autoscaler 20 | data: 21 | autoscaler-config.yaml: "" 22 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/bad-invalid-props.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "projectId": "basic-configuration", 4 | "regionId": "us-central1", 5 | "clusterId": "another-memorystore1", 6 | "scalerPubSubTopic": "projects/my-memorystore-project/topics/memorystore-scaling", 7 | "units": "SHARDS", 8 | "minSize": 5, 9 | "maxSize": 30, 10 | "scalingMethod": "DIRECT", 11 | "garbage": "value" 12 | } 13 | ] 14 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/bad-invalid-props.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: ConfigMap 17 | metadata: 18 | name: autoscaler-config 19 | namespace: memorystore-autoscaler 20 | data: 21 | autoscaler-config.yaml: | 22 | --- 23 | - projectId: memorystore-autoscaler-test 24 | regionId: us-central1 25 | clusterId: memorystore-scaling-direct 26 | units: SHARDS 27 | minSize: 5 28 | maxSize: 30 29 | scalingMethod: DIRECT 30 | garbage: value 31 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/bad-invalid-value.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "projectId": "basic-configuration", 4 | "regionId": "us-central1", 5 | "clusterId": "another-memorystore1", 6 | "scalerPubSubTopic": "projects/my-memorystore-project/topics/memorystore-scaling", 7 | "units": "SHARDS", 8 | "minSize": 5, 9 | "maxSize": "30", 10 | "scalingMethod": "DIRECT" 11 | } 12 | ] 13 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/bad-invalid-value.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: ConfigMap 17 | metadata: 18 | name: autoscaler-config 19 | namespace: memorystore-autoscaler 20 | data: 21 | autoscaler-config.yaml: | 22 | --- 23 | - projectId: memorystore-autoscaler-test 24 | regionId: us-central1 25 | clusterId: memorystore-scaling-direct 26 | units: SHARDS 27 | minSize: 5 28 | maxSize: rubbish 29 | scalingMethod: DIRECT 30 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/bad-missing-props.json: -------------------------------------------------------------------------------- 1 | [{}] 2 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/bad-missing-props.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: ConfigMap 17 | metadata: 18 | name: autoscaler-config 19 | namespace: memorystore-autoscaler 20 | data: 21 | autoscaler-config.yaml: | 22 | --- 23 | - projectId: memorystore-autoscaler-test 24 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/bad-not-configmap.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: dsdssfdfdsa 17 | metadata: 18 | name: autoscaler-config 19 | namespace: memorystore-autoscaler 20 | data: 21 | autoscaler-config.yaml: | 22 | --- 23 | - projectId: memorystore-autoscaler-test 24 | regionId: us-central1 25 | clusterId: memorystore-scaling-direct 26 | units: SHARDS 27 | minSize: 5 28 | maxSize: 30 29 | scalingMethod: DIRECT 30 | - projectId: memorystore-autoscaler-test 31 | regionId: us-central1 32 | clusterId: memorystore-scaling-threshold 33 | units: SHARDS 34 | minSize: 100 35 | maxSize: 3000 36 | metrics: 37 | - name: high_priority_cpu 38 | regional_threshold: 40 39 | regional_margin: 3 40 | - projectId: memorystore-autoscaler-test 41 | regionId: us-central1 42 | clusterId: memorystore-scaling-custom 43 | units: SHARDS 44 | minSize: 5 45 | maxSize: 30 46 | scalingMethod: STEPWISE 47 | scaleInLimit: 25 48 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/bad-not-yaml.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | some garbage... 16 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/good-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "$comment": "test data", 4 | "projectId": "basic-configuration", 5 | "regionId": "us-central1", 6 | "clusterId": "another-memorystore1", 7 | "scalerPubSubTopic": "projects/my-memorystore-project/topics/memorystore-scaling", 8 | "units": "SHARDS", 9 | "minSize": 5, 10 | "maxSize": 30, 11 | "scalingMethod": "DIRECT" 12 | }, 13 | { 14 | "projectId": "custom-threshold", 15 | "regionId": "us-central1", 16 | "clusterId": "memorystore1", 17 | "scalerPubSubTopic": "projects/my-memorystore-project/topics/memorystore-scaling", 18 | "units": "SHARDS", 19 | "minSize": 3, 20 | "maxSize": 5, 21 | "scalingProfile": "CPU" 22 | }, 23 | { 24 | "projectId": "custom-metric", 25 | "regionId": "us-central1", 26 | "clusterId": "another-memorystore1", 27 | "scalerPubSubTopic": "projects/my-memorystore-project/topics/memorystore-scaling", 28 | "units": "SHARDS", 29 | "minSize": 5, 30 | "maxSize": 30, 31 | "scalingMethod": "STEPWISE" 32 | } 33 | ] 34 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/good-config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: ConfigMap 17 | metadata: 18 | name: autoscaler-config 19 | namespace: memorystore-autoscaler 20 | data: 21 | autoscaler-config.yaml: | 22 | --- 23 | - $comment: test data 24 | projectId: memorystore-autoscaler-test 25 | regionId: us-central1 26 | clusterId: memorystore-scaling-direct 27 | units: SHARDS 28 | minSize: 5 29 | maxSize: 30 30 | scalingMethod: DIRECT 31 | - projectId: memorystore-autoscaler-test 32 | regionId: us-central1 33 | clusterId: memorystore-scaling-threshold 34 | units: SHARDS 35 | minSize: 1 36 | maxSize: 5 37 | - projectId: memorystore-autoscaler-test 38 | regionId: us-central1 39 | clusterId: memorystore-scaling-custom 40 | units: SHARDS 41 | minSize: 5 42 | maxSize: 30 43 | scalingMethod: STEPWISE 44 | metrics: 45 | - name: my_custom_metric 46 | filter: metric.type="redis.googleapis.com/cluster/stats/metric" 47 | -------------------------------------------------------------------------------- /src/poller/poller-core/test/resources/good-multi-config.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: ConfigMap 17 | metadata: 18 | name: autoscaler-config 19 | namespace: memorystore-autoscaler 20 | data: 21 | autoscaler-config.yaml: | 22 | --- 23 | - projectId: memorystore-autoscaler-test 24 | regionId: us-central1 25 | clusterId: memorystore-scaling-threshold 26 | units: SHARDS 27 | minSize: 100 28 | maxSize: 3000 29 | - projectId: memorystore-autoscaler-test 30 | regionId: us-central1 31 | clusterId: memorystore-scaling-custom 32 | units: SHARDS 33 | minSize: 5 34 | maxSize: 30 35 | scalingMethod: STEPWISE 36 | autoscaler-config-direct.yaml: | 37 | --- 38 | - projectId: memorystore-autoscaler-test 39 | regionId: us-central1 40 | clusterId: memorystore-scaling-direct 41 | units: SHARDS 42 | minSize: 5 43 | maxSize: 30 44 | scalingMethod: DIRECT 45 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/downstream.schema.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | message DownstreamEvent { 18 | 19 | message Metric { 20 | reserved 5 to 1000; 21 | string name = 1; 22 | float threshold = 2; 23 | float value = 3; 24 | float margin = 4; 25 | } 26 | 27 | reserved 8 to 1000; 28 | string project_id = 1; 29 | string region_id = 2; 30 | string instance_id = 3; 31 | optional int32 current_size = 4; 32 | optional int32 suggested_size = 5; 33 | optional Units units = 6; 34 | repeated Metric metrics = 7; 35 | } 36 | 37 | enum Units { 38 | SHARDS = 0; 39 | } 40 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-methods/direct.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | /* 17 | * Direct scaling method 18 | * 19 | * Sets the instance to the maxSize directly (avoiding forbidden sizes) 20 | */ 21 | const baseModule = require('./base'); 22 | 23 | /** 24 | * @typedef {import('../../../autoscaler-common/types') 25 | * .AutoscalerMemorystoreCluster} AutoscalerMemorystoreCluster 26 | * @typedef {import('../../../autoscaler-common/types').AutoscalerDirection} 27 | * AutoscalerDirection 28 | * @typedef {import('../../../autoscaler-common/types').RuleSet} 29 | * RuleSet 30 | * @typedef {import('../../../autoscaler-common/types').RuleEngineAnalysis} 31 | * RuleEngineAnalysis 32 | */ 33 | 34 | /** 35 | * Calculates the suggested cluster size for a given metric. 36 | * 37 | * Always scales to the max size. 38 | * 39 | * @param {AutoscalerMemorystoreCluster} cluster for which to suggest a new 40 | * size. 41 | * @param {AutoscalerDirection} direction Direction in which to scale. Not in 42 | * use. 43 | * @param {?RuleEngineAnalysis} engineAnalysis Results from the engine analysis. 44 | * Not in use. 45 | * @return {number} Final suggested size for the cluster. 46 | */ 47 | function getSuggestedSize(cluster, direction, engineAnalysis) { 48 | return cluster.maxSize; 49 | } 50 | 51 | /** 52 | * Scaling calculation for Direct method. Always scales to max size no matter 53 | * what the conditions of the cluster. 54 | * 55 | * @param {AutoscalerMemorystoreCluster} cluster 56 | * @param {RuleSet} ruleSet to use to determine scaling decisions. 57 | * @return {Promise} 58 | */ 59 | async function calculateSize(cluster, ruleSet) { 60 | return baseModule.calculateScalingDecision( 61 | cluster, 62 | // The only rule is there are no rules. 63 | null, 64 | getSuggestedSize, 65 | ); 66 | } 67 | 68 | module.exports = {calculateSize}; 69 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-methods/stepwise.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | /* 17 | * Stepwise scaling method 18 | * 19 | * Default method used by the scaler. 20 | * Suggests adding or removing shards using a fixed step size. 21 | */ 22 | const {AutoscalerDirection} = require('../../../autoscaler-common/types'); 23 | const baseModule = require('./base'); 24 | 25 | /** 26 | * @typedef {import('../../../autoscaler-common/types') 27 | * .AutoscalerMemorystoreCluster} AutoscalerMemorystoreCluster 28 | * @typedef {import('../../../autoscaler-common/types.js').RuleSet} RuleSet 29 | * @typedef {import('../../../autoscaler-common/types').RuleEngineAnalysis} 30 | * RuleEngineAnalysis 31 | */ 32 | 33 | /** 34 | * Calculates the suggested cluster size for a given metric. 35 | * 36 | * @param {AutoscalerMemorystoreCluster} cluster for which to suggest a new 37 | * size. 38 | * @param {AutoscalerDirection} direction Direction in which to scale. 39 | * @param {?RuleEngineAnalysis} engineAnalysis Results from the engine analysis. 40 | * Not in use. 41 | * @return {number} Final suggested size for the cluster. 42 | */ 43 | function getSuggestedSize(cluster, direction, engineAnalysis) { 44 | if (direction === AutoscalerDirection.OUT) { 45 | return cluster.currentSize + cluster.stepSize; 46 | } else if (direction === AutoscalerDirection.IN) { 47 | return cluster.currentSize - cluster.stepSize; 48 | } else { 49 | return cluster.currentSize; 50 | } 51 | } 52 | 53 | /** 54 | * Scaling calculation for Stepwise method 55 | * 56 | * @param {AutoscalerMemorystoreCluster} cluster 57 | * @param {RuleSet} ruleSet to use to determine scaling decisions. 58 | * @return {Promise} 59 | */ 60 | async function calculateSize(cluster, ruleSet) { 61 | return baseModule.calculateScalingDecision( 62 | cluster, 63 | ruleSet, 64 | getSuggestedSize, 65 | ); 66 | } 67 | 68 | module.exports = {calculateSize}; 69 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/profiles/README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 |

OSS Memorystore Cluster Autoscaler

4 | Autoscaler 5 |

6 | 7 | ## Overview 8 | 9 | This directory contains profiles for scaling based on: 10 | 11 | * [CPU utilization](./cpu.js) 12 | * [Memory utilization](./memory.js) 13 | * [CPU and Memory utilization](./cpu_and_memory.js) 14 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/profiles/cpu.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const cpuHighAverageUtilization = require('../rules/cpu/cpu-high-average-utilization.js'); 18 | const cpuHighMaximumUtilization = require('../rules/cpu/cpu-high-maximum-utilization.js'); 19 | const cpuLowAverageUtilization = require('../rules/cpu/cpu-low-average-utilization.js'); 20 | const cpuLowMaximumUtilization = require('../rules/cpu/cpu-low-maximum-utilization.js'); 21 | 22 | /** 23 | * @typedef {import('../../../../autoscaler-common/types.js').RuleSet} 24 | * RuleSet 25 | */ 26 | 27 | /** @type {RuleSet} */ 28 | module.exports.ruleSet = { 29 | cpuHighMaximumUtilization, 30 | cpuHighAverageUtilization, 31 | cpuLowMaximumUtilization, 32 | cpuLowAverageUtilization, 33 | }; 34 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/profiles/cpu_and_memory.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const cpuHighAverageUtilization = require('../rules/cpu/cpu-high-average-utilization.js'); 18 | const cpuHighMaximumUtilization = require('../rules/cpu/cpu-high-maximum-utilization.js'); 19 | const cpuLowAverageUtilization = require('../rules/cpu/cpu-low-average-utilization.js'); 20 | const cpuLowMaximumUtilization = require('../rules/cpu/cpu-low-maximum-utilization.js'); 21 | 22 | const memoryHighAverageUtilization = require('../rules/memory/memory-high-average-utilization.js'); 23 | const memoryHighMaximumUtilization = require('../rules/memory/memory-high-maximum-utilization.js'); 24 | const memoryLowAverageUtilization = require('../rules/memory/memory-low-average-utilization.js'); 25 | const memoryLowMaximumUtilization = require('../rules/memory/memory-low-maximum-utilization.js'); 26 | 27 | /** 28 | * @typedef {import('../../../../autoscaler-common/types.js').RuleSet} RuleSet 29 | */ 30 | 31 | /** @type {RuleSet} */ 32 | module.exports.ruleSet = { 33 | cpuHighMaximumUtilization, 34 | cpuHighAverageUtilization, 35 | cpuLowMaximumUtilization, 36 | cpuLowAverageUtilization, 37 | memoryHighAverageUtilization, 38 | memoryHighMaximumUtilization, 39 | memoryLowAverageUtilization, 40 | memoryLowMaximumUtilization, 41 | }; 42 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/profiles/memory.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const memoryHighAverageUtilization = require('../rules/memory/memory-high-average-utilization.js'); 18 | const memoryHighMaximumUtilization = require('../rules/memory/memory-high-maximum-utilization.js'); 19 | const memoryLowAverageUtilization = require('../rules/memory/memory-low-average-utilization.js'); 20 | const memoryLowMaximumUtilization = require('../rules/memory/memory-low-maximum-utilization.js'); 21 | 22 | /** 23 | * @typedef {import('../../../../autoscaler-common/types.js').RuleSet} 24 | * RuleSet 25 | */ 26 | 27 | /** @type {RuleSet} */ 28 | module.exports.ruleSet = { 29 | memoryHighAverageUtilization, 30 | memoryHighMaximumUtilization, 31 | memoryLowAverageUtilization, 32 | memoryLowMaximumUtilization, 33 | }; 34 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/rules/README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 |

OSS Memorystore Cluster Autoscaler

4 | Autoscaler 5 |

6 | 7 | ## Overview 8 | 9 | This directory contains rules for scaling based on: 10 | 11 | * [CPU utilization](./cpu/README.md) 12 | * [Memory utilization](./memory/README.md) 13 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/rules/cpu/README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 |

OSS Memorystore Cluster Autoscaler

4 | Autoscaler 5 |

6 | 7 | ## Overview 8 | 9 | This directory contains rules for scaling based on CPU utilization. 10 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/rules/cpu/cpu-high-average-utilization.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | const {basename} = require('path'); 16 | 17 | /** 18 | * @fileoverview Rule which triggers when the average CPU utilization is > 70% 19 | * 20 | * @type {import('json-rules-engine').RuleProperties} 21 | */ 22 | module.exports = { 23 | name: basename(__filename, '.js'), 24 | conditions: { 25 | all: [ 26 | { 27 | fact: 'cpu_average_utilization', 28 | operator: 'greaterThan', 29 | value: 70, 30 | }, 31 | ], 32 | }, 33 | event: { 34 | type: 'OUT', 35 | params: { 36 | message: 'high average CPU utilization', 37 | scalingMetrics: ['cpu_average_utilization'], 38 | }, 39 | }, 40 | }; 41 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/rules/cpu/cpu-high-maximum-utilization.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | const {basename} = require('path'); 16 | 17 | /** 18 | * @fileoverview Rule which triggers when the CPU utilization is high based 19 | * on average cpu > 50% and max cpu > 80% 20 | * 21 | * @type {import('json-rules-engine').RuleProperties} 22 | */ 23 | module.exports = { 24 | name: basename(__filename, '.js'), 25 | conditions: { 26 | all: [ 27 | { 28 | fact: 'cpu_maximum_utilization', 29 | operator: 'greaterThan', 30 | value: 80, 31 | }, 32 | { 33 | fact: 'cpu_average_utilization', 34 | operator: 'greaterThan', 35 | value: 50, 36 | }, 37 | ], 38 | }, 39 | event: { 40 | type: 'OUT', 41 | params: { 42 | message: 'high maximum CPU utilization', 43 | scalingMetrics: ['cpu_maximum_utilization'], 44 | }, 45 | }, 46 | }; 47 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/rules/cpu/cpu-low-average-utilization.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | const {basename} = require('path'); 16 | 17 | /** 18 | * @fileoverview Rule which triggers when the average CPU utilization low 19 | * based on < 50% average CPU with no evicted keys. 20 | * 21 | * @type {import('json-rules-engine').RuleProperties} 22 | */ 23 | module.exports = { 24 | name: basename(__filename, '.js'), 25 | conditions: { 26 | all: [ 27 | { 28 | fact: 'cpu_average_utilization', 29 | operator: 'lessThan', 30 | value: 50, 31 | }, 32 | { 33 | fact: 'maximum_evicted_keys', 34 | operator: 'equal', 35 | value: 0, 36 | }, 37 | { 38 | fact: 'average_evicted_keys', 39 | operator: 'equal', 40 | value: 0, 41 | }, 42 | ], 43 | }, 44 | event: { 45 | type: 'IN', 46 | params: { 47 | message: 'low average CPU utilization', 48 | scalingMetrics: ['cpu_average_utilization'], 49 | }, 50 | }, 51 | }; 52 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/rules/cpu/cpu-low-maximum-utilization.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | const {basename} = require('path'); 16 | 17 | /** 18 | * @fileoverview Rule which triggers when the max CPU utilization is low and no keys are being evicted 19 | * 20 | * @type {import('json-rules-engine').RuleProperties} 21 | */ 22 | module.exports = { 23 | name: basename(__filename, '.js'), 24 | conditions: { 25 | all: [ 26 | { 27 | fact: 'cpu_maximum_utilization', 28 | operator: 'lessThan', 29 | value: 60, 30 | }, 31 | { 32 | fact: 'cpu_average_utilization', 33 | operator: 'lessThan', 34 | value: 40, 35 | }, 36 | { 37 | fact: 'maximum_evicted_keys', 38 | operator: 'equal', 39 | value: 0, 40 | }, 41 | { 42 | fact: 'average_evicted_keys', 43 | operator: 'equal', 44 | value: 0, 45 | }, 46 | ], 47 | }, 48 | event: { 49 | type: 'IN', 50 | params: { 51 | message: 'low maximum CPU utilization', 52 | scalingMetrics: ['cpu_maximum_utilization'], 53 | }, 54 | }, 55 | }; 56 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/rules/memory/README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 |

OSS Memorystore Cluster Autoscaler

4 | Autoscaler 5 |

6 | 7 | ## Overview 8 | 9 | This directory contains rules for scaling based on memory utilization. 10 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/rules/memory/memory-high-average-utilization.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | const {basename} = require('path'); 16 | 17 | /** 18 | * @fileoverview Rule which triggers when average memory usage is > 70% 19 | * 20 | * @type {import('json-rules-engine').RuleProperties} 21 | */ 22 | module.exports = { 23 | name: basename(__filename, '.js'), 24 | conditions: { 25 | all: [ 26 | { 27 | fact: 'memory_average_utilization', 28 | operator: 'greaterThan', 29 | value: 70, 30 | }, 31 | ], 32 | }, 33 | event: { 34 | type: 'OUT', 35 | params: { 36 | message: 'high average memory utilization', 37 | scalingMetrics: ['memory_average_utilization'], 38 | }, 39 | }, 40 | }; 41 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/rules/memory/memory-high-maximum-utilization.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | const {basename} = require('path'); 16 | 17 | /** 18 | * @fileoverview Rule which triggers when max memory usage is > 80% 19 | * 20 | * @type {import('json-rules-engine').RuleProperties} 21 | */ 22 | module.exports = { 23 | name: basename(__filename, '.js'), 24 | conditions: { 25 | all: [ 26 | { 27 | fact: 'memory_maximum_utilization', 28 | operator: 'greaterThan', 29 | value: 80, 30 | }, 31 | { 32 | fact: 'memory_average_utilization', 33 | operator: 'greaterThan', 34 | value: 50, 35 | }, 36 | ], 37 | }, 38 | event: { 39 | type: 'OUT', 40 | params: { 41 | message: 'high maximum memory utilization', 42 | scalingMetrics: ['memory_maximum_utilization'], 43 | }, 44 | }, 45 | }; 46 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/rules/memory/memory-low-average-utilization.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | const {basename} = require('path'); 16 | 17 | /** 18 | * @fileoverview Rule which triggers when average memory usage is less than 50% 19 | * and no keys are being evicted 20 | * 21 | * @type {import('json-rules-engine').RuleProperties} 22 | */ 23 | module.exports = { 24 | name: basename(__filename, '.js'), 25 | conditions: { 26 | all: [ 27 | { 28 | fact: 'memory_average_utilization', 29 | operator: 'lessThan', 30 | value: 50, 31 | }, 32 | { 33 | fact: 'maximum_evicted_keys', 34 | operator: 'equal', 35 | value: 0, 36 | }, 37 | { 38 | fact: 'average_evicted_keys', 39 | operator: 'equal', 40 | value: 0, 41 | }, 42 | ], 43 | }, 44 | event: { 45 | type: 'IN', 46 | params: { 47 | message: 'low average memory utilization', 48 | scalingMetrics: ['memory_average_utilization'], 49 | }, 50 | }, 51 | }; 52 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/scaling-profiles/rules/memory/memory-low-maximum-utilization.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | const {basename} = require('path'); 16 | 17 | /** 18 | * @fileoverview Rule which triggers when max and average memory usage is low 19 | * and no keys are being evicted. 20 | * 21 | * @type {import('json-rules-engine').RuleProperties} 22 | */ 23 | module.exports = { 24 | name: basename(__filename, '.js'), 25 | conditions: { 26 | all: [ 27 | { 28 | fact: 'memory_maximum_utilization', 29 | operator: 'lessThan', 30 | value: 60, 31 | }, 32 | { 33 | fact: 'memory_average_utilization', 34 | operator: 'lessThan', 35 | value: 40, 36 | }, 37 | { 38 | fact: 'maximum_evicted_keys', 39 | operator: 'equal', 40 | value: 0, 41 | }, 42 | { 43 | fact: 'average_evicted_keys', 44 | operator: 'equal', 45 | value: 0, 46 | }, 47 | ], 48 | }, 49 | event: { 50 | type: 'IN', 51 | params: { 52 | message: 'low maximum memory utilization', 53 | scalingMetrics: ['memory_maximum_utilization'], 54 | }, 55 | }, 56 | }; 57 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/test/samples/custom-scaling-rules.json: -------------------------------------------------------------------------------- 1 | { 2 | "scalingRules": [ 3 | { 4 | "name": "custom_max_memory_rule", 5 | "conditions": { 6 | "all": [ 7 | { 8 | "fact": "memory_maximum_utilization", 9 | "operator": "lessThan", 10 | "value": 70 11 | } 12 | ] 13 | }, 14 | "event": { 15 | "type": "IN", 16 | "params": { 17 | "message": "low maximum memory utilization", 18 | "scalingMetrics": ["memory_maximum_utilization"] 19 | } 20 | }, 21 | "priority": 1 22 | }, 23 | { 24 | "name": "custom_average_memory_rule", 25 | "conditions": { 26 | "all": [ 27 | { 28 | "fact": "memory_average_utilization", 29 | "operator": "lessThan", 30 | "value": 60 31 | } 32 | ] 33 | }, 34 | "event": { 35 | "type": "IN", 36 | "params": { 37 | "message": "low average memory utilization", 38 | "scalingMetrics": ["memory_average_utilization"] 39 | } 40 | }, 41 | "priority": 1 42 | } 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/test/samples/downstream-msg.json: -------------------------------------------------------------------------------- 1 | { 2 | "projectId": "project1", 3 | "regionId": "region1", 4 | "instanceId": "cluster1", 5 | "currentSize": 100, 6 | "suggestedSize": 300, 7 | "units": "SHARDS", 8 | "metrics": [ 9 | { 10 | "name": "cpu_maximum_utilization", 11 | "scaleInThreshold": 50, 12 | "scaleOutThreshold": 70, 13 | "value": 0.19835128894461815 14 | }, 15 | { 16 | "name": "cpu_average_utilization", 17 | "scaleInThreshold": 50, 18 | "scaleOutThreshold": 70, 19 | "value": 0.18477335171747497 20 | }, 21 | { 22 | "name": "memory_maximum_utilization", 23 | "scaleInThreshold": 50, 24 | "scaleOutThreshold": 70, 25 | "value": 0.0186809696873731 26 | }, 27 | { 28 | "name": "memory_average_utilization", 29 | "scaleInThreshold": 50, 30 | "scaleOutThreshold": 70, 31 | "value": 0.018497523020197155 32 | } 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/test/samples/parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "projectId": "project1", 3 | "regionId": "region1", 4 | "clusterId": "cluster1", 5 | "engine": "VALKEY", 6 | "units": "SHARDS", 7 | "minSize": 5, 8 | "maxSize": 10, 9 | "stepSize": 1, 10 | "scalingProfile": "CPU_AND_MEMORY", 11 | "scalingMethod": "STEPWISE", 12 | "minFreeMemoryPercent": 30, 13 | "scaleOutCoolingMinutes": 5, 14 | "scaleInCoolingMinutes": 30, 15 | "metrics": [ 16 | { 17 | "name": "cpu_maximum_utilization", 18 | "value": 0 19 | }, 20 | { 21 | "name": "cpu_average_utilization", 22 | "value": 0 23 | }, 24 | { 25 | "name": "memory_maximum_utilization", 26 | "value": 0 27 | }, 28 | { 29 | "name": "memory_average_utilization", 30 | "value": 0 31 | }, 32 | { 33 | "name": "maximum_evicted_keys", 34 | "value": 0 35 | }, 36 | { 37 | "name": "average_evicted_keys", 38 | "value": 0 39 | } 40 | ], 41 | "currentSize": 5 42 | } 43 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/test/scaling-methods/direct.test.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | /* 17 | * ESLINT: Ignore max line length errors on lines starting with 'it(' 18 | * (test descriptions) 19 | */ 20 | /* eslint max-len: ["error", { "ignorePattern": "^\\s*it\\(" }] */ 21 | 22 | const rewire = require('rewire'); 23 | const sinon = require('sinon'); 24 | // @ts-ignore 25 | const referee = require('@sinonjs/referee'); 26 | // @ts-ignore 27 | const assert = referee.assert; 28 | const {createClusterParameters} = require('../test-utils.js'); 29 | 30 | const app = rewire('../../scaling-methods/direct.js'); 31 | 32 | /** 33 | * @typedef {import('../../../../autoscaler-common/types') 34 | * .AutoscalerMemorystoreCluster} AutoscalerMemorystoreCluster 35 | */ 36 | 37 | afterEach(() => { 38 | // Restore the default sandbox here 39 | sinon.restore(); 40 | }); 41 | 42 | const calculateSize = app.__get__('calculateSize'); 43 | describe('#direct.calculateSize', () => { 44 | /** @type {sinon.SinonSpy} */ 45 | let calculateScalingDecisionSpy; 46 | beforeEach(() => { 47 | const baseModule = app.__get__('baseModule'); 48 | calculateScalingDecisionSpy = sinon.spy( 49 | baseModule, 50 | 'calculateScalingDecision', 51 | ); 52 | }); 53 | 54 | it('should return max size', async () => { 55 | const cluster = createClusterParameters({ 56 | currentSize: 5, 57 | maxSize: 10, 58 | minSize: 1, 59 | scalingMethod: 'DIRECT', 60 | }); 61 | const size = await calculateSize(cluster); 62 | assert.equals(size, 10); 63 | assert.equals(calculateScalingDecisionSpy.callCount, 1); 64 | }); 65 | 66 | it('should return min 1 when less than 1 is suggested', async () => { 67 | const cluster = createClusterParameters({ 68 | currentSize: 1, 69 | maxSize: 0, 70 | minSize: 0, 71 | scalingMethod: 'DIRECT', 72 | }); 73 | const size = await calculateSize(cluster); 74 | assert.equals(size, 1); 75 | assert.equals(calculateScalingDecisionSpy.callCount, 1); 76 | }); 77 | }); 78 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/test/scaling-methods/stepwise.test.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | /* 17 | * ESLINT: Ignore max line length errors on lines starting with 'it(' 18 | * (test descriptions) 19 | */ 20 | /* eslint max-len: ["error", { "ignorePattern": "^\\s*it\\(" }] */ 21 | 22 | const rewire = require('rewire'); 23 | const sinon = require('sinon'); 24 | // @ts-ignore 25 | const referee = require('@sinonjs/referee'); 26 | // @ts-ignore 27 | const assert = referee.assert; 28 | const {createClusterParameters} = require('../test-utils.js'); 29 | const {AutoscalerDirection} = require('../../../../autoscaler-common/types'); 30 | const app = rewire('../../scaling-methods/stepwise.js'); 31 | 32 | /** 33 | * @typedef {import('../../../../autoscaler-common/types') 34 | * .AutoscalerMemorystoreCluster} AutoscalerMemorystoreCluster 35 | */ 36 | 37 | afterEach(() => { 38 | // Restore the default sandbox here 39 | sinon.restore(); 40 | }); 41 | 42 | /** 43 | * 44 | * @param {AutoscalerMemorystoreCluster} cluster 45 | * @param {AutoscalerDirection} direction 46 | * @return {sinon.SinonStub} base module 47 | */ 48 | function stubBaseModule(cluster, direction) { 49 | const callbackStub = sinon.stub().callsArgWith(2, cluster, direction); 50 | app.__set__('baseModule.calculateScalingDecision', callbackStub); 51 | app.__set__('baseModule.getScalingDirection', () => direction); 52 | return callbackStub; 53 | } 54 | 55 | const calculateSize = app.__get__('calculateSize'); 56 | describe('#stepwise.calculateSize', () => { 57 | it('should return current size if no scaling is needed', async () => { 58 | const cluster = createClusterParameters({currentSize: 10, stepSize: 2}); 59 | const callbackStub = stubBaseModule(cluster, AutoscalerDirection.NONE); 60 | const size = await calculateSize(cluster, null); 61 | size.should.equal(10); 62 | assert.equals(callbackStub.callCount, 1); 63 | }); 64 | 65 | it('should return current size increased by stepSize if scale OUT is suggested', async () => { 66 | const cluster = createClusterParameters({currentSize: 6, stepSize: 1}); 67 | const callbackStub = stubBaseModule(cluster, AutoscalerDirection.OUT); 68 | const size = await calculateSize(cluster, null); 69 | size.should.equal(7); 70 | assert.equals(callbackStub.callCount, 1); 71 | }); 72 | 73 | it('should return current size decreased by stepSize if scale IN is suggested', async () => { 74 | const cluster = createClusterParameters({currentSize: 6, stepSize: 1}); 75 | const callbackStub = stubBaseModule(cluster, AutoscalerDirection.IN); 76 | const size = await calculateSize(cluster, null); 77 | size.should.equal(5); 78 | assert.equals(callbackStub.callCount, 1); 79 | }); 80 | }); 81 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/test/test-utils.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | const sinon = require('sinon'); 16 | const State = require('../state.js'); 17 | const unionBy = require('lodash.unionby'); 18 | 19 | const parameters = require('./samples/parameters.json'); 20 | 21 | /** 22 | * @typedef {import('../../../autoscaler-common/types') 23 | * .AutoscalerMemorystoreCluster} AutoscalerMemorystoreCluster 24 | * @typedef {import('../../../autoscaler-common/types').MemorystoreClusterMetric 25 | * } MemorystoreClusterMetric 26 | * @typedef {import('../../../autoscaler-common/types') 27 | * .MemorystoreClusterMetricValue} MemorystoreClusterMetricValue 28 | * @typedef {State.StateData} StateData 29 | */ 30 | 31 | const DUMMY_TIMESTAMP = 1704110400000; 32 | 33 | /** 34 | * Read Spanner params from file 35 | * 36 | * @param {Object} [overrideParams] 37 | * @return {AutoscalerMemorystoreCluster} 38 | */ 39 | function createClusterParameters(overrideParams) { 40 | return /** @type {AutoscalerMemorystoreCluster} */ ({ 41 | ...parameters, 42 | ...overrideParams, 43 | }); 44 | } 45 | 46 | /** 47 | * Merge metrics objects 48 | * 49 | * @param {AutoscalerMemorystoreCluster} cluster 50 | * @param {(MemorystoreClusterMetric | MemorystoreClusterMetricValue)[]} 51 | * metricsOverlay 52 | * @return {(MemorystoreClusterMetric | MemorystoreClusterMetricValue)[]} 53 | */ 54 | function metricsOverlay(cluster, metricsOverlay) { 55 | return unionBy(metricsOverlay, cluster.metrics, 'name'); 56 | } 57 | 58 | /** 59 | * @return {sinon.SinonStubbedInstance} state class stub 60 | */ 61 | function createStubState() { 62 | const stubState = sinon.createStubInstance(State); 63 | stubState.updateState.resolves(); 64 | sinon.replaceGetter(stubState, 'now', () => DUMMY_TIMESTAMP); 65 | return stubState; 66 | } 67 | 68 | /** 69 | * @return {StateData} StateData object 70 | */ 71 | function createStateData() { 72 | return { 73 | lastScalingTimestamp: 0, 74 | createdOn: 0, 75 | updatedOn: 0, 76 | lastScalingCompleteTimestamp: 0, 77 | scalingOperationId: null, 78 | scalingRequestedSize: null, 79 | scalingPreviousSize: null, 80 | scalingMethod: null, 81 | }; 82 | } 83 | 84 | /** 85 | * @return {string} downstream message 86 | */ 87 | function createDownstreamMsg() { 88 | return JSON.stringify(require('./samples/downstream-msg.json'), null, 2); 89 | } 90 | 91 | module.exports = { 92 | createClusterParameters, 93 | createStubState, 94 | createDownstreamMsg, 95 | createStateData, 96 | metricsOverlay, 97 | }; 98 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/test/utils.test.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | const {Topic} = require('@google-cloud/pubsub'); 17 | const rewire = require('rewire'); 18 | // eslint-disable-next-line no-unused-vars 19 | const should = require('should'); 20 | const sinon = require('sinon'); 21 | // @ts-ignore 22 | const referee = require('@sinonjs/referee'); 23 | // @ts-ignore 24 | const assert = referee.assert; 25 | const {createDownstreamMsg} = require('./test-utils.js'); 26 | 27 | const app = rewire('../utils.js'); 28 | 29 | const {PubSub} = require('@google-cloud/pubsub'); 30 | const pubsub = new PubSub(); 31 | const protobuf = require('protobufjs'); 32 | 33 | const publishProtoMsgDownstream = app.__get__('publishProtoMsgDownstream'); 34 | describe('#publishProtoMsgDownstream', () => { 35 | beforeEach(function () { 36 | sinon.restore(); 37 | }); 38 | 39 | it('should not instantiate downstream topic if not defined in config', async function () { 40 | const stubPubSub = sinon.stub(pubsub); 41 | app.__set__('pubsub', stubPubSub); 42 | 43 | await publishProtoMsgDownstream('EVENT', '', undefined); 44 | 45 | assert(stubPubSub.topic.notCalled); 46 | }); 47 | 48 | it('should publish downstream message', async function () { 49 | const stubTopic = sinon.createStubInstance(Topic); 50 | stubTopic.publishMessage.resolves(); 51 | const stubPubSub = sinon.stub(pubsub); 52 | stubPubSub.topic.returns(stubTopic); 53 | 54 | app.__set__('pubsub', stubPubSub); 55 | app.__set__( 56 | 'createProtobufMessage', 57 | sinon.stub().returns(Buffer.from('{}')), 58 | ); 59 | 60 | await publishProtoMsgDownstream('EVENT', '', 'the/topic'); 61 | assert(stubTopic.publishMessage.calledOnce); 62 | }); 63 | }); 64 | 65 | const createProtobufMessage = app.__get__('createProtobufMessage'); 66 | describe('#createProtobufMessage', () => { 67 | it('should create a Protobuf message that can be validated', async function () { 68 | const message = await createProtobufMessage(createDownstreamMsg()); 69 | const result = message.toJSON(); 70 | 71 | const root = await protobuf.load( 72 | 'src/scaler/scaler-core/downstream.schema.proto', 73 | ); 74 | const DownstreamEvent = root.lookupType('DownstreamEvent'); 75 | assert.equals(DownstreamEvent.verify(result), null); 76 | }); 77 | }); 78 | -------------------------------------------------------------------------------- /src/scaler/scaler-core/utils.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | /* 17 | * Helper functions 18 | */ 19 | 20 | // Create PubSub client and cache it 21 | const {PubSub} = require('@google-cloud/pubsub'); 22 | const pubsub = new PubSub(); 23 | const protobuf = require('protobufjs'); 24 | const {logger} = require('../../autoscaler-common/logger'); 25 | 26 | /** 27 | * Format duration as human-readable text 28 | * 29 | * @param {number} millisec 30 | * @return {string} 31 | */ 32 | function convertMillisecToHumanReadable(millisec) { 33 | // By Nofi @ https://stackoverflow.com/a/32180863 34 | const seconds = millisec / 1000; 35 | const minutes = millisec / (1000 * 60); 36 | const hours = millisec / (1000 * 60 * 60); 37 | const days = millisec / (1000 * 60 * 60 * 24); 38 | 39 | if (seconds < 60) { 40 | return seconds.toFixed(1) + ' Sec'; 41 | } else if (minutes < 60) { 42 | return minutes.toFixed(1) + ' Min'; 43 | } else if (hours < 24) { 44 | return hours.toFixed(1) + ' Hrs'; 45 | } else { 46 | return days.toFixed(1) + ' Days'; 47 | } 48 | } 49 | 50 | /** 51 | * Create Pub/Sub messages with Protobuf schema 52 | * @param {Object} jsonData 53 | * @return {Promise} 54 | */ 55 | async function createProtobufMessage(jsonData) { 56 | const root = await protobuf.load( 57 | 'src/scaler/scaler-core/downstream.schema.proto', 58 | ); 59 | const DownstreamEvent = root.lookupType('DownstreamEvent'); 60 | return DownstreamEvent.create(jsonData); 61 | } 62 | 63 | /** 64 | * Publish pub/sub message 65 | * 66 | * @param {string} eventName 67 | * @param {Object} jsonData 68 | * @param {string} [topicId] 69 | * @return {Promise<*>} 70 | */ 71 | async function publishProtoMsgDownstream(eventName, jsonData, topicId) { 72 | if (!topicId) { 73 | logger.debug( 74 | `If you want ${eventName} messages published downstream then specify ` + 75 | 'downstreamPubSubTopic in your config.', 76 | ); 77 | return Promise.resolve(); 78 | } 79 | 80 | const topic = pubsub.topic(topicId); 81 | const message = await createProtobufMessage(jsonData); 82 | const data = Buffer.from(JSON.stringify(message.toJSON())); 83 | const attributes = {event: eventName}; 84 | 85 | return topic 86 | .publishMessage({data: data, attributes: attributes}) 87 | .then(() => 88 | logger.info( 89 | `Published ${eventName} message downstream to topic: ${topicId}`, 90 | ), 91 | ) 92 | .catch((err) => { 93 | logger.error({ 94 | message: `An error occurred publishing ${eventName} message downstream to topic: ${topicId}: ${err}`, 95 | err: err, 96 | }); 97 | }); 98 | } 99 | 100 | module.exports = { 101 | convertMillisecToHumanReadable, 102 | publishProtoMsgDownstream, 103 | }; 104 | -------------------------------------------------------------------------------- /src/unified-scaler.js: -------------------------------------------------------------------------------- 1 | /* Copyright 2024 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License 14 | */ 15 | 16 | const pollerCore = require('./poller/poller-core'); 17 | const scalerCore = require('./scaler/scaler-core'); 18 | const {logger} = require('./autoscaler-common/logger'); 19 | const yaml = require('js-yaml'); 20 | const fs = require('fs/promises'); 21 | const CountersBase = require('./autoscaler-common/counters-base'); 22 | const {version: packageVersion} = require('../package.json'); 23 | 24 | /** 25 | * Startup function for unified poller/scaler 26 | */ 27 | async function main() { 28 | const DEFAULT_CONFIG_LOCATION = 29 | '/etc/autoscaler-config/autoscaler-config.yaml'; 30 | 31 | logger.info( 32 | `Autoscaler unified Poller/Scaler v${packageVersion} job started`, 33 | ); 34 | 35 | // This is not a long-running process, but we only want to flush the counters 36 | // when it has completed. So disable flushing here, and enable and flush in 37 | // the finally {} block 38 | CountersBase.setTryFlushEnabled(false); 39 | 40 | let configLocation = DEFAULT_CONFIG_LOCATION; 41 | 42 | /* 43 | * If set, the AUTOSCALER_CONFIG environment variable is used to 44 | * retrieve the configuration for this instance of the Poller. 45 | * Please refer to the documentation in the README.md for GKE 46 | * deployment for more details. 47 | */ 48 | 49 | if (process.env.AUTOSCALER_CONFIG) { 50 | configLocation = process.env.AUTOSCALER_CONFIG; 51 | logger.debug(`Using custom config location ${configLocation}`); 52 | } else { 53 | logger.debug(`Using default config location ${configLocation}`); 54 | } 55 | 56 | try { 57 | const config = await fs.readFile(configLocation, {encoding: 'utf8'}); 58 | const clusters = await pollerCore.checkMemorystoreClusterScaleMetricsLocal( 59 | JSON.stringify(yaml.load(config)), 60 | ); 61 | for (const cluster of clusters) { 62 | await scalerCore.scaleMemorystoreClusterLocal(cluster); 63 | } 64 | } catch (err) { 65 | logger.error({ 66 | message: 'Error in unified poller/scaler wrapper: ${err}', 67 | err: err, 68 | }); 69 | } finally { 70 | CountersBase.setTryFlushEnabled(true); 71 | await CountersBase.tryFlush(); 72 | } 73 | } 74 | 75 | module.exports = { 76 | main, 77 | }; 78 | -------------------------------------------------------------------------------- /terraform/cloud-functions/distributed/app-project/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/archive" { 5 | version = "2.6.0" 6 | hashes = [ 7 | "h1:rYAubRk7UHC/fzYqFV/VHc+7VIY01ugCxauyTYCNf9E=", 8 | "zh:29273484f7423b7c5b3f5df34ccfc53e52bb5e3d7f46a81b65908e7a8fd69072", 9 | "zh:3cba58ec3aea5f301caf2acc31e184c55d994cc648126cac39c63ae509a14179", 10 | "zh:55170cd17dbfdea842852c6ae2416d057fec631ba49f3bb6466a7268cd39130e", 11 | "zh:7197db402ba35631930c3a4814520f0ebe980ae3acb7f8b5a6f70ec90dc4a388", 12 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 13 | "zh:8bf7fe0915d7fb152a3a6b9162614d2ec82749a06dba13fab3f98d33c020ec4f", 14 | "zh:8ce811844fd53adb0dabc9a541f8cb43aacfa7d8e39324e4bd3592b3428f5bfb", 15 | "zh:bca795bca815b8ac90e3054c0a9ab1ccfb16eedbb3418f8ad473fc5ad6bf0ef7", 16 | "zh:d9355a18df5a36cf19580748b23249de2eb445c231c36a353709f8f40a6c8432", 17 | "zh:dc32cc32cfd8abf8752d34f2a783de0d3f7200c573b885ecb64ece5acea173b4", 18 | "zh:ef498e20391bf7a280d0fd6fd6675621c85fbe4e92f0f517ae4394747db89bde", 19 | "zh:f2bc5226c765b0c8055a7b6207d0fe1eb9484e3ec8880649d158827ac6ed3b22", 20 | ] 21 | } 22 | 23 | provider "registry.terraform.io/hashicorp/google" { 24 | version = "6.24.0" 25 | constraints = "6.24.0" 26 | hashes = [ 27 | "h1:0g0VLxQFohTh0HQ3YnRs9z/cl+RtIxU8Zd9EYjZDm/8=", 28 | "h1:18nQEvcmcR7nTC4ma/1LBBKSldnaZpPNfF1m5XMZNG0=", 29 | "h1:3NQ4+5rIrSR78tXHCeWRuhiHxp0OFE0rAJqBhsgm6cw=", 30 | "h1:BkGI/656AfOb78XrOFS1bWjuFjvOOgmepr6gBNOyIxg=", 31 | "h1:OJYiiWmCEouSlzLQR5AMeb/c2W869qwhUUIRY34JQoU=", 32 | "h1:ORt5a/ebg4aqGJalvgN9s+Lk+qz40Hj+SmZz1mxZLDM=", 33 | "h1:Oo5n66o4fJVPz8b7zCnhrd9KNmXb90Z0DxD2u9tz5pU=", 34 | "h1:Y9f/Q1dBiYpd8BvfSrkvSF3smM0SlHCoh66+KF0uzB8=", 35 | "h1:aTJQx01EoVjhnHP5DfErYpW+BOG/zT4q5h7go/BV+WM=", 36 | "h1:vNgWebq55mC+z/juRr3ZpJYxIYot5QOtLospK8BOAbU=", 37 | "h1:wF8iekISdAP+RJVX/Xb0gNxCImAiRPKlOBTgQ+P6qvw=", 38 | "zh:0e7bb01149f50eabab725e8a0efadcb1cbfd7389f45adfb12e04f4f15a4fb5eb", 39 | "zh:4172d07d61168e4246125e77ba5c67e96309783e2a8cd885cc51f3a73e7f14e2", 40 | "zh:6952c1305d10b456170b2b7c34f0013ce4fd67161f6e7aa6daef61490da60252", 41 | "zh:8ab7621209b352b12a0947865975ff83048c55a870a11306603b1b8052a3926b", 42 | "zh:ba93efa1562d17f65001f8cce016ba903289ed985a7bec4b4d6339e3f52af3eb", 43 | "zh:bc70ee209b816f74c9ffeaca9d3c85191ba8173f9f3f19425821a1ae9e4d47ea", 44 | "zh:c9e8432861770f86a38a29c74d57cd5ecd7bec38fff0c719ed6136d34ae95ccd", 45 | "zh:dfabe73e6de0cefa0b158f82647ca15325aa42bd0d8894ff82de02aed1c5814f", 46 | "zh:e2798adc0d6edf9eb5e9ccbc2f4cd3914a0c76258e20690c86d7404490c10904", 47 | "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", 48 | "zh:f8884173e9334c3c408ecb869e44478061ffd1f23de6a204f5ab454a55ea9f12", 49 | "zh:f8ecbc3274389f6fbb5ff5fc10f06db2390d02f50c0b35ef1c07f0203c341717", 50 | ] 51 | } 52 | 53 | provider "registry.terraform.io/hashicorp/time" { 54 | version = "0.12.1" 55 | hashes = [ 56 | "h1:6BhxSYBJdBBKyuqatOGkuPKVenfx6UmLdiI13Pb3his=", 57 | "zh:090023137df8effe8804e81c65f636dadf8f9d35b79c3afff282d39367ba44b2", 58 | "zh:26f1e458358ba55f6558613f1427dcfa6ae2be5119b722d0b3adb27cd001efea", 59 | "zh:272ccc73a03384b72b964918c7afeb22c2e6be22460d92b150aaf28f29a7d511", 60 | "zh:438b8c74f5ed62fe921bd1078abe628a6675e44912933100ea4fa26863e340e9", 61 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 62 | "zh:85c8bd8eefc4afc33445de2ee7fbf33a7807bc34eb3734b8eefa4e98e4cddf38", 63 | "zh:98bbe309c9ff5b2352de6a047e0ec6c7e3764b4ed3dfd370839c4be2fbfff869", 64 | "zh:9c7bf8c56da1b124e0e2f3210a1915e778bab2be924481af684695b52672891e", 65 | "zh:d2200f7f6ab8ecb8373cda796b864ad4867f5c255cff9d3b032f666e4c78f625", 66 | "zh:d8c7926feaddfdc08d5ebb41b03445166df8c125417b28d64712dccd9feef136", 67 | "zh:e2412a192fc340c61b373d6c20c9d805d7d3dee6c720c34db23c2a8ff0abd71b", 68 | "zh:e6ac6bba391afe728a099df344dbd6481425b06d61697522017b8f7a59957d44", 69 | ] 70 | } 71 | -------------------------------------------------------------------------------- /terraform/cloud-functions/distributed/app-project/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "scheduler_job_id" { 18 | value = module.autoscaler-scheduler.scheduler_job_id 19 | description = "ID of the Scheduler job" 20 | } 21 | 22 | output "memorystore_discovery_endpoint" { 23 | value = module.autoscaler-memorystore-cluster.memorystore_discovery_endpoint != null ? module.autoscaler-memorystore-cluster.memorystore_discovery_endpoint.address : null 24 | description = "Memorystore discovery endpoint (currently single value)" 25 | } 26 | 27 | output "test_vm_zone" { 28 | value = length(module.autoscaler-test-vm) > 0 ? one(module.autoscaler-test-vm).zone : null 29 | description = "Zone of the test VM" 30 | } 31 | 32 | output "test_vm_name" { 33 | value = length(module.autoscaler-test-vm) > 0 ? one(module.autoscaler-test-vm).instance_name : null 34 | description = "Name of the test VM" 35 | } 36 | -------------------------------------------------------------------------------- /terraform/cloud-functions/distributed/app-project/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | } 20 | 21 | variable "region" { 22 | type = string 23 | } 24 | 25 | variable "memorystore_cluster_name" { 26 | type = string 27 | default = "autoscaler-target-memorystore-cluster" 28 | } 29 | 30 | variable "memorystore_shard_count" { 31 | type = number 32 | default = 1 33 | } 34 | 35 | variable "memorystore_replica_count" { 36 | type = number 37 | default = 1 38 | } 39 | 40 | variable "app_project_id" { 41 | description = "The project where the Memorystore Cluster(s) live. If specified and different than project_id => centralized deployment" 42 | type = string 43 | default = "" 44 | } 45 | 46 | variable "state_project_id" { 47 | type = string 48 | } 49 | 50 | variable "terraform_memorystore_cluster" { 51 | description = "If set to true, Terraform will create a test Memorystore cluster." 52 | type = bool 53 | default = true 54 | } 55 | 56 | variable "terraform_spanner_state" { 57 | description = "If set to true, Terraform will create a Spanner instance for autoscaler state." 58 | type = bool 59 | default = false 60 | } 61 | 62 | variable "spanner_state_name" { 63 | type = string 64 | default = "memorystore-autoscaler-state" 65 | } 66 | 67 | variable "spanner_state_database" { 68 | type = string 69 | default = "memorystore-autoscaler-state" 70 | } 71 | 72 | variable "firestore_state_database" { 73 | type = string 74 | default = "memorystore-autoscaler-state" 75 | } 76 | 77 | variable "terraform_test_vm" { 78 | description = "If set to true, Terraform will create a test VM with Memorystore utils installed." 79 | type = bool 80 | default = false 81 | } 82 | 83 | variable "terraform_test_vm_name" { 84 | description = "Name for the optional test VM" 85 | type = string 86 | default = "terraform-test-vm" 87 | } 88 | 89 | variable "terraform_dashboard" { 90 | description = "If set to true, Terraform will create a Cloud Monitoring dashboard including important Memorystore Cluster metrics." 91 | type = bool 92 | default = true 93 | } 94 | 95 | variable "ip_range" { 96 | description = "IP range for the network" 97 | type = string 98 | default = "10.0.0.0/24" 99 | } 100 | 101 | variable "memorystore_engine" { 102 | description = "The underlying engine to use" 103 | type = string 104 | default = "REDIS" 105 | } 106 | 107 | locals { 108 | # By default, these config files produce a per-project deployment 109 | # If you want a centralized deployment instead, then specify 110 | # an app_project_id that is different from project_id 111 | app_project_id = var.app_project_id == "" ? var.project_id : var.app_project_id 112 | } 113 | -------------------------------------------------------------------------------- /terraform/cloud-functions/distributed/autoscaler-project/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_providers { 19 | google = { 20 | source = "hashicorp/google" 21 | version = "6.24.0" 22 | } 23 | } 24 | } 25 | 26 | provider "google" { 27 | project = var.project_id 28 | region = var.region 29 | } 30 | 31 | resource "google_service_account" "poller_sa" { 32 | account_id = "poller-sa" 33 | display_name = "Memorystore Cluster Autoscaler - Poller SA" 34 | } 35 | 36 | resource "google_service_account" "scaler_sa" { 37 | account_id = "scaler-sa" 38 | display_name = "Memorystore Cluster Autoscaler - Scaler SA" 39 | } 40 | 41 | module "autoscaler-base" { 42 | source = "../../../modules/autoscaler-base" 43 | 44 | project_id = var.project_id 45 | poller_sa_email = google_service_account.poller_sa.email 46 | scaler_sa_email = google_service_account.scaler_sa.email 47 | } 48 | 49 | module "autoscaler-functions" { 50 | source = "../../../modules/autoscaler-functions" 51 | 52 | project_id = var.project_id 53 | region = var.region 54 | poller_sa_email = google_service_account.poller_sa.email 55 | scaler_sa_email = google_service_account.scaler_sa.email 56 | 57 | forwarder_sa_emails = var.forwarder_sa_emails 58 | build_sa_id = module.autoscaler-base.build_sa_id 59 | } 60 | 61 | module "firestore" { 62 | count = !var.terraform_spanner_state ? 1 : 0 63 | source = "../../../modules/autoscaler-firestore" 64 | 65 | project_id = var.project_id 66 | region = var.region 67 | firestore_state_database = var.firestore_state_database 68 | 69 | poller_sa_email = google_service_account.poller_sa.email 70 | scaler_sa_email = google_service_account.scaler_sa.email 71 | } 72 | 73 | module "autoscaler-spanner" { 74 | source = "../../../modules/autoscaler-spanner" 75 | 76 | region = var.region 77 | project_id = var.project_id 78 | terraform_spanner_state = var.terraform_spanner_state 79 | spanner_state_name = var.spanner_state_name 80 | spanner_state_database = var.spanner_state_database 81 | 82 | poller_sa_email = google_service_account.poller_sa.email 83 | scaler_sa_email = google_service_account.scaler_sa.email 84 | } 85 | -------------------------------------------------------------------------------- /terraform/cloud-functions/distributed/autoscaler-project/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "poller_sa_email" { 18 | value = google_service_account.poller_sa.email 19 | } 20 | 21 | output "scaler_sa_email" { 22 | value = google_service_account.scaler_sa.email 23 | } 24 | 25 | output "poller_topic" { 26 | value = module.autoscaler-functions.poller_topic 27 | } 28 | 29 | output "scaler_topic" { 30 | value = module.autoscaler-functions.scaler_topic 31 | } 32 | -------------------------------------------------------------------------------- /terraform/cloud-functions/distributed/autoscaler-project/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | } 20 | 21 | variable "region" { 22 | type = string 23 | } 24 | 25 | variable "forwarder_sa_emails" { 26 | type = list(string) 27 | // Example ["serviceAccount:forwarder_sa@app-project.iam.gserviceaccount.com"] 28 | default = [] 29 | } 30 | 31 | variable "terraform_spanner_state" { 32 | description = "If set to true, Terraform will create a Spanner instance for autoscaler state." 33 | type = bool 34 | default = false 35 | } 36 | 37 | variable "spanner_state_name" { 38 | type = string 39 | default = "memorystore-autoscaler-state" 40 | } 41 | 42 | variable "spanner_state_database" { 43 | type = string 44 | default = "memorystore-autoscaler-state" 45 | } 46 | 47 | variable "firestore_state_database" { 48 | type = string 49 | default = "memorystore-autoscaler-state" 50 | } 51 | 52 | variable "terraform_dashboard" { 53 | description = "If set to true, Terraform will create a Cloud Monitoring dashboard including important Spanner metrics." 54 | type = bool 55 | default = true 56 | } 57 | -------------------------------------------------------------------------------- /terraform/cloud-functions/per-project/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "scheduler_job_id" { 18 | value = module.autoscaler-scheduler.scheduler_job_id 19 | description = "ID of the Scheduler job" 20 | } 21 | 22 | output "memorystore_discovery_endpoint" { 23 | value = module.autoscaler-memorystore-cluster.memorystore_discovery_endpoint != null ? module.autoscaler-memorystore-cluster.memorystore_discovery_endpoint.address : null 24 | description = "Memorystore discovery endpoint (currently single value)" 25 | } 26 | 27 | output "test_vm_zone" { 28 | value = length(module.autoscaler-test-vm) > 0 ? one(module.autoscaler-test-vm).zone : null 29 | description = "Zone of the test VM" 30 | } 31 | 32 | output "test_vm_name" { 33 | value = length(module.autoscaler-test-vm) > 0 ? one(module.autoscaler-test-vm).instance_name : null 34 | description = "Name of the test VM" 35 | } 36 | -------------------------------------------------------------------------------- /terraform/cloud-functions/per-project/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | } 20 | 21 | variable "region" { 22 | type = string 23 | } 24 | 25 | variable "memorystore_cluster_name" { 26 | type = string 27 | default = "autoscaler-target-memorystore-cluster" 28 | } 29 | 30 | variable "memorystore_shard_count" { 31 | type = number 32 | default = 1 33 | } 34 | 35 | variable "memorystore_replica_count" { 36 | type = number 37 | default = 1 38 | } 39 | 40 | variable "app_project_id" { 41 | description = "The project where the Memorystore Cluster(s) live. If specified and different than project_id => centralized deployment" 42 | type = string 43 | default = "" 44 | } 45 | 46 | variable "terraform_memorystore_cluster" { 47 | description = "If set to true, Terraform will create a test Memorystore cluster." 48 | type = bool 49 | default = true 50 | } 51 | 52 | variable "terraform_spanner_state" { 53 | description = "If set to true, Terraform will create a Spanner instance for autoscaler state." 54 | type = bool 55 | default = false 56 | } 57 | 58 | variable "spanner_state_name" { 59 | type = string 60 | default = "memorystore-autoscaler-state" 61 | } 62 | 63 | variable "spanner_state_database" { 64 | type = string 65 | default = "memorystore-autoscaler-state" 66 | } 67 | 68 | variable "firestore_state_database" { 69 | type = string 70 | default = "memorystore-autoscaler-state" 71 | } 72 | 73 | variable "terraform_test_vm" { 74 | description = "If set to true, Terraform will create a test VM with Memorystore utils installed." 75 | type = bool 76 | default = false 77 | } 78 | 79 | variable "terraform_test_vm_name" { 80 | description = "Name for the optional test VM" 81 | type = string 82 | default = "terraform-test-vm" 83 | } 84 | 85 | variable "terraform_dashboard" { 86 | description = "If set to true, Terraform will create a Cloud Monitoring dashboard including important Memorystore Cluster metrics." 87 | type = bool 88 | default = true 89 | } 90 | 91 | variable "ip_range" { 92 | description = "IP range for the network" 93 | type = string 94 | default = "10.0.0.0/24" 95 | } 96 | 97 | variable "memorystore_engine" { 98 | description = "The underlying engine to use" 99 | type = string 100 | default = "REDIS" 101 | } 102 | 103 | locals { 104 | # By default, these config files produce a per-project deployment 105 | # If you want a centralized deployment instead, then specify 106 | # an app_project_id that is different from project_id 107 | app_project_id = var.app_project_id == "" ? var.project_id : var.app_project_id 108 | } 109 | -------------------------------------------------------------------------------- /terraform/gke/unified/README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 |

OSS Memorystore Cluster Autoscaler

4 | Autoscaler 5 | 6 |

7 | 8 | Set up the Autoscaler in GKE using Terraform configuration files 9 |
10 | Home 11 | · 12 | Scaler component 13 | · 14 | Poller component 15 | · 16 | Forwarder component 17 | · 18 | Terraform configuration 19 | · 20 | Monitoring 21 |
22 | Cloud Run functions 23 | · 24 | Google Kubernetes Engine 25 |

26 | 27 |

28 | 29 | ## Overview 30 | 31 | This directory contains Terraform configuration files to quickly set 32 | up the infrastructure for your Autoscaler for a unified deployment to 33 | [Google Kubernetes Engine (GKE)][gke]. 34 | 35 | Please see the documentation [here](../README.md). 36 | 37 | [gke]: https://cloud.google.com/kubernetes-engine 38 | -------------------------------------------------------------------------------- /terraform/gke/unified/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "memorystore_discovery_endpoint" { 18 | value = module.autoscaler-memorystore-cluster.memorystore_discovery_endpoint != null ? module.autoscaler-memorystore-cluster.memorystore_discovery_endpoint.address : null 19 | description = "Memorystore discovery endpoint (currently single value)" 20 | } 21 | 22 | output "test_vm_zone" { 23 | value = length(module.autoscaler-test-vm) > 0 ? one(module.autoscaler-test-vm).zone : null 24 | description = "Zone of the test VM" 25 | } 26 | 27 | output "test_vm_name" { 28 | value = length(module.autoscaler-test-vm) > 0 ? one(module.autoscaler-test-vm).instance_name : null 29 | description = "Name of the test VM" 30 | } 31 | -------------------------------------------------------------------------------- /terraform/gke/unified/test/gke_deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright 2024 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -ex 18 | 19 | export PROJECT_ID=$1 20 | export REGION=$2 21 | export TARGET_SHARDS=$3 22 | 23 | export REPO_ROOT=$(git rev-parse --show-toplevel) 24 | 25 | # Build the image from the root of the repo 26 | 27 | cd ${REPO_ROOT} 28 | 29 | gcloud config set project ${PROJECT_ID} 30 | gcloud container clusters get-credentials memorystore-cluster-autoscaler --region=${REGION} 31 | gcloud beta builds submit . --config=cloudbuild-unified.yaml --region=${REGION} \ 32 | --service-account="projects/${PROJECT_ID}/serviceAccounts/build-sa@${PROJECT_ID}.iam.gserviceaccount.com" 33 | 34 | SCALER_PATH="${REGION}-docker.pkg.dev/${PROJECT_ID}/memorystore-cluster-autoscaler/scaler" 35 | SCALER_SHA=$(gcloud artifacts docker images describe ${SCALER_PATH}:latest --format='value(image_summary.digest)') 36 | SCALER_IMAGE="${SCALER_PATH}@${SCALER_SHA}" 37 | 38 | # Render the manifests from the templates, note we cannot use kpt/envsubst for some operations 39 | 40 | cd kubernetes/unified 41 | 42 | for template in $(ls autoscaler-config/*.template) ; do envsubst < ${template} > ${template%.*} ; done 43 | sed -i "s|image: scaler-image|image: ${SCALER_IMAGE}|g" autoscaler-pkg/scaler/scaler.yaml 44 | sed -i "s/minSize: 1/minSize: ${TARGET_SHARDS}/g" autoscaler-config/autoscaler-config.yaml 45 | 46 | # Apply the manifests 47 | 48 | kubectl apply -f autoscaler-pkg/ --recursive 49 | kubectl apply -f autoscaler-config/ 50 | -------------------------------------------------------------------------------- /terraform/gke/unified/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | } 20 | 21 | variable "region" { 22 | type = string 23 | } 24 | 25 | variable "memorystore_cluster_name" { 26 | type = string 27 | default = "autoscaler-target-memorystore-cluster" 28 | } 29 | 30 | variable "memorystore_shard_count" { 31 | type = number 32 | default = 1 33 | } 34 | 35 | variable "memorystore_replica_count" { 36 | type = number 37 | default = 1 38 | } 39 | 40 | variable "terraform_memorystore_cluster" { 41 | description = "If set to true, Terraform will create a test Memorystore cluster." 42 | type = bool 43 | default = true 44 | } 45 | 46 | variable "terraform_spanner_state" { 47 | description = "If set to true, Terraform will create a Spanner instance for autoscaler state." 48 | type = bool 49 | default = false 50 | } 51 | 52 | variable "spanner_state_name" { 53 | type = string 54 | default = "memorystore-autoscaler-state" 55 | } 56 | 57 | variable "spanner_state_database" { 58 | type = string 59 | default = "memorystore-autoscaler-state" 60 | } 61 | 62 | variable "firestore_state_database" { 63 | type = string 64 | default = "memorystore-autoscaler-state" 65 | } 66 | 67 | variable "terraform_test_vm" { 68 | description = "If set to true, Terraform will create a test VM with Memorystore utils installed." 69 | type = bool 70 | default = false 71 | } 72 | 73 | variable "terraform_test_vm_name" { 74 | description = "Name for the optional test VM" 75 | type = string 76 | default = "terraform-test-vm" 77 | } 78 | 79 | variable "terraform_dashboard" { 80 | description = "If set to true, Terraform will create a Cloud Monitoring dashboard including important Memorystore Cluster metrics." 81 | type = bool 82 | default = true 83 | } 84 | 85 | variable "ip_range" { 86 | description = "IP range for the network" 87 | type = string 88 | default = "10.0.0.0/24" 89 | } 90 | 91 | variable "memorystore_engine" { 92 | description = "The underlying engine to use" 93 | type = string 94 | default = "REDIS" 95 | } 96 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-base/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | resource "random_id" "role_suffix" { 19 | byte_length = 4 20 | } 21 | 22 | # Limited role for Poller 23 | resource "google_project_iam_custom_role" "metrics_viewer_iam_role" { 24 | project = var.project_id 25 | role_id = "memorystoreClusterAutoscalerMetricsViewer_${random_id.role_suffix.hex}" 26 | title = "Memorystore Cluster Autoscaler Metrics Viewer Role" 27 | description = "Allows a principal to get Memorystore Cluster instances and view time series metrics" 28 | permissions = [ 29 | "memorystore.instances.get", 30 | "memorystore.instances.list", 31 | "monitoring.timeSeries.list", 32 | "redis.clusters.get", 33 | "redis.clusters.list" 34 | ] 35 | } 36 | 37 | # Assign custom role to Poller 38 | resource "google_project_iam_member" "poller_metrics_viewer_iam" { 39 | role = google_project_iam_custom_role.metrics_viewer_iam_role.name 40 | project = var.project_id 41 | member = "serviceAccount:${var.poller_sa_email}" 42 | } 43 | 44 | # Limited role for Scaler 45 | resource "google_project_iam_custom_role" "capacity_manager_iam_role" { 46 | project = var.project_id 47 | role_id = "memorystoreClusterAutoscalerCapacityManager_${random_id.role_suffix.hex}" 48 | title = "Memorystore Cluster Autoscaler Capacity Manager Role" 49 | description = "Allows a principal to scale Memorystore Cluster instances" 50 | permissions = [ 51 | "memorystore.instances.get", 52 | "memorystore.instances.update", 53 | "memorystore.operations.get", 54 | "redis.clusters.get", 55 | "redis.clusters.update", 56 | "redis.operations.get" 57 | ] 58 | } 59 | 60 | # Assign custom role to Scaler 61 | resource "google_project_iam_member" "scaler_update_capacity_iam" { 62 | role = google_project_iam_custom_role.capacity_manager_iam_role.name 63 | project = var.project_id 64 | member = "serviceAccount:${var.scaler_sa_email}" 65 | } 66 | 67 | resource "google_pubsub_topic_iam_member" "scaler_downstream_pub_iam" { 68 | project = var.project_id 69 | topic = google_pubsub_topic.downstream_topic.name 70 | role = "roles/pubsub.publisher" 71 | member = "serviceAccount:${var.scaler_sa_email}" 72 | } 73 | 74 | resource "google_pubsub_schema" "scaler_downstream_pubsub_schema" { 75 | name = "downstream-schema" 76 | type = "PROTOCOL_BUFFER" 77 | definition = file("${path.module}/../../../src/scaler/scaler-core/downstream.schema.proto") 78 | } 79 | 80 | resource "google_project_iam_member" "metrics_publisher_iam_poller" { 81 | project = var.project_id 82 | role = "roles/monitoring.metricWriter" 83 | member = "serviceAccount:${var.poller_sa_email}" 84 | } 85 | 86 | resource "google_project_iam_member" "metrics_publisher_iam_scaler" { 87 | project = var.project_id 88 | role = "roles/monitoring.metricWriter" 89 | member = "serviceAccount:${var.scaler_sa_email}" 90 | } 91 | 92 | resource "google_service_account" "build_sa" { 93 | account_id = "build-sa" 94 | display_name = "Autoscaler - Cloud Build Builder Service Account" 95 | } 96 | 97 | resource "google_project_iam_member" "build_iam" { 98 | for_each = toset([ 99 | "roles/artifactregistry.writer", 100 | "roles/logging.logWriter", 101 | "roles/storage.objectViewer", 102 | ]) 103 | project = var.project_id 104 | role = each.value 105 | member = "serviceAccount:${google_service_account.build_sa.email}" 106 | } 107 | 108 | resource "time_sleep" "wait_for_iam" { 109 | depends_on = [google_project_iam_member.build_iam] 110 | create_duration = "90s" 111 | } 112 | 113 | resource "google_pubsub_topic" "downstream_topic" { 114 | name = "downstream-topic" 115 | 116 | depends_on = [google_pubsub_schema.scaler_downstream_pubsub_schema] 117 | 118 | schema_settings { 119 | schema = google_pubsub_schema.scaler_downstream_pubsub_schema.id 120 | encoding = "JSON" 121 | } 122 | 123 | lifecycle { 124 | replace_triggered_by = [google_pubsub_schema.scaler_downstream_pubsub_schema] 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-base/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "build_sa_id" { 18 | value = google_service_account.build_sa.id 19 | description = "Service account ID for Builder SA" 20 | depends_on = [time_sleep.wait_for_iam] 21 | } 22 | 23 | output "build_sa_email" { 24 | value = google_service_account.build_sa.email 25 | description = "Service account email for Builder SA" 26 | depends_on = [time_sleep.wait_for_iam] 27 | } 28 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-base/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | } 20 | 21 | variable "poller_sa_email" { 22 | type = string 23 | } 24 | 25 | variable "scaler_sa_email" { 26 | type = string 27 | } 28 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-firestore/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_project_iam_member" "scaler_sa_firestore" { 18 | project = var.project_id 19 | role = "roles/datastore.user" 20 | member = "serviceAccount:${var.scaler_sa_email}" 21 | } 22 | 23 | resource "google_firestore_database" "database" { 24 | project = var.project_id 25 | name = var.firestore_state_database 26 | location_id = var.region 27 | type = "FIRESTORE_NATIVE" 28 | app_engine_integration_mode = "DISABLED" 29 | delete_protection_state = "DELETE_PROTECTION_DISABLED" 30 | deletion_policy = "DELETE" 31 | } 32 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-firestore/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | } 20 | 21 | variable "region" { 22 | type = string 23 | } 24 | 25 | variable "poller_sa_email" { 26 | type = string 27 | } 28 | 29 | variable "scaler_sa_email" { 30 | type = string 31 | } 32 | 33 | variable "firestore_state_database" { 34 | type = string 35 | } 36 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-forwarder/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "forwarder_topic" { 18 | value = google_pubsub_topic.forwarder_topic.id 19 | description = "PubSub topic used by the forwarder function" 20 | } 21 | 22 | output "forwarder_sa_email" { 23 | value = google_service_account.forwarder_sa.email 24 | description = "Email of the forwarder service account" 25 | } 26 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-forwarder/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | } 20 | 21 | variable "region" { 22 | type = string 23 | } 24 | 25 | variable "nodejs_version" { 26 | type = string 27 | default = "20" 28 | } 29 | 30 | variable "local_output_path" { 31 | type = string 32 | default = "build" 33 | } 34 | 35 | variable "target_pubsub_topic" { 36 | type = string 37 | } 38 | 39 | variable "uniform_bucket_level_access" { 40 | type = bool 41 | default = true 42 | } 43 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-functions/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "poller_topic" { 18 | value = google_pubsub_topic.poller_topic.id 19 | description = "PubSub topic used by the poller function" 20 | } 21 | 22 | output "scaler_topic" { 23 | value = google_pubsub_topic.scaler_topic.id 24 | description = "PubSub topic used by the scaler function" 25 | } 26 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-functions/variables.tf: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | } 20 | 21 | variable "region" { 22 | type = string 23 | } 24 | 25 | variable "nodejs_version" { 26 | type = string 27 | default = "20" 28 | } 29 | 30 | variable "local_output_path" { 31 | type = string 32 | default = "build" 33 | } 34 | 35 | variable "uniform_bucket_level_access" { 36 | type = bool 37 | default = true 38 | } 39 | 40 | variable "poller_sa_email" { 41 | type = string 42 | } 43 | 44 | variable "scaler_sa_email" { 45 | type = string 46 | } 47 | variable "build_sa_id" { 48 | type = string 49 | // projects/{{project}}/serviceAccounts/{{email}} 50 | } 51 | 52 | variable "forwarder_sa_emails" { 53 | type = list(string) 54 | // Example ["serviceAccount:forwarder_sa@app-project.iam.gserviceaccount.com"] 55 | default = [] 56 | } 57 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-gke/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "service_account" { 18 | value = module.autoscaler-gke.service_account 19 | description = "Service account used to create the cluster and node pool(s)" 20 | } 21 | 22 | output "region" { 23 | value = module.autoscaler-gke.region 24 | description = "Region for development cluster" 25 | } 26 | 27 | output "cluster-name" { 28 | value = module.autoscaler-gke.name 29 | description = "Cluster Name" 30 | } 31 | 32 | output "endpoint" { 33 | value = module.autoscaler-gke.endpoint 34 | description = "Cluster endpoint used to identify the cluster" 35 | } 36 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-gke/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | description = "Project ID where the cluster will run" 19 | } 20 | 21 | variable "region" { 22 | description = "The name of the region to run the cluster" 23 | } 24 | 25 | variable "name" { 26 | description = "A unique name for the resource" 27 | } 28 | 29 | variable "network" { 30 | description = "The name of the network to use" 31 | } 32 | 33 | variable "subnetwork" { 34 | description = "The name of the subnet to use" 35 | } 36 | 37 | variable "ip_range_master" { 38 | description = "The range for the private master" 39 | } 40 | 41 | variable "ip_range_pods" { 42 | description = "The secondary range for the pods" 43 | } 44 | 45 | variable "ip_range_services" { 46 | description = "The secondary range for the services" 47 | } 48 | 49 | variable "namespace" { 50 | description = "The namespace to use for the services" 51 | default = "memorystore-cluster-autoscaler" 52 | } 53 | 54 | variable "poller_sa_email" { 55 | type = string 56 | } 57 | 58 | variable "scaler_sa_email" { 59 | type = string 60 | } 61 | 62 | variable "machine_type" { 63 | description = "Type of node to use to run the cluster" 64 | default = "e2-standard-2" 65 | } 66 | 67 | variable "minimum_node_pool_instances" { 68 | type = number 69 | description = "Number of node-pool instances to have active" 70 | default = 1 71 | } 72 | 73 | variable "maximum_node_pool_instances" { 74 | type = number 75 | description = "Maximum number of node-pool instances to scale to" 76 | default = 3 77 | } 78 | 79 | variable "release_channel" { 80 | type = string 81 | description = "(Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`." 82 | default = "STABLE" 83 | } 84 | 85 | variable "otel_collector_sa_name" { 86 | type = string 87 | description = "The name of the service account and workload identity to be created and used by the OpenTelemetry Collector workload" 88 | default = "otel-collector-sa" 89 | } 90 | 91 | variable "unified_components" { 92 | description = "Whether Poller and Scaler are unified" 93 | type = bool 94 | default = true 95 | } 96 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-memorystore-cluster/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | provider_meta "google" { 19 | module_name = "cloud-solutions/memorystore-cluster-autoscaler-deploy-cluster-v1.0" 20 | } 21 | } 22 | 23 | locals { 24 | is_redis = var.memorystore_engine == "REDIS" 25 | } 26 | 27 | resource "google_redis_cluster" "memorystore_cluster" { 28 | count = var.terraform_memorystore_cluster && local.is_redis ? 1 : 0 29 | name = var.memorystore_cluster_name 30 | shard_count = var.memorystore_shard_count 31 | 32 | region = var.region 33 | replica_count = var.memorystore_replica_count 34 | transit_encryption_mode = "TRANSIT_ENCRYPTION_MODE_DISABLED" 35 | authorization_mode = "AUTH_MODE_DISABLED" 36 | 37 | psc_configs { 38 | network = var.network 39 | } 40 | 41 | zone_distribution_config { 42 | mode = "MULTI_ZONE" 43 | } 44 | 45 | deletion_protection_enabled = false 46 | 47 | lifecycle { 48 | ignore_changes = [shard_count, replica_count] 49 | } 50 | } 51 | 52 | resource "google_memorystore_instance" "memorystore_cluster" { 53 | count = var.terraform_memorystore_cluster && local.is_redis ? 0 : 1 54 | instance_id = var.memorystore_cluster_name 55 | shard_count = var.memorystore_shard_count 56 | 57 | location = var.region 58 | replica_count = var.memorystore_replica_count 59 | transit_encryption_mode = "TRANSIT_ENCRYPTION_DISABLED" 60 | authorization_mode = "AUTH_DISABLED" 61 | 62 | engine_version = var.valkey_version 63 | 64 | desired_psc_auto_connections { 65 | network = var.network 66 | project_id = var.project_id 67 | } 68 | 69 | zone_distribution_config { 70 | mode = "MULTI_ZONE" 71 | } 72 | 73 | deletion_protection_enabled = false 74 | 75 | lifecycle { 76 | ignore_changes = [shard_count, replica_count] 77 | } 78 | } 79 | 80 | resource "google_dns_record_set" "memorystore_cluster" { 81 | count = var.terraform_memorystore_cluster ? 1 : 0 82 | name = "cluster.${var.dns_zone.dns_name}" 83 | type = "A" 84 | ttl = 300 85 | 86 | managed_zone = var.dns_zone.name 87 | 88 | rrdatas = local.is_redis ? [one(google_redis_cluster.memorystore_cluster).discovery_endpoints[0].address] : [one(google_memorystore_instance.memorystore_cluster).discovery_endpoints[0].address] 89 | } 90 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-memorystore-cluster/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "memorystore_discovery_endpoint" { 18 | value = length(google_redis_cluster.memorystore_cluster) > 0 ? one(google_redis_cluster.memorystore_cluster).discovery_endpoints[0] : null 19 | description = "Memorystore discovery endpoint (currently single value)" 20 | } 21 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-memorystore-cluster/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | description = "Project ID where the cluster will run" 20 | } 21 | 22 | variable "memorystore_cluster_name" { 23 | type = string 24 | description = "A unique name for the cluster" 25 | } 26 | 27 | variable "region" { 28 | type = string 29 | description = "The name of the region to run the cluster" 30 | } 31 | 32 | variable "network" { 33 | type = string 34 | description = "The VPC network to host the cluster in" 35 | } 36 | 37 | variable "subnetwork" { 38 | type = string 39 | description = "The subnetwork to host the cluster in" 40 | } 41 | 42 | variable "memorystore_engine" { 43 | type = string 44 | description = "The underlying engine to use" 45 | validation { 46 | condition = contains(["REDIS", "VALKEY"], var.memorystore_engine) 47 | error_message = "Valid values for var: memorystore_engine are (REDIS, VALKEY)." 48 | } 49 | } 50 | 51 | variable "valkey_version" { 52 | type = string 53 | description = "The version of the valkey engine to use" 54 | default = "VALKEY_7_2" 55 | } 56 | 57 | variable "poller_sa_email" { 58 | type = string 59 | description = "The email of the poller service account" 60 | } 61 | 62 | variable "scaler_sa_email" { 63 | type = string 64 | description = "The email of the scaler service account" 65 | } 66 | 67 | variable "terraform_memorystore_cluster" { 68 | type = bool 69 | description = "If set to true, Terraform will create a test Memorystore cluster" 70 | } 71 | 72 | variable "dns_zone" { 73 | description = "The DNS zone to use" 74 | } 75 | 76 | variable "memorystore_shard_count" { 77 | type = number 78 | } 79 | 80 | variable "memorystore_replica_count" { 81 | type = number 82 | } 83 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-monitoring/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | locals { 18 | metrics_api_domain = "googleapis.com" 19 | metrics_api_subdomain = var.memorystore_engine == "REDIS" ? "redis" : "memorystore" 20 | metrics_engine_desc = var.memorystore_engine == "REDIS" ? "cluster" : "instance" 21 | metrics_prefix = "${local.metrics_api_subdomain}.${local.metrics_api_domain}/${local.metrics_engine_desc}" 22 | metrics_type = "${local.metrics_api_subdomain}.${local.metrics_api_domain}/${title(local.metrics_engine_desc)}" 23 | metrics_id_label = "${local.metrics_engine_desc}_id" 24 | } 25 | 26 | resource "google_monitoring_dashboard" "dashboard" { 27 | project = var.project_id 28 | 29 | dashboard_json = templatefile("${path.module}/dashboard.json.tftpl", { 30 | region = var.region 31 | memorystore_cluster_name = var.memorystore_cluster_name 32 | metrics_prefix = local.metrics_prefix 33 | metrics_type = local.metrics_type 34 | metrics_id_label = local.metrics_id_label 35 | 36 | cpu_average_threshold_out = var.cpu_average_threshold_out 37 | cpu_max_threshold_out = var.cpu_max_threshold_out 38 | cpu_average_threshold_in = var.cpu_average_threshold_in 39 | cpu_max_threshold_in = var.cpu_max_threshold_in 40 | 41 | memory_average_threshold_out = var.memory_average_threshold_out 42 | memory_max_threshold_out = var.memory_max_threshold_out 43 | memory_average_threshold_in = var.memory_average_threshold_in 44 | memory_max_threshold_in = var.memory_max_threshold_in 45 | }) 46 | 47 | lifecycle { 48 | ignore_changes = [ 49 | dashboard_json 50 | ] 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-monitoring/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "dashboard_id" { 18 | value = google_monitoring_dashboard.dashboard.id 19 | description = "Dashboard ID of important Memorystore Cluster metrics." 20 | } 21 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-monitoring/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | } 20 | 21 | variable "region" { 22 | type = string 23 | } 24 | 25 | variable "memorystore_cluster_name" { 26 | type = string 27 | } 28 | 29 | variable "memorystore_engine" { 30 | type = string 31 | description = "The underlying engine to use" 32 | validation { 33 | condition = contains(["REDIS", "VALKEY"], var.memorystore_engine) 34 | error_message = "Valid values for var: memorystore_engine are (REDIS, VALKEY)." 35 | } 36 | } 37 | 38 | variable "cpu_average_threshold_out" { 39 | type = number 40 | default = 0.7 41 | } 42 | 43 | variable "cpu_max_threshold_out" { 44 | type = number 45 | default = 0.8 46 | } 47 | 48 | variable "cpu_average_threshold_in" { 49 | type = number 50 | default = 0.5 51 | } 52 | 53 | variable "cpu_max_threshold_in" { 54 | type = number 55 | default = 0.6 56 | } 57 | 58 | variable "memory_average_threshold_out" { 59 | type = number 60 | default = 0.7 61 | } 62 | 63 | variable "memory_max_threshold_out" { 64 | type = number 65 | default = 0.8 66 | } 67 | 68 | variable "memory_average_threshold_in" { 69 | type = number 70 | default = 0.5 71 | } 72 | 73 | variable "memory_max_threshold_in" { 74 | type = number 75 | default = 0.6 76 | } 77 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-network/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | provider_meta "google" { 19 | module_name = "cloud-solutions/memorystore-cluster-autoscaler-deploy-network-v1.0" 20 | } 21 | } 22 | 23 | locals { 24 | psc_service_class = var.memorystore_engine == "REDIS" ? "gcp-memorystore-redis" : "gcp-memorystore" 25 | } 26 | 27 | resource "google_compute_network" "autoscaler_network" { 28 | name = "memorystore-cluster-autoscaler-network" 29 | auto_create_subnetworks = false 30 | } 31 | 32 | resource "google_compute_subnetwork" "autoscaler_subnetwork" { 33 | name = "memorystore-cluster-autoscaler-subnetwork" 34 | network = google_compute_network.autoscaler_network.id 35 | ip_cidr_range = var.ip_range 36 | private_ip_google_access = true 37 | } 38 | 39 | resource "google_compute_router" "router" { 40 | name = "app-router" 41 | region = var.region 42 | network = google_compute_network.autoscaler_network.id 43 | } 44 | 45 | resource "google_compute_router_nat" "nat" { 46 | name = "memorystore-cluster-autoscaler-nat" 47 | router = google_compute_router.router.name 48 | region = google_compute_router.router.region 49 | nat_ip_allocate_option = "AUTO_ONLY" 50 | source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" 51 | 52 | log_config { 53 | enable = true 54 | filter = "ERRORS_ONLY" 55 | } 56 | } 57 | 58 | resource "google_network_connectivity_service_connection_policy" "policy" { 59 | name = "memorystore-cluster-autoscaler-policy" 60 | location = var.region 61 | service_class = local.psc_service_class 62 | description = "Basic service connection policy" 63 | network = google_compute_network.autoscaler_network.id 64 | project = var.project_id 65 | 66 | psc_config { 67 | subnetworks = [google_compute_subnetwork.autoscaler_subnetwork.id] 68 | } 69 | } 70 | 71 | resource "google_dns_managed_zone" "private_zone" { 72 | name = "memorystore-cluster-autoscaler-private-zone" 73 | dns_name = "memorystore.private." 74 | description = "Private DNS zone for Memorystore Cluster Autoscaler" 75 | 76 | visibility = "private" 77 | labels = { 78 | "managed-by" = "terraform" 79 | } 80 | 81 | private_visibility_config { 82 | networks { 83 | network_url = google_compute_network.autoscaler_network.self_link 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-network/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "network" { 18 | value = google_compute_network.autoscaler_network.id 19 | } 20 | 21 | output "subnetwork" { 22 | value = google_compute_subnetwork.autoscaler_subnetwork.id 23 | } 24 | 25 | output "network_name" { 26 | value = google_compute_network.autoscaler_network.name 27 | } 28 | 29 | output "subnetwork_name" { 30 | value = google_compute_subnetwork.autoscaler_subnetwork.name 31 | } 32 | 33 | output "dns_zone" { 34 | value = google_dns_managed_zone.private_zone 35 | } 36 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-network/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | description = "Project ID where the cluster will run" 20 | } 21 | 22 | variable "region" { 23 | type = string 24 | description = "The name of the region to create the network" 25 | } 26 | 27 | variable "ip_range" { 28 | type = string 29 | description = "The range for the network" 30 | } 31 | 32 | variable "memorystore_engine" { 33 | type = string 34 | description = "The underlying engine to use" 35 | validation { 36 | condition = contains(["REDIS", "VALKEY"], var.memorystore_engine) 37 | error_message = "Valid values for var: memorystore_engine are (REDIS, VALKEY)." 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-scheduler/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | locals { 18 | config = var.json_config != "" ? var.json_config : base64encode(jsonencode([ 19 | merge({ 20 | "projectId" : "${var.project_id}", 21 | "regionId" : "${var.location}", 22 | "clusterId" : "${var.memorystore_cluster_name}", 23 | "engine" : "${var.memorystore_engine}", 24 | "scalerPubSubTopic" : "${var.target_pubsub_topic}", 25 | "units" : "${var.units}", 26 | "minSize" : var.min_size, 27 | "maxSize" : var.max_size, 28 | "scalingProfile" : "${var.scaling_profile}", 29 | "scalingMethod" : "${var.scaling_method}", 30 | "stateDatabase" : var.terraform_spanner_state ? { 31 | "name" : "spanner", 32 | "instanceId" : "${var.spanner_state_name}", 33 | "databaseId" : "${var.spanner_state_database}", 34 | } : { 35 | "name" : "firestore", 36 | "databaseId" : "${var.firestore_state_database}" 37 | } 38 | }, 39 | var.state_project_id != null ? { 40 | "stateProjectId" : "${var.state_project_id}" 41 | } : {}) 42 | ])) 43 | } 44 | 45 | resource "google_cloud_scheduler_job" "poller_job" { 46 | name = "poll-cluster-metrics" 47 | description = "Poll metrics for the configured Memorystore cluster" 48 | schedule = var.schedule 49 | time_zone = var.time_zone 50 | 51 | pubsub_target { 52 | topic_name = var.pubsub_topic 53 | data = local.config 54 | } 55 | 56 | retry_config { 57 | retry_count = 0 58 | max_backoff_duration = "3600s" 59 | max_retry_duration = "0s" 60 | max_doublings = 5 61 | min_backoff_duration = "5s" 62 | } 63 | 64 | /** 65 | * Uncomment this stanza if you would prefer to manage the Cloud Scheduler 66 | * configuration manually following its initial creation, i.e. using the 67 | * Google Cloud Web Console, the gcloud CLI, or any other non-Terraform 68 | * mechanism. Without this change, the Terraform configuration will remain 69 | * the source of truth, and any direct modifications to the Cloud Scheduler 70 | * configuration will be reset on the next Terraform run. Please see the 71 | * following link for more details: 72 | * 73 | * https://developer.hashicorp.com/terraform/language/meta-arguments/lifecycle#ignore_changes 74 | */ 75 | /* 76 | lifecycle { 77 | ignore_changes = [pubsub_target[0].data] 78 | } 79 | */ 80 | } 81 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-scheduler/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "scheduler_job_id" { 18 | value = google_cloud_scheduler_job.poller_job.id 19 | description = "ID of the Scheduler job" 20 | } 21 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-scheduler/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | } 20 | 21 | variable "location" { 22 | type = string 23 | } 24 | 25 | variable "schedule" { 26 | type = string 27 | default = "*/1 * * * *" 28 | } 29 | 30 | variable "time_zone" { 31 | type = string 32 | default = "Etc/UTC" 33 | } 34 | 35 | variable "pubsub_topic" { 36 | type = string 37 | } 38 | 39 | variable "memorystore_cluster_name" { 40 | type = string 41 | } 42 | 43 | variable "memorystore_engine" { 44 | type = string 45 | description = "The underlying engine to use" 46 | validation { 47 | condition = contains(["REDIS", "VALKEY"], var.memorystore_engine) 48 | error_message = "Valid values for var: memorystore_engine are (REDIS, VALKEY)." 49 | } 50 | } 51 | 52 | variable "target_pubsub_topic" { 53 | type = string 54 | } 55 | 56 | variable "units" { 57 | type = string 58 | default = "SHARDS" 59 | description = "The measure that Memorystore Cluster size is being specified in. Currently supported values are: \"SHARDS\". " 60 | } 61 | 62 | variable "min_size" { 63 | type = number 64 | default = 1 65 | description = "Minimum size that the Memorystore Cluster can be scaled in to." 66 | } 67 | 68 | variable "max_size" { 69 | type = number 70 | default = 10 71 | description = "Maximum size that the Memorystore Cluster can be scaled out to." 72 | } 73 | 74 | variable "scaling_profile" { 75 | type = string 76 | default = "CPU_AND_MEMORY" 77 | description = "Scaling profile to be used for the Memorystore Cluster: CPU_AND_MEMORY, CPU, MEMORY" 78 | } 79 | 80 | variable "scaling_method" { 81 | type = string 82 | default = "STEPWISE" 83 | description = "Algorithm that should be used to manage the scaling of the cluster: STEPWISE, LINEAR, DIRECT" 84 | } 85 | 86 | variable "terraform_spanner_state" { 87 | description = "If set to true, Terraform will create a Cloud Spanner DB to hold the Autoscaler state." 88 | type = bool 89 | default = false 90 | } 91 | 92 | variable "spanner_state_name" { 93 | type = string 94 | nullable = true 95 | default = null 96 | } 97 | 98 | variable "spanner_state_database" { 99 | type = string 100 | nullable = true 101 | default = null 102 | } 103 | 104 | variable "firestore_state_database" { 105 | type = string 106 | } 107 | 108 | variable "state_project_id" { 109 | type = string 110 | nullable = true 111 | default = null 112 | } 113 | 114 | variable "json_config" { 115 | type = string 116 | default = "" 117 | description = "Base 64 encoded json that is the autoscaler configuration for the Cloud Scheduler payload. Using this allows for setting autoscaler configuration for multiple Memorystore Clusters and parameters that are not directly exposed through variables." 118 | } 119 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-spanner/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_spanner_instance" "state_spanner_instance" { 18 | count = var.terraform_spanner_state ? 1 : 0 19 | 20 | name = var.spanner_state_name 21 | config = "regional-${var.region}" 22 | display_name = var.spanner_state_name 23 | project = var.project_id 24 | 25 | processing_units = var.spanner_state_processing_units 26 | } 27 | 28 | resource "google_spanner_database" "state_database" { 29 | count = var.terraform_spanner_state ? 1 : 0 30 | 31 | instance = var.spanner_state_name 32 | name = var.spanner_state_database 33 | ddl = [ 34 | < /etc/security/limits.d/test-vm-limits.conf 20 | root soft nofile 1048576 21 | root hard nofile 1048576 22 | * soft nofile 1048576 23 | * hard nofile 1048576 24 | EOF 25 | 26 | # Write instructional banner 27 | cat << 'EOF' > /etc/motd 28 | 29 | __ ___ __ ______ __ __ __ 30 | / |/ /__ ____ ___ ____ _______ _______/ /_____ ________ /_ __/__ _____/ /_/ /_ ___ ____ _____/ /_ 31 | / /|_/ / _ \/ __ `__ \/ __ \/ ___/ / / / ___/ __/ __ \/ ___/ _ \ / / / _ \/ ___/ __/ __ \/ _ \/ __ \/ ___/ __ \ 32 | / / / / __/ / / / / / /_/ / / / /_/ (__ ) /_/ /_/ / / / __/ / / / __(__ ) /_/ /_/ / __/ / / / /__/ / / / 33 | /_/ /_/\___/_/ /_/ /_/\____/_/ \__, /____/\__/\____/_/ \___/ /_/ \___/____/\__/_.___/\___/_/ /_/\___/_/ /_/ 34 | /____/ 35 | 36 | Generate CPU load: $ memorystore-cpu-load 37 | Bulk-write to increase memory utilisation (1GB): $ memorystore-write-1gb 38 | Bulk-write to increase memory utilisation (10GB): $ memorystore-write-10gb 39 | Flush all keys: $ memorystore-flush-all 40 | Connect to the cluster in interactive mode: $ redis-cli -c -h cluster.memorystore.private 41 | 42 | Functions are defined in /etc/profile.d/memorystore-functions.sh. 43 | 44 | Note that the underlying utilities may take a few minutes to become available on first boot. 45 | 46 | EOF 47 | 48 | # Install dependencies 49 | export DEBIAN_FRONTEND=noninteractive 50 | apt-get update && apt-get upgrade -y 51 | apt-get install redis-tools build-essential autoconf automake libpcre3-dev libevent-dev pkg-config zlib1g-dev git libssl-dev htop -y 52 | 53 | # Install utility for load generation 54 | export reddissim_version='v0.1.7' 55 | export golang_version='1.21.4' 56 | cd /root 57 | git clone https://github.com/maguec/RedisSim.git && cd RedisSim 58 | git checkout ${reddissim_version} 59 | wget https://go.dev/dl/go${golang_version}.linux-amd64.tar.gz 60 | tar -C /usr/local -xzf go${golang_version}.linux-amd64.tar.gz 61 | export HOME=/root 62 | export GOPATH=${HOME}/go 63 | export GOMODCACHE=${GOPATH}/pkg/mod 64 | export PATH=$PATH:/usr/local/go/bin 65 | make 66 | mv RedisSim /usr/local/bin/RedisSim 67 | 68 | # Define commands and make available 69 | memorystore_cpu_load_cmd="RedisSim cpukill --clients 100 --size 1000 --loop-forever --server cluster.memorystore.private --cluster" 70 | memorystore_write_1gb_cmd="RedisSim stringfill --prefix \$(date +'%s') --clients 100 --size 1000 --string-count 1000000 --server cluster.memorystore.private --cluster" 71 | memorystore_write_10gb_cmd="RedisSim stringfill --prefix \$(date +'%s') --clients 100 --size 1000 --string-count 10000000 --server cluster.memorystore.private --cluster" 72 | memorystore_flush_all_cmd="redis-cli --cluster call --cluster-only-masters cluster.memorystore.private:6379 FLUSHALL" 73 | 74 | cat << EOF > /etc/profile.d/memorystore-functions.sh 75 | function memorystore-cpu-load () { 76 | ${memorystore_cpu_load_cmd} 77 | } 78 | function memorystore-write-10gb () { 79 | ${memorystore_write_10gb_cmd} 80 | } 81 | function memorystore-write-1gb () { 82 | ${memorystore_write_1gb_cmd} 83 | } 84 | function memorystore-flush-all () { 85 | ${memorystore_flush_all_cmd} 86 | } 87 | export -f memorystore-cpu-load 88 | export -f memorystore-write-10gb 89 | export -f memorystore-write-1gb 90 | export -f memorystore-flush-all 91 | EOF 92 | -------------------------------------------------------------------------------- /terraform/modules/autoscaler-test-vm/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | description = "Project ID where the cluster will run" 20 | } 21 | 22 | variable "name" { 23 | type = string 24 | description = "A unique name for the resource" 25 | } 26 | 27 | variable "region" { 28 | type = string 29 | description = "The name of the region to run the cluster" 30 | } 31 | 32 | variable "network" { 33 | type = string 34 | description = "The subnetwork to host the cluster in" 35 | } 36 | 37 | variable "subnetwork" { 38 | type = string 39 | description = "The subnetwork to host the cluster in" 40 | } 41 | 42 | variable "machine_type" { 43 | type = string 44 | description = "The machine type to use for the test VM" 45 | default = "c3-standard-4" 46 | } 47 | 48 | variable "machine_image" { 49 | type = string 50 | default = "debian-cloud/debian-12" 51 | } 52 | --------------------------------------------------------------------------------