├── .dockerignore ├── .github ├── CODEOWNERS └── workflows │ ├── docker-publish.yml │ └── helm-chart.yaml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── activities └── activity.go ├── charts └── benchmark-workers │ ├── Chart.yaml │ ├── README.md │ ├── templates │ ├── _helpers.tpl │ ├── deployment-soak-test.yaml │ ├── deployment-workers.yaml │ ├── secret-tls.yaml │ ├── service.yaml │ └── servicemonitor.yaml │ └── values.yaml ├── cmd ├── runner │ ├── log.go │ ├── main.go │ └── metrics.go └── worker │ ├── main.go │ └── metrics.go ├── deployment.yaml ├── go.mod ├── go.sum └── workflows ├── workflow.go └── workflow_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .github 2 | .git 3 | runner 4 | worker 5 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # This repository is maintained by the Temporal DevEx team 2 | @temporalio/selfhosted 3 | -------------------------------------------------------------------------------- /.github/workflows/docker-publish.yml: -------------------------------------------------------------------------------- 1 | name: Docker 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | # Publish semver tags as releases. 7 | tags: [ 'v*.*.*' ] 8 | pull_request: 9 | branches: [ "main" ] 10 | 11 | env: 12 | REGISTRY: ghcr.io 13 | IMAGE_NAME: ${{ github.repository }} 14 | 15 | jobs: 16 | build-and-push-image: 17 | runs-on: ubuntu-latest 18 | permissions: 19 | contents: read 20 | packages: write 21 | 22 | steps: 23 | - name: Checkout repository 24 | uses: actions/checkout@v3 25 | 26 | - name: Setup Docker buildx 27 | uses: docker/setup-buildx-action@v2.4.1 28 | 29 | - name: Log into registry ${{ env.REGISTRY }} 30 | if: github.event_name != 'pull_request' 31 | uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 32 | with: 33 | registry: ${{ env.REGISTRY }} 34 | username: ${{ github.actor }} 35 | password: ${{ secrets.GITHUB_TOKEN }} 36 | 37 | - name: Extract metadata (tags, labels) for Docker 38 | id: meta 39 | uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 40 | with: 41 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 42 | 43 | - name: Build and push Docker image 44 | uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc 45 | with: 46 | context: . 47 | platforms: linux/amd64,linux/arm64 48 | push: ${{ github.event_name != 'pull_request' }} 49 | tags: ${{ steps.meta.outputs.tags }} 50 | labels: ${{ steps.meta.outputs.labels }} 51 | -------------------------------------------------------------------------------- /.github/workflows/helm-chart.yaml: -------------------------------------------------------------------------------- 1 | name: Package and Publish Helm Chart 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | version_bump: 7 | description: 'Type of version bump to perform' 8 | required: true 9 | default: 'patch' 10 | type: choice 11 | options: 12 | - patch 13 | - minor 14 | - major 15 | 16 | permissions: 17 | packages: write 18 | contents: write 19 | 20 | jobs: 21 | release: 22 | runs-on: ubuntu-latest 23 | steps: 24 | - name: Checkout 25 | uses: actions/checkout@v3 26 | with: 27 | fetch-depth: 0 28 | 29 | - name: Configure Git 30 | run: | 31 | git config user.name "$GITHUB_ACTOR" 32 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 33 | 34 | - name: Install Helm 35 | uses: azure/setup-helm@v3 36 | with: 37 | version: v3.12.0 38 | 39 | - name: Bump Chart Version 40 | id: bump_version 41 | run: | 42 | # Get current version from Chart.yaml 43 | CURRENT_VERSION=$(grep 'version:' charts/benchmark-workers/Chart.yaml | awk '{print $2}') 44 | echo "Current version: $CURRENT_VERSION" 45 | 46 | # Split version into parts 47 | IFS='.' read -r -a VERSION_PARTS <<< "$CURRENT_VERSION" 48 | MAJOR=${VERSION_PARTS[0]} 49 | MINOR=${VERSION_PARTS[1]} 50 | PATCH=${VERSION_PARTS[2]} 51 | 52 | # Increment version based on input 53 | if [[ "${{ inputs.version_bump }}" == "major" ]]; then 54 | MAJOR=$((MAJOR + 1)) 55 | MINOR=0 56 | PATCH=0 57 | elif [[ "${{ inputs.version_bump }}" == "minor" ]]; then 58 | MINOR=$((MINOR + 1)) 59 | PATCH=0 60 | else 61 | PATCH=$((PATCH + 1)) 62 | fi 63 | 64 | NEW_VERSION="$MAJOR.$MINOR.$PATCH" 65 | echo "New version: $NEW_VERSION" 66 | 67 | # Update Chart.yaml with new version 68 | sed -i "s/version: $CURRENT_VERSION/version: $NEW_VERSION/g" charts/benchmark-workers/Chart.yaml 69 | 70 | # Set output variable for use in later steps 71 | echo "version=$NEW_VERSION" >> $GITHUB_OUTPUT 72 | 73 | # Commit the change 74 | git add charts/benchmark-workers/Chart.yaml 75 | git commit -m "Bump chart version to $NEW_VERSION [skip ci]" 76 | git push 77 | 78 | - name: Login to GHCR 79 | uses: docker/login-action@v2 80 | with: 81 | registry: ghcr.io 82 | username: ${{ github.actor }} 83 | password: ${{ secrets.GITHUB_TOKEN }} 84 | 85 | - name: Package and Push Helm chart 86 | run: | 87 | # Use version from previous step 88 | VERSION=${{ steps.bump_version.outputs.version }} 89 | echo "Chart version: $VERSION" 90 | 91 | # Package the chart 92 | helm package charts/benchmark-workers 93 | 94 | # Push to GHCR 95 | helm push benchmark-workers-${VERSION}.tgz oci://ghcr.io/temporalio/charts 96 | 97 | echo "✅ Chart pushed successfully to oci://ghcr.io/temporalio/charts/benchmark-workers:${VERSION}" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /worker 2 | /runner 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.21 AS builder 2 | 3 | WORKDIR /usr/src/worker 4 | 5 | COPY go.mod go.sum ./ 6 | RUN go mod download && go mod verify 7 | 8 | COPY . . 9 | RUN CGO_ENABLED=0 go build -v -o /usr/local/bin/worker ./cmd/worker 10 | RUN CGO_ENABLED=0 go build -v -o /usr/local/bin/runner ./cmd/runner 11 | 12 | FROM scratch 13 | 14 | COPY --from=builder /usr/local/bin/worker /usr/local/bin/worker 15 | COPY --from=builder /usr/local/bin/runner /usr/local/bin/runner 16 | 17 | CMD ["/usr/local/bin/worker"] 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 temporal.io 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # benchmark-workers 2 | 3 | Pre-written workflows and activities useful for benchmarking Temporal. 4 | 5 | This worker can be used alongside Maru or other benchmarking tools to mimic different workloads. 6 | 7 | Also included is a simple workflow runner which will keep a configurable number of workflow executions running concurrently to provide load for testing, starting a new execution each time one completes. 8 | 9 | ## Usage 10 | 11 | ### Worker 12 | 13 | The worker is available as docker image for use in Docker or Kubernetes setups. 14 | 15 | You can pull the latest image from: `ghcr.io/temporalio/benchmark-workers:main`. 16 | 17 | In future we will provide releases with appropriate image tags to make benchmarks more easily repeatable. 18 | 19 | The worker can be configured via environment variables. Currently only a small number of options are available, please let us know if there is a particular option you would like to be exposed by filing an issue. 20 | 21 | The table below lists the environment variables available and the relevant Temporal Go SDK options they relate to (the worker is currently written using the Temporal Go SDK). 22 | 23 | | Environment Variable | Relevant Client or Worker option | Description | 24 | | --- | --- | --- | 25 | | TEMPORAL_GRPC_ENDPOINT | [ClientOptions.HostPort](https://pkg.go.dev/go.temporal.io/sdk@v1.15.0/internal#ClientOptions) | The Temporal Frontend GRPC endpoint (supports comma-separated values for multiple namespaces) | 26 | | TEMPORAL_TLS_KEY | [ClientOptions.ConnectionOptions.TLS](https://pkg.go.dev/go.temporal.io/sdk@v1.15.0/internal#ConnectionOptions) | Path to TLS Key file (supports comma-separated values for multiple namespaces) | 27 | | TEMPORAL_TLS_CERT | [ClientOptions.ConnectionOptions.TLS](https://pkg.go.dev/go.temporal.io/sdk@v1.15.0/internal#ConnectionOptions) | Path to TLS Cert file (supports comma-separated values for multiple namespaces) | 28 | | TEMPORAL_TLS_CA | [ClientOptions.ConnectionOptions.TLS](https://pkg.go.dev/go.temporal.io/sdk@v1.15.0/internal#ConnectionOptions) | Path to TLS CA Cert file (supports comma-separated values for multiple namespaces) | 29 | | TEMPORAL_NAMESPACE | [ClientOptions.Namespace](https://pkg.go.dev/go.temporal.io/sdk@v1.15.0/internal#ClientOptions) | The Temporal Namespace (supports comma-separated values for multiple namespaces) | 30 | | TEMPORAL_TASK_QUEUE | [TaskQueue](https://pkg.go.dev/go.temporal.io/sdk@v1.15.0/worker#New) | The Temporal Task Queue | 31 | | TEMPORAL_WORKFLOW_TASK_POLLERS | [WorkerOptions.MaxConcurrentWorkflowTaskPollers](https://pkg.go.dev/go.temporal.io/sdk@v1.15.0/internal#WorkerOptions) | Number of workflow task pollers | 32 | | TEMPORAL_ACTIVITY_TASK_POLLERS | [WorkerOptions.MaxConcurrentActivityTaskPollers](https://pkg.go.dev/go.temporal.io/sdk@v1.15.0/internal#WorkerOptions) | Number of activity task pollers | 33 | | PROMETHEUS_ENDPOINT | n/a | The address to serve prometheus metrics on | 34 | 35 | #### Multi-Namespace Support 36 | 37 | The worker supports working with multiple namespaces simultaneously. This allows you to spread load across multiple namespaces with a single worker deployment, providing a more realistic load pattern. 38 | 39 | **Configuration Options:** 40 | 41 | 1. **Same configuration for all namespaces:** 42 | ```bash 43 | export TEMPORAL_NAMESPACE="ns1,ns2,ns3" 44 | export TEMPORAL_GRPC_ENDPOINT="temporal.example.com:7233" 45 | export TEMPORAL_TLS_CERT="/path/to/cert.pem" 46 | export TEMPORAL_TLS_KEY="/path/to/key.pem" 47 | ``` 48 | 49 | 2. **Different configurations per namespace:** 50 | ```bash 51 | export TEMPORAL_NAMESPACE="ns1,ns2,ns3" 52 | export TEMPORAL_GRPC_ENDPOINT="temporal1.example.com:7233,temporal2.example.com:7233,temporal3.example.com:7233" 53 | export TEMPORAL_TLS_CERT="/certs/ns1.pem,/certs/ns2.pem,/certs/ns3.pem" 54 | export TEMPORAL_TLS_KEY="/keys/ns1.key,/keys/ns2.key,/keys/ns3.key" 55 | ``` 56 | 57 | 3. **Mixed configuration (some shared, some different):** 58 | ```bash 59 | export TEMPORAL_NAMESPACE="ns1,ns2,ns3" 60 | export TEMPORAL_GRPC_ENDPOINT="temporal.example.com:7233" # Same endpoint for all 61 | export TEMPORAL_TLS_CERT="/certs/ns1.pem,/certs/ns2.pem" # ns3 will reuse ns2's cert 62 | export TEMPORAL_TLS_KEY="/keys/shared.key" # Same key for all 63 | ``` 64 | 65 | **How it works:** 66 | - If you provide exactly as many values as namespaces, each namespace uses its corresponding value 67 | - If you provide only one value but multiple namespaces, that single value is reused for all namespaces 68 | - If you provide fewer values than namespaces, the last value is repeated for the remaining namespaces 69 | 70 | The worker will create a separate worker instance for each namespace, all running concurrently within the same process. 71 | 72 | #### Kubernetes Deployment 73 | 74 | There are several ways to deploy the worker in Kubernetes: 75 | 76 | 1. **Using kubectl run**: 77 | 78 | ``` 79 | kubectl run benchmark-worker --image ghcr.io/temporalio/benchmark-workers:main \ 80 | --image-pull-policy Always \ 81 | --env "TEMPORAL_GRPC_ENDPOINT=temporal-frontend.temporal:7233" \ 82 | --env "TEMPORAL_NAMESPACE=default" \ 83 | --env "TEMPORAL_TASK_QUEUE=benchmark" \ 84 | --env "TEMPORAL_WORKFLOW_TASK_POLLERS=16" \ 85 | --env "TEMPORAL_ACTIVITY_TASK_POLLERS=8" 86 | ``` 87 | 88 | 2. **Multi-namespace deployment example**: 89 | 90 | ``` 91 | kubectl run benchmark-worker --image ghcr.io/temporalio/benchmark-workers:main \ 92 | --image-pull-policy Always \ 93 | --env "TEMPORAL_GRPC_ENDPOINT=temporal-frontend.temporal:7233" \ 94 | --env "TEMPORAL_NAMESPACE=namespace1,namespace2,namespace3" \ 95 | --env "TEMPORAL_TASK_QUEUE=benchmark" \ 96 | --env "TEMPORAL_WORKFLOW_TASK_POLLERS=16" \ 97 | --env "TEMPORAL_ACTIVITY_TASK_POLLERS=8" 98 | ``` 99 | 100 | 3. **Using the example deployment YAML**: 101 | 102 | We provide an [example deployment spec](./deployment.yaml) for you to customize to your requirements. Once you have edited the environment variables in the deployment.yaml you can create the deployment with `kubectl apply -f ./deployment.yaml`. 103 | 104 | 4. **Using the Helm chart (Recommended)**: 105 | 106 | We provide a Helm chart that can be installed from the GitHub Container Registry: 107 | 108 | ```bash 109 | # Install the chart 110 | helm install benchmark-workers oci://ghcr.io/temporalio/charts/benchmark-workers 111 | ``` 112 | 113 | For more details and configuration options, see the [Helm chart documentation](./charts/benchmark-workers/README.md). 114 | 115 | #### Prometheus Metrics 116 | 117 | The worker can expose Prometheus metrics to help monitor the performance of your Temporal workers and cluster. To enable metrics: 118 | 119 | 1. **Using kubectl or deployment YAML**: 120 | ``` 121 | --env "PROMETHEUS_ENDPOINT=:9090" 122 | ``` 123 | 124 | 2. **Using the Helm chart**: 125 | ```bash 126 | helm install benchmark-workers oci://ghcr.io/temporalio/charts/benchmark-workers \ 127 | --set metrics.enabled=true 128 | ``` 129 | 130 | When using the Helm chart, it will automatically create a headless service for service discovery and can optionally create a ServiceMonitor resource for Prometheus Operator: 131 | 132 | ```bash 133 | helm install benchmark-workers oci://ghcr.io/temporalio/charts/benchmark-workers \ 134 | --set metrics.enabled=true \ 135 | --set metrics.serviceMonitor.enabled=true 136 | ``` 137 | 138 | You can then use the benchmark workflows with your benchmark tool. To test with `tctl` you could run: 139 | 140 | ``` 141 | tctl workflow start --taskqueue benchmark --workflow_type ExecuteActivity --execution_timeout 60 -i '{"Count":1,"Activity":"Sleep","Input":{"SleepTimeInSeconds":3}}' 142 | ``` 143 | 144 | This will run the ExecuteActivity workflow, described below. 145 | 146 | ### Runner 147 | 148 | The runner is available as docker image for use in Docker or Kubernetes setups. 149 | 150 | You can pull the latest image from: `ghcr.io/temporalio/benchmark-workers:main`. 151 | 152 | The runner can be configured via environment variables and command line arguments. Currently only a small number of options are available, please let us know if there is a particular option you would like to be exposed by filing an issue. 153 | 154 | The table below lists the environment variables available and the relevant Temporal Go SDK options they relate to (the runner is currently written using the Temporal Go SDK). 155 | 156 | | Environment Variable | Relevant Client or Worker option | Description | 157 | | --- | --- | --- | 158 | | TEMPORAL_GRPC_ENDPOINT | [ClientOptions.HostPort](https://pkg.go.dev/go.temporal.io/sdk@v1.15.0/internal#ClientOptions) | The Temporal Frontend GRPC endpoint (supports comma-separated values for multiple namespaces) | 159 | | TEMPORAL_TLS_KEY | [ClientOptions.ConnectionOptions.TLS.Certificates](https://pkg.go.dev/go.temporal.io/sdk@v1.15.0/internal#ConnectionOptions) | Path to TLS Key file (supports comma-separated values for multiple namespaces) | 160 | | TEMPORAL_TLS_CERT | [ClientOptions.ConnectionOptions.TLS.Certificates](https://pkg.go.dev/go.temporal.io/sdk@v1.15.0/internal#ConnectionOptions) | Path to TLS Cert file (supports comma-separated values for multiple namespaces) | 161 | | TEMPORAL_TLS_CA | [ClientOptions.ConnectionOptions.TLS](https://pkg.go.dev/go.temporal.io/sdk@v1.15.0/internal#ConnectionOptions) | Path to TLS CA Cert file (supports comma-separated values for multiple namespaces) | 162 | | PROMETHEUS_ENDPOINT | n/a | The address to serve prometheus metrics on | 163 | 164 | The runner is also configured via command line options: 165 | 166 | ``` 167 | Usage: runner [flags] [workflow input] ... 168 | -c int 169 | concurrent workflows (default 10) 170 | -n string 171 | namespace (comma-separated list supported) (default "default") 172 | -s string 173 | signal type 174 | -t string 175 | workflow type 176 | -tq string 177 | task queue (default "benchmark") 178 | -w wait for workflows to complete (default true) 179 | ``` 180 | 181 | #### Multi-Namespace Support in Runner 182 | 183 | The runner supports distributing workflow executions across multiple namespaces. This provides a more realistic load pattern by spreading the load across different namespaces. 184 | 185 | **Configuration Examples:** 186 | 187 | 1. **Command line flag for multiple namespaces:** 188 | ```bash 189 | runner -n "namespace1,namespace2,namespace3" -t ExecuteActivity '{"Count":1,"Activity":"Sleep","Input":{"SleepTimeInSeconds":3}}' 190 | ``` 191 | 192 | 2. **Environment variable for multiple namespaces:** 193 | ```bash 194 | export TEMPORAL_NAMESPACE="namespace1,namespace2,namespace3" 195 | runner -t ExecuteActivity '{"Count":1,"Activity":"Sleep","Input":{"SleepTimeInSeconds":3}}' 196 | ``` 197 | 198 | 3. **Different GRPC endpoints per namespace:** 199 | ```bash 200 | export TEMPORAL_NAMESPACE="ns1,ns2,ns3" 201 | export TEMPORAL_GRPC_ENDPOINT="temporal1.example.com:7233,temporal2.example.com:7233,temporal3.example.com:7233" 202 | runner -t ExecuteActivity '{"Count":1,"Activity":"Sleep","Input":{"SleepTimeInSeconds":3}}' 203 | ``` 204 | 205 | **How it works:** 206 | - The runner creates a separate client for each namespace 207 | - Workflow executions are distributed across namespaces using round-robin rotation 208 | - Each client can have different connection settings (endpoints, TLS certificates, etc.) 209 | 210 | To use the runner in a Kubernetes cluster you could use: 211 | 212 | ``` 213 | kubectl run benchmark-runner --image ghcr.io/temporalio/benchmark-workers:main \ 214 | --image-pull-policy Always \ 215 | --env "TEMPORAL_GRPC_ENDPOINT=temporal-frontend.temporal:7233" \ 216 | --env "TEMPORAL_NAMESPACE=namespace1,namespace2,namespace3" \ 217 | --command -- runner -t ExecuteActivity '{ "Count": 3, "Activity": "Echo", "Input": { "Message": "test" } }' 218 | ``` 219 | 220 | ## Workflows 221 | 222 | The worker provides the following workflows for you to use during benchmarking: 223 | 224 | ### ExecuteActivity 225 | 226 | `ExecuteActivity({ Count: int, Activity: string, Input: interface{} })` 227 | 228 | This workflow takes a count, an activity name and an activity input. The activity `Activity` will be run `Count` times with the given `input`. If the activity returns an error the workflow will fail with that error. 229 | 230 | ### ReceiveSignal 231 | 232 | `ReceiveSignal()` 233 | 234 | This workflow waits to receive a signal. It can be used with the runner's signal functionality to test signal-based workflows. 235 | 236 | ### DSLWorkflow 237 | 238 | `DSLWorkflow([]DSLStep)` 239 | 240 | This workflow takes an array of steps, each of which can execute an activity or a child workflow (which is another invocation of DSLWorkflow). This allows you to compose complex benchmarking scenarios, including nested and repeated activities and child workflows. 241 | 242 | Each step can have the following fields: 243 | - `a`: (string) Activity name to execute 244 | - `i`: (object, optional) Input to pass to the activity 245 | - `c`: (array of steps, optional) Child steps to execute as a child workflow 246 | - `r`: (int, optional) Number of times to repeat this step (default 1) 247 | 248 | #### Example 249 | 250 | This example runs the `Echo` activity 3 times, then starts a child workflow which also runs the `Echo` activity 3 times: 251 | 252 | ``` 253 | [ 254 | {"a": "Echo", "i": {"Message": "test"}, "r": 3}, 255 | {"c": [ 256 | {"a": "Echo", "i": {"Message": "test"}, "r": 3} 257 | ]} 258 | ] 259 | ``` 260 | 261 | You can start this workflow using `tctl` or any Temporal client, for example: 262 | 263 | ``` 264 | tctl workflow start --taskqueue benchmark --workflow_type DSLWorkflow --execution_timeout 60 -i '[{"a": "Echo", "i": {"Message": "test"}, "r": 3}, {"c": [{"a": "Echo", "i": {"Message": "test"}, "r": 3}]}]' 265 | ``` 266 | 267 | ## Activities 268 | 269 | The worker provides the following activities for you to use during benchmarking: 270 | 271 | ### Sleep 272 | 273 | `Sleep({ SleepTimeInSeconds: int })` 274 | 275 | This activity sleeps for the given number of seconds. It never returns an error. This can be used to simulate activities which take a while to complete. 276 | 277 | ### Echo 278 | 279 | `Echo({ Message: string }) result` 280 | 281 | This activity simply returns the message as it's result. This can be used for stress testing polling with activities that return instantly. -------------------------------------------------------------------------------- /activities/activity.go: -------------------------------------------------------------------------------- 1 | package activities 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "go.temporal.io/sdk/activity" 8 | ) 9 | 10 | type SleepActivityInput struct { 11 | SleepTimeInSeconds int 12 | } 13 | 14 | func SleepActivity(ctx context.Context, input SleepActivityInput) error { 15 | sleepTimer := time.After(time.Duration(input.SleepTimeInSeconds) * time.Second) 16 | heartbeatTimeout := activity.GetInfo(ctx).HeartbeatTimeout 17 | if heartbeatTimeout == 0 { 18 | // If no heartbeat timeout is set, assume we don't want heartbeating. 19 | // Set the heartbeat time longer than our sleep time so that we never send a heartbeat. 20 | heartbeatTimeout = time.Duration(input.SleepTimeInSeconds) * 2 21 | } 22 | heartbeatTick := time.Duration(0.8 * float64(heartbeatTimeout)) 23 | t := time.NewTicker(heartbeatTick) 24 | 25 | for { 26 | select { 27 | case <-sleepTimer: 28 | return nil 29 | case <-ctx.Done(): 30 | return ctx.Err() 31 | case <-t.C: 32 | activity.RecordHeartbeat(ctx) 33 | } 34 | } 35 | } 36 | 37 | type EchoActivityInput struct { 38 | Message string 39 | } 40 | 41 | func EchoActivity(ctx context.Context, input EchoActivityInput) (string, error) { 42 | return input.Message, nil 43 | } 44 | -------------------------------------------------------------------------------- /charts/benchmark-workers/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: benchmark-workers 3 | description: A Helm chart for Temporal benchmark workers 4 | type: application 5 | version: 0.4.2 6 | appVersion: latest 7 | maintainers: 8 | - name: Temporal 9 | url: https://temporal.io -------------------------------------------------------------------------------- /charts/benchmark-workers/README.md: -------------------------------------------------------------------------------- 1 | # Temporal Benchmark Workers 2 | 3 | This Helm chart deploys Temporal benchmark workers for load testing and performance evaluation of a Temporal cluster. 4 | 5 | ## TL;DR 6 | 7 | ```bash 8 | helm install benchmark-workers oci://ghcr.io/temporalio/charts/benchmark-workers 9 | ``` 10 | 11 | ## Introduction 12 | 13 | This chart deploys two components: 14 | 1. **Benchmark Workers**: Temporal workers that execute activities and workflows for benchmarking 15 | 2. **Soak Test** (optional): A runner component that continuously creates workflows to generate load 16 | 17 | ## Prerequisites 18 | 19 | - Kubernetes 1.16+ 20 | - Helm 3.8.0+ 21 | - A running Temporal cluster accessible from the Kubernetes cluster 22 | - (Optional) Prometheus Operator for ServiceMonitor support 23 | 24 | ## Installing the Chart 25 | 26 | ### From OCI Registry (Recommended) 27 | 28 | To install the chart from the GitHub Container Registry: 29 | 30 | ```bash 31 | # Authenticate with GHCR (if needed) 32 | # For public repositories, this step is optional 33 | # For private repositories: 34 | # echo $GITHUB_TOKEN | helm registry login ghcr.io -u $GITHUB_USERNAME --password-stdin 35 | 36 | # Install the chart 37 | helm install benchmark-workers oci://ghcr.io/temporalio/charts/benchmark-workers 38 | ``` 39 | 40 | ### From Local Chart 41 | 42 | To install the chart from a local clone of this repository: 43 | 44 | ```bash 45 | git clone https://github.com/temporalio/benchmark-workers.git 46 | cd benchmark-workers 47 | helm install benchmark-workers ./charts/benchmark-workers 48 | ``` 49 | 50 | ## Multi-Namespace Support 51 | 52 | The chart supports deploying workers and runners that work with multiple Temporal namespaces simultaneously. This provides a more realistic load pattern by distributing work across multiple namespaces with a single deployment. 53 | 54 | ### Configuration Examples 55 | 56 | #### Single Namespace (Traditional) 57 | ```yaml 58 | temporal: 59 | grpcEndpoint: "temporal-frontend.temporal:7233" 60 | namespace: "default" 61 | ``` 62 | 63 | #### Multiple Namespaces with Same Configuration 64 | ```yaml 65 | temporal: 66 | grpcEndpoint: "temporal-frontend.temporal:7233" # Same endpoint for all 67 | namespace: ["namespace1", "namespace2", "namespace3"] 68 | ``` 69 | 70 | #### Multiple Namespaces with Different Endpoints 71 | ```yaml 72 | temporal: 73 | grpcEndpoint: ["temporal1.temporal:7233", "temporal2.temporal:7233", "temporal3.temporal:7233"] 74 | namespace: ["namespace1", "namespace2", "namespace3"] 75 | ``` 76 | 77 | #### Multiple Namespaces with Mixed TLS Configuration 78 | ```yaml 79 | temporal: 80 | grpcEndpoint: ["temporal1.temporal:7233", "temporal2.temporal:7233"] 81 | namespace: ["namespace1", "namespace2", "namespace3"] 82 | tls: 83 | enabled: true 84 | # Use arrays for different TLS configs per namespace 85 | keys: ["key1-content", "key2-content"] # namespace3 will reuse key2-content 86 | certs: ["cert1-content", "cert2-content"] # namespace3 will reuse cert2-content 87 | ca: "shared-ca-content" # Same CA for all namespaces 88 | ``` 89 | 90 | ## Configuration 91 | 92 | The following table lists the configurable parameters for the benchmark-workers chart and their default values. 93 | 94 | | Parameter | Description | Default | 95 | |-----------|-------------|---------| 96 | | `image.repository` | Image repository | `ghcr.io/temporalio/benchmark-workers` | 97 | | `image.tag` | Image tag | `latest` | 98 | | `image.pullPolicy` | Image pull policy | `Always` | 99 | | `temporal.grpcEndpoint` | Temporal frontend endpoint(s) (string or array) | `temporal-frontend.temporal:7233` | 100 | | `temporal.namespace` | Temporal namespace(s) (string or array) | `default` | 101 | | `temporal.taskQueue` | Task queue name | `benchmark` | 102 | | `temporal.workflowTaskPollers` | Number of workflow task pollers | `16` | 103 | | `temporal.activityTaskPollers` | Number of activity task pollers | `8` | 104 | | `temporal.tls.enabled` | Enable TLS | `false` | 105 | | `temporal.tls.key` | TLS key content (base64 encoded) | `""` | 106 | | `temporal.tls.cert` | TLS certificate content (base64 encoded) | `""` | 107 | | `temporal.tls.ca` | TLS CA certificate content (base64 encoded) | `""` | 108 | | `temporal.tls.keys` | Array of TLS key contents for multi-namespace | `[]` | 109 | | `temporal.tls.certs` | Array of TLS certificate contents for multi-namespace | `[]` | 110 | | `temporal.tls.cas` | Array of TLS CA certificate contents for multi-namespace | `[]` | 111 | | `temporal.tls.existingSecret` | Use existing Kubernetes secret for TLS | `""` | 112 | | `temporal.tls.existingSecrets` | Array of existing Kubernetes secrets for multi-namespace | `[]` | 113 | | `temporal.tls.disableHostVerification` | Disable TLS host verification | `false` | 114 | | `metrics.enabled` | Enable Prometheus metrics | `true` | 115 | | `metrics.port` | Port to expose metrics on | `9090` | 116 | | `metrics.prometheusEndpoint` | Prometheus metrics endpoint | `:9090` | 117 | | `metrics.service.annotations` | Annotations for the metrics service | `{}` | 118 | | `metrics.serviceMonitor.enabled` | Enable ServiceMonitor for Prometheus Operator | `true` | 119 | | `metrics.serviceMonitor.additionalLabels` | Additional labels for the ServiceMonitor | `{}` | 120 | | `metrics.serviceMonitor.interval` | Scrape interval | `15s` | 121 | | `metrics.serviceMonitor.scrapeTimeout` | Scrape timeout | `10s` | 122 | | `workers.replicaCount` | Number of worker pods | `1` | 123 | | `workers.resources` | Resource requests and limits for worker pods | `{}` | 124 | | `additionalEnv` | Additional environment variables for worker pods | `[]` | 125 | | `soakTest.enabled` | Enable soak test deployment | `true` | 126 | | `soakTest.replicaCount` | Number of soak test pods | `1` | 127 | | `soakTest.concurrentWorkflows` | Number of concurrent workflows | `10` | 128 | | `soakTest.workflowType` | Workflow type to execute | `ExecuteActivity` | 129 | | `soakTest.workflowArgs` | Arguments for the workflow | `{ "Count": 3, "Activity": "Echo", "Input": { "Message": "test" } }` | 130 | | `soakTest.resources` | Resource requests and limits for soak test pods | `{}` | 131 | | `nodeSelector` | Node labels for pod assignment | `{}` | 132 | | `tolerations` | Tolerations for pod assignment | `[]` | 133 | | `affinity` | Affinity for pod assignment | `{}` | 134 | 135 | ## TLS Configuration 136 | 137 | ### Single Namespace TLS 138 | 139 | To use TLS with a single namespace, set `temporal.tls.enabled` to `true` and either: 140 | 141 | 1. Provide the TLS materials in the values file (not recommended for production): 142 | 143 | ```yaml 144 | temporal: 145 | tls: 146 | enabled: true 147 | key: 148 | cert: 149 | ca: 150 | ``` 151 | 152 | 2. Create a secret manually and reference it: 153 | 154 | ```bash 155 | kubectl create secret generic temporal-tls \ 156 | --from-file=key=/path/to/key.pem \ 157 | --from-file=cert=/path/to/cert.pem \ 158 | --from-file=ca=/path/to/ca.pem 159 | ``` 160 | 161 | Then reference it in your values: 162 | 163 | ```yaml 164 | temporal: 165 | tls: 166 | enabled: true 167 | existingSecret: "temporal-tls" 168 | ``` 169 | 170 | ### Multi-Namespace TLS 171 | 172 | For multiple namespaces, you can either: 173 | 174 | 1. **Use the same TLS configuration for all namespaces:** 175 | 176 | ```yaml 177 | temporal: 178 | namespace: ["ns1", "ns2", "ns3"] 179 | tls: 180 | enabled: true 181 | key: # Same key for all namespaces 182 | cert: # Same cert for all namespaces 183 | ca: # Same CA for all namespaces 184 | ``` 185 | 186 | 2. **Use different TLS configurations per namespace:** 187 | 188 | ```yaml 189 | temporal: 190 | namespace: ["ns1", "ns2", "ns3"] 191 | tls: 192 | enabled: true 193 | keys: ["", "", ""] 194 | certs: ["", "", ""] 195 | cas: ["", "", ""] 196 | ``` 197 | 198 | 3. **Mix single and array values (arrays take precedence):** 199 | 200 | ```yaml 201 | temporal: 202 | namespace: ["ns1", "ns2", "ns3"] 203 | tls: 204 | enabled: true 205 | keys: ["", ""] # ns3 will reuse key2 206 | certs: ["", ""] # ns3 will reuse cert2 207 | ca: "" # Same CA for all namespaces 208 | ``` 209 | 210 | ## Prometheus Metrics Integration 211 | 212 | ### Basic Metrics 213 | 214 | To enable basic metrics exposure: 215 | 216 | ```yaml 217 | metrics: 218 | enabled: true 219 | port: 9090 220 | prometheusEndpoint: ":9090" 221 | ``` 222 | 223 | This will: 224 | 1. Configure the workers to expose Prometheus metrics 225 | 2. Create a headless service to make the metrics endpoints discoverable 226 | 227 | ### Prometheus Operator Integration 228 | 229 | If you have the Prometheus Operator installed in your cluster, you can enable automatic service discovery: 230 | 231 | ```yaml 232 | metrics: 233 | enabled: true 234 | serviceMonitor: 235 | enabled: true 236 | # Optional: Add custom labels for the Prometheus instance you want to use 237 | additionalLabels: 238 | release: monitoring 239 | ``` 240 | 241 | ## Additional Environment Variables 242 | 243 | The chart supports adding custom environment variables to the worker pods using the `additionalEnv` parameter. This is useful for configuring application-specific settings or integrating with external services. 244 | 245 | ### Simple Environment Variables 246 | 247 | ```yaml 248 | additionalEnv: 249 | - name: CUSTOM_SETTING 250 | value: "my-value" 251 | - name: LOG_LEVEL 252 | value: "DEBUG" 253 | ``` 254 | 255 | ### Environment Variables from Secrets 256 | 257 | ```yaml 258 | additionalEnv: 259 | - name: DATABASE_PASSWORD 260 | valueFrom: 261 | secretKeyRef: 262 | name: my-secret 263 | key: password 264 | - name: API_KEY 265 | valueFrom: 266 | secretKeyRef: 267 | name: api-credentials 268 | key: api-key 269 | ``` 270 | 271 | ### Environment Variables from ConfigMaps 272 | 273 | ```yaml 274 | additionalEnv: 275 | - name: APP_CONFIG 276 | valueFrom: 277 | configMapKeyRef: 278 | name: app-config 279 | key: config.json 280 | ``` 281 | 282 | ### Mixed Environment Variables 283 | 284 | ```yaml 285 | additionalEnv: 286 | - name: ENVIRONMENT 287 | value: "production" 288 | - name: DATABASE_URL 289 | valueFrom: 290 | secretKeyRef: 291 | name: database-credentials 292 | key: url 293 | - name: FEATURE_FLAGS 294 | valueFrom: 295 | configMapKeyRef: 296 | name: feature-config 297 | key: flags 298 | ``` 299 | 300 | ## Examples 301 | 302 | ### Deploy workers with multiple namespaces 303 | 304 | ```bash 305 | helm install benchmark-workers oci://ghcr.io/temporalio/charts/benchmark-workers \ 306 | --set temporal.namespace="{namespace1,namespace2,namespace3}" 307 | ``` 308 | 309 | ### Deploy with different endpoints per namespace 310 | 311 | ```bash 312 | helm install benchmark-workers oci://ghcr.io/temporalio/charts/benchmark-workers \ 313 | --set temporal.namespace="{namespace1,namespace2}" \ 314 | --set temporal.grpcEndpoint="{temporal1.temporal:7233,temporal2.temporal:7233}" 315 | ``` 316 | 317 | ### Deploy workers with increased pollers 318 | 319 | ```bash 320 | helm install benchmark-workers oci://ghcr.io/temporalio/charts/benchmark-workers \ 321 | --set temporal.workflowTaskPollers=32 \ 322 | --set temporal.activityTaskPollers=16 323 | ``` 324 | 325 | ### Deploy with a high load soak test 326 | 327 | ```bash 328 | helm install benchmark-workers oci://ghcr.io/temporalio/charts/benchmark-workers \ 329 | --set soakTest.concurrentWorkflows=50 330 | ``` 331 | 332 | ### Deploy with TLS enabled for multiple namespaces 333 | 334 | ```bash 335 | helm install benchmark-workers oci://ghcr.io/temporalio/charts/benchmark-workers \ 336 | --set temporal.namespace="{namespace1,namespace2}" \ 337 | --set temporal.tls.enabled=true \ 338 | --set-file temporal.tls.key=/path/to/key.pem \ 339 | --set-file temporal.tls.cert=/path/to/cert.pem \ 340 | --set-file temporal.tls.ca=/path/to/ca.pem 341 | ``` 342 | 343 | ### Deploy with Prometheus metrics enabled 344 | 345 | ```bash 346 | helm install benchmark-workers oci://ghcr.io/temporalio/charts/benchmark-workers \ 347 | --set metrics.enabled=true \ 348 | --set metrics.serviceMonitor.enabled=true 349 | ``` 350 | 351 | ### Deploy with additional environment variables 352 | 353 | ```bash 354 | helm install benchmark-workers oci://ghcr.io/temporalio/charts/benchmark-workers \ 355 | --set additionalEnv[0].name=LOG_LEVEL \ 356 | --set additionalEnv[0].value=DEBUG \ 357 | --set additionalEnv[1].name=CUSTOM_SETTING \ 358 | --set additionalEnv[1].value=production-value 359 | ``` 360 | 361 | ### Scale worker or soak test replicas 362 | 363 | ```bash 364 | helm install benchmark-workers oci://ghcr.io/temporalio/charts/benchmark-workers \ 365 | --set workers.replicaCount=3 \ 366 | --set soakTest.replicaCount=2 367 | ``` -------------------------------------------------------------------------------- /charts/benchmark-workers/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "benchmark-workers.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "benchmark-workers.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "benchmark-workers.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "benchmark-workers.labels" -}} 37 | helm.sh/chart: {{ include "benchmark-workers.chart" . }} 38 | {{ include "benchmark-workers.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | app: benchmark 44 | {{- end }} 45 | 46 | {{/* 47 | Selector labels 48 | */}} 49 | {{- define "benchmark-workers.selectorLabels" -}} 50 | app.kubernetes.io/name: {{ include "benchmark-workers.name" . }} 51 | app.kubernetes.io/instance: {{ .Release.Name }} 52 | app: benchmark 53 | {{- end }} 54 | 55 | {{/* 56 | Convert single value or array to comma-separated string 57 | */}} 58 | {{- define "benchmark-workers.toCommaSeparated" -}} 59 | {{- if kindIs "slice" . -}} 60 | {{- join "," . -}} 61 | {{- else -}} 62 | {{- . -}} 63 | {{- end -}} 64 | {{- end }} 65 | 66 | {{/* 67 | Get GRPC endpoints as comma-separated string 68 | */}} 69 | {{- define "benchmark-workers.grpcEndpoints" -}} 70 | {{- include "benchmark-workers.toCommaSeparated" .Values.temporal.grpcEndpoint -}} 71 | {{- end }} 72 | 73 | {{/* 74 | Get namespaces as comma-separated string 75 | */}} 76 | {{- define "benchmark-workers.namespaces" -}} 77 | {{- include "benchmark-workers.toCommaSeparated" .Values.temporal.namespace -}} 78 | {{- end }} 79 | 80 | {{/* 81 | Get TLS keys as comma-separated string 82 | Uses array values if provided, otherwise falls back to single value 83 | */}} 84 | {{- define "benchmark-workers.tlsKeys" -}} 85 | {{- if .Values.temporal.tls.keys -}} 86 | {{- include "benchmark-workers.toCommaSeparated" .Values.temporal.tls.keys -}} 87 | {{- else if .Values.temporal.tls.key -}} 88 | {{- .Values.temporal.tls.key -}} 89 | {{- end -}} 90 | {{- end }} 91 | 92 | {{/* 93 | Get TLS certs as comma-separated string 94 | Uses array values if provided, otherwise falls back to single value 95 | */}} 96 | {{- define "benchmark-workers.tlsCerts" -}} 97 | {{- if .Values.temporal.tls.certs -}} 98 | {{- include "benchmark-workers.toCommaSeparated" .Values.temporal.tls.certs -}} 99 | {{- else if .Values.temporal.tls.cert -}} 100 | {{- .Values.temporal.tls.cert -}} 101 | {{- end -}} 102 | {{- end }} 103 | 104 | {{/* 105 | Get TLS CAs as comma-separated string 106 | Uses array values if provided, otherwise falls back to single value 107 | */}} 108 | {{- define "benchmark-workers.tlsCas" -}} 109 | {{- if .Values.temporal.tls.cas -}} 110 | {{- include "benchmark-workers.toCommaSeparated" .Values.temporal.tls.cas -}} 111 | {{- else if .Values.temporal.tls.ca -}} 112 | {{- .Values.temporal.tls.ca -}} 113 | {{- end -}} 114 | {{- end }} 115 | 116 | {{/* 117 | Get existing secrets as comma-separated string 118 | Uses array values if provided, otherwise falls back to single value 119 | */}} 120 | {{- define "benchmark-workers.existingSecrets" -}} 121 | {{- if .Values.temporal.tls.existingSecrets -}} 122 | {{- include "benchmark-workers.toCommaSeparated" .Values.temporal.tls.existingSecrets -}} 123 | {{- else if .Values.temporal.tls.existingSecret -}} 124 | {{- .Values.temporal.tls.existingSecret -}} 125 | {{- end -}} 126 | {{- end }} -------------------------------------------------------------------------------- /charts/benchmark-workers/templates/deployment-soak-test.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.soakTest.enabled }} 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{ include "benchmark-workers.fullname" . }}-soak-test 6 | labels: 7 | app: benchmark 8 | component: soak-test 9 | {{- include "benchmark-workers.labels" . | nindent 4 }} 10 | spec: 11 | replicas: {{ .Values.soakTest.replicaCount }} 12 | selector: 13 | matchLabels: 14 | app: benchmark 15 | component: soak-test 16 | {{- include "benchmark-workers.selectorLabels" . | nindent 6 }} 17 | template: 18 | metadata: 19 | labels: 20 | app: benchmark 21 | component: soak-test 22 | {{- include "benchmark-workers.selectorLabels" . | nindent 8 }} 23 | spec: 24 | containers: 25 | - name: benchmark-soak-test 26 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 27 | imagePullPolicy: {{ .Values.image.pullPolicy }} 28 | env: 29 | - name: TEMPORAL_GRPC_ENDPOINT 30 | value: {{ include "benchmark-workers.grpcEndpoints" . | quote }} 31 | - name: TEMPORAL_NAMESPACE 32 | value: {{ include "benchmark-workers.namespaces" . | quote }} 33 | - name: TEMPORAL_TASK_QUEUE 34 | value: {{ .Values.temporal.taskQueue | quote }} 35 | - name: CONCURRENT_WORKFLOWS 36 | value: {{ .Values.soakTest.concurrentWorkflows | quote }} 37 | {{- if .Values.temporal.tls.enabled }} 38 | {{- $tlsKeys := include "benchmark-workers.tlsKeys" . }} 39 | {{- $tlsCerts := include "benchmark-workers.tlsCerts" . }} 40 | {{- $tlsCas := include "benchmark-workers.tlsCas" . }} 41 | {{- if or $tlsKeys $tlsCerts $tlsCas }} 42 | - name: TEMPORAL_TLS_KEY 43 | value: {{ $tlsKeys | quote }} 44 | - name: TEMPORAL_TLS_CERT 45 | value: {{ $tlsCerts | quote }} 46 | {{- if $tlsCas }} 47 | - name: TEMPORAL_TLS_CA 48 | value: {{ $tlsCas | quote }} 49 | {{- end }} 50 | {{- else }} 51 | - name: TEMPORAL_TLS_KEY 52 | value: "/etc/temporal/tls/key" 53 | - name: TEMPORAL_TLS_CERT 54 | value: "/etc/temporal/tls/cert" 55 | - name: TEMPORAL_TLS_CA 56 | value: "/etc/temporal/tls/ca" 57 | {{- end }} 58 | {{- if .Values.temporal.tls.disableHostVerification }} 59 | - name: TEMPORAL_TLS_DISABLE_HOST_VERIFICATION 60 | value: "true" 61 | {{- end }} 62 | {{- end }} 63 | command: 64 | - "runner" 65 | - "-w" 66 | - "-c" 67 | - "$(CONCURRENT_WORKFLOWS)" 68 | - "-t" 69 | - {{ .Values.soakTest.workflowType | quote }} 70 | - {{ .Values.soakTest.workflowArgs | quote }} 71 | {{- if .Values.soakTest.resources }} 72 | resources: 73 | {{- toYaml .Values.soakTest.resources | nindent 10 }} 74 | {{- end }} 75 | {{- if and .Values.temporal.tls.enabled (not (or (include "benchmark-workers.tlsKeys" .) (include "benchmark-workers.tlsCerts" .) (include "benchmark-workers.tlsCas" .))) }} 76 | volumeMounts: 77 | - name: tls 78 | mountPath: /etc/temporal/tls 79 | readOnly: true 80 | {{- end }} 81 | {{- if and .Values.temporal.tls.enabled (not (or (include "benchmark-workers.tlsKeys" .) (include "benchmark-workers.tlsCerts" .) (include "benchmark-workers.tlsCas" .))) }} 82 | volumes: 83 | - name: tls 84 | secret: 85 | secretName: {{ if .Values.temporal.tls.existingSecret }}{{ .Values.temporal.tls.existingSecret }}{{ else }}{{ include "benchmark-workers.fullname" . }}-tls{{ end }} 86 | {{- end }} 87 | {{- with .Values.nodeSelector }} 88 | nodeSelector: 89 | {{- toYaml . | nindent 8 }} 90 | {{- end }} 91 | {{- with .Values.affinity }} 92 | affinity: 93 | {{- toYaml . | nindent 8 }} 94 | {{- end }} 95 | {{- with .Values.tolerations }} 96 | tolerations: 97 | {{- toYaml . | nindent 8 }} 98 | {{- end }} 99 | restartPolicy: Always 100 | {{- end }} -------------------------------------------------------------------------------- /charts/benchmark-workers/templates/deployment-workers.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "benchmark-workers.fullname" . }}-workers 5 | labels: 6 | app: benchmark 7 | component: workers 8 | {{- include "benchmark-workers.labels" . | nindent 4 }} 9 | spec: 10 | replicas: {{ .Values.workers.replicaCount }} 11 | selector: 12 | matchLabels: 13 | app: benchmark 14 | component: workers 15 | {{- include "benchmark-workers.selectorLabels" . | nindent 6 }} 16 | template: 17 | metadata: 18 | labels: 19 | app: benchmark 20 | component: workers 21 | {{- include "benchmark-workers.selectorLabels" . | nindent 8 }} 22 | spec: 23 | containers: 24 | - name: benchmark-workers 25 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 26 | imagePullPolicy: {{ .Values.image.pullPolicy }} 27 | {{- if .Values.metrics.enabled }} 28 | ports: 29 | - name: metrics 30 | containerPort: {{ .Values.metrics.port }} 31 | protocol: TCP 32 | {{- end }} 33 | env: 34 | - name: TEMPORAL_GRPC_ENDPOINT 35 | value: {{ include "benchmark-workers.grpcEndpoints" . | quote }} 36 | - name: TEMPORAL_NAMESPACE 37 | value: {{ include "benchmark-workers.namespaces" . | quote }} 38 | - name: TEMPORAL_TASK_QUEUE 39 | value: {{ .Values.temporal.taskQueue | quote }} 40 | - name: TEMPORAL_WORKFLOW_TASK_POLLERS 41 | value: {{ .Values.temporal.workflowTaskPollers | quote }} 42 | - name: TEMPORAL_ACTIVITY_TASK_POLLERS 43 | value: {{ .Values.temporal.activityTaskPollers | quote }} 44 | {{- if .Values.metrics.enabled }} 45 | - name: PROMETHEUS_ENDPOINT 46 | value: {{ .Values.metrics.prometheusEndpoint | quote }} 47 | {{- end }} 48 | {{- if .Values.temporal.tls.enabled }} 49 | {{- $tlsKeys := include "benchmark-workers.tlsKeys" . }} 50 | {{- $tlsCerts := include "benchmark-workers.tlsCerts" . }} 51 | {{- $tlsCas := include "benchmark-workers.tlsCas" . }} 52 | {{- if or $tlsKeys $tlsCerts $tlsCas }} 53 | - name: TEMPORAL_TLS_KEY 54 | value: {{ $tlsKeys | quote }} 55 | - name: TEMPORAL_TLS_CERT 56 | value: {{ $tlsCerts | quote }} 57 | {{- if $tlsCas }} 58 | - name: TEMPORAL_TLS_CA 59 | value: {{ $tlsCas | quote }} 60 | {{- end }} 61 | {{- else }} 62 | - name: TEMPORAL_TLS_KEY 63 | value: "/etc/temporal/tls/key" 64 | - name: TEMPORAL_TLS_CERT 65 | value: "/etc/temporal/tls/cert" 66 | - name: TEMPORAL_TLS_CA 67 | value: "/etc/temporal/tls/ca" 68 | {{- end }} 69 | {{- if .Values.temporal.tls.disableHostVerification }} 70 | - name: TEMPORAL_TLS_DISABLE_HOST_VERIFICATION 71 | value: "true" 72 | {{- end }} 73 | {{- end }} 74 | {{- if .Values.additionalEnv }} 75 | {{- toYaml .Values.additionalEnv | nindent 8 }} 76 | {{- end }} 77 | {{- if .Values.workers.resources }} 78 | resources: 79 | {{- toYaml .Values.workers.resources | nindent 10 }} 80 | {{- end }} 81 | {{- if and .Values.temporal.tls.enabled (not (or (include "benchmark-workers.tlsKeys" .) (include "benchmark-workers.tlsCerts" .) (include "benchmark-workers.tlsCas" .))) }} 82 | volumeMounts: 83 | - name: tls 84 | mountPath: /etc/temporal/tls 85 | readOnly: true 86 | {{- end }} 87 | {{- if and .Values.temporal.tls.enabled (not (or (include "benchmark-workers.tlsKeys" .) (include "benchmark-workers.tlsCerts" .) (include "benchmark-workers.tlsCas" .))) }} 88 | volumes: 89 | - name: tls 90 | secret: 91 | secretName: {{ if .Values.temporal.tls.existingSecret }}{{ .Values.temporal.tls.existingSecret }}{{ else }}{{ include "benchmark-workers.fullname" . }}-tls{{ end }} 92 | {{- end }} 93 | {{- with .Values.nodeSelector }} 94 | nodeSelector: 95 | {{- toYaml . | nindent 8 }} 96 | {{- end }} 97 | {{- with .Values.affinity }} 98 | affinity: 99 | {{- toYaml . | nindent 8 }} 100 | {{- end }} 101 | {{- with .Values.tolerations }} 102 | tolerations: 103 | {{- toYaml . | nindent 8 }} 104 | {{- end }} 105 | restartPolicy: Always -------------------------------------------------------------------------------- /charts/benchmark-workers/templates/secret-tls.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.temporal.tls.enabled }} 2 | {{- if and .Values.temporal.tls.key .Values.temporal.tls.cert }} 3 | apiVersion: v1 4 | kind: Secret 5 | metadata: 6 | name: {{ include "benchmark-workers.fullname" . }}-tls 7 | labels: 8 | {{- include "benchmark-workers.labels" . | nindent 4 }} 9 | type: Opaque 10 | data: 11 | key: {{ .Values.temporal.tls.key | b64enc }} 12 | cert: {{ .Values.temporal.tls.cert | b64enc }} 13 | {{- if .Values.temporal.tls.ca }} 14 | ca: {{ .Values.temporal.tls.ca | b64enc }} 15 | {{- end }} 16 | {{- end }} 17 | {{- end }} -------------------------------------------------------------------------------- /charts/benchmark-workers/templates/service.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.metrics.enabled }} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ include "benchmark-workers.fullname" . }} 6 | labels: 7 | app: benchmark 8 | {{- include "benchmark-workers.labels" . | nindent 4 }} 9 | {{- with .Values.metrics.service.annotations }} 10 | annotations: 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | spec: 14 | type: ClusterIP 15 | clusterIP: None 16 | ports: 17 | - port: {{ .Values.metrics.port }} 18 | targetPort: metrics 19 | protocol: TCP 20 | name: metrics 21 | selector: 22 | app: benchmark 23 | {{- include "benchmark-workers.selectorLabels" . | nindent 4 }} 24 | {{- end }} -------------------------------------------------------------------------------- /charts/benchmark-workers/templates/servicemonitor.yaml: -------------------------------------------------------------------------------- 1 | {{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} 2 | apiVersion: monitoring.coreos.com/v1 3 | kind: ServiceMonitor 4 | metadata: 5 | name: {{ include "benchmark-workers.fullname" . }} 6 | labels: 7 | app: benchmark 8 | {{- include "benchmark-workers.labels" . | nindent 4 }} 9 | {{- with .Values.metrics.serviceMonitor.additionalLabels }} 10 | {{- toYaml . | nindent 4 }} 11 | {{- end }} 12 | spec: 13 | selector: 14 | matchLabels: 15 | app: benchmark 16 | {{- include "benchmark-workers.selectorLabels" . | nindent 6 }} 17 | endpoints: 18 | - port: metrics 19 | interval: {{ .Values.metrics.serviceMonitor.interval }} 20 | {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} 21 | scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} 22 | {{- end }} 23 | namespaceSelector: 24 | matchNames: 25 | - {{ .Release.Namespace }} 26 | {{- end }} -------------------------------------------------------------------------------- /charts/benchmark-workers/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for benchmark-workers chart 2 | nameOverride: "" 3 | fullnameOverride: "" 4 | 5 | image: 6 | repository: ghcr.io/temporalio/benchmark-workers 7 | tag: v1.3.1 8 | pullPolicy: Always 9 | 10 | temporal: 11 | # GRPC endpoint(s) - can be a single string or array of strings 12 | # Examples: 13 | # grpcEndpoint: "temporal-frontend.temporal:7233" 14 | # grpcEndpoint: ["temporal1.temporal:7233", "temporal2.temporal:7233"] 15 | grpcEndpoint: "temporal-frontend.temporal:7233" 16 | 17 | # Namespace(s) - can be a single string or array of strings 18 | # Examples: 19 | # namespace: "default" 20 | # namespace: ["namespace1", "namespace2", "namespace3"] 21 | namespace: "default" 22 | 23 | taskQueue: "benchmark" 24 | workflowTaskPollers: "16" 25 | activityTaskPollers: "8" 26 | 27 | tls: 28 | enabled: false 29 | # TLS configuration can be provided as single values (reused for all namespaces) 30 | # or as arrays (one per namespace) 31 | # Provide these values or use an existing Secret 32 | key: "" 33 | cert: "" 34 | ca: "" 35 | disableHostVerification: false 36 | 37 | # For multi-namespace with different TLS configs, use arrays: 38 | # keys: ["key1-content", "key2-content"] 39 | # certs: ["cert1-content", "cert2-content"] 40 | # cas: ["ca1-content", "ca2-content"] 41 | keys: [] 42 | certs: [] 43 | cas: [] 44 | 45 | # If using an existing secret, specify the name(s) 46 | # Can be a single secret (reused for all namespaces) or array of secrets 47 | existingSecret: "" 48 | existingSecrets: [] 49 | 50 | metrics: 51 | enabled: true 52 | # The port to expose metrics on 53 | port: 9090 54 | # The Prometheus endpoint path and listening address 55 | prometheusEndpoint: ":9090" 56 | # Headless service configuration 57 | service: 58 | annotations: {} 59 | # ServiceMonitor configuration for Prometheus Operator 60 | serviceMonitor: 61 | enabled: true 62 | # Additional labels to add to the ServiceMonitor 63 | additionalLabels: {} 64 | # Scrape interval 65 | interval: 15s 66 | # Scrape timeout 67 | scrapeTimeout: 10s 68 | 69 | workers: 70 | # Number of worker replicas 71 | replicaCount: 1 72 | # Resources configuration 73 | resources: {} 74 | # limits: 75 | # cpu: 1000m 76 | # memory: 1Gi 77 | # requests: 78 | # cpu: 500m 79 | # memory: 512Mi 80 | 81 | soakTest: 82 | enabled: true 83 | # Number of soak test replicas 84 | replicaCount: 1 85 | concurrentWorkflows: "10" 86 | workflowType: "ExecuteActivity" 87 | workflowArgs: '{ "Count": 3, "Activity": "Echo", "Input": { "Message": "test" } }' 88 | # Resources configuration 89 | resources: {} 90 | # limits: 91 | # cpu: 500m 92 | # memory: 512Mi 93 | # requests: 94 | # cpu: 200m 95 | # memory: 256Mi 96 | 97 | nodeSelector: {} 98 | tolerations: [] 99 | affinity: {} -------------------------------------------------------------------------------- /cmd/runner/log.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "go.temporal.io/sdk/log" 5 | ) 6 | 7 | // NoopLogger is Logger implementation that doesn't produce any logs. 8 | type NoopLogger struct { 9 | } 10 | 11 | // NewNopLogger creates new instance of NoopLogger. 12 | func NewNopLogger() *NoopLogger { 13 | return &NoopLogger{} 14 | } 15 | 16 | // Debug does nothing. 17 | func (l *NoopLogger) Debug(string, ...interface{}) {} 18 | 19 | // Info does nothing. 20 | func (l *NoopLogger) Info(string, ...interface{}) {} 21 | 22 | // Warn does nothing. 23 | func (l *NoopLogger) Warn(string, ...interface{}) {} 24 | 25 | // Error does nothing. 26 | func (l *NoopLogger) Error(string, ...interface{}) {} 27 | 28 | // With returns new NoopLogger. 29 | func (l *NoopLogger) With(...interface{}) log.Logger { 30 | return NewNopLogger() 31 | } 32 | -------------------------------------------------------------------------------- /cmd/runner/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "encoding/json" 8 | "flag" 9 | "fmt" 10 | "log" 11 | "os" 12 | "strings" 13 | "sync/atomic" 14 | "time" 15 | 16 | "github.com/alitto/pond" 17 | "github.com/pborman/uuid" 18 | "github.com/uber-go/tally/v4/prometheus" 19 | sdktally "go.temporal.io/sdk/contrib/tally" 20 | "go.uber.org/automaxprocs/maxprocs" 21 | 22 | "go.temporal.io/sdk/client" 23 | ) 24 | 25 | var nWorfklows = flag.Int("c", 10, "concurrent workflows") 26 | var sWorkflow = flag.String("t", "", "workflow type") 27 | var sSignalType = flag.String("s", "", "signal type") 28 | var bWait = flag.Bool("w", true, "wait for workflows to complete") 29 | var sNamespace = flag.String("n", "default", "namespace (comma-separated list supported)") 30 | var sTaskQueue = flag.String("tq", "benchmark", "task queue") 31 | 32 | // parseCommaSeparatedEnv parses a comma-separated environment variable and returns a slice 33 | // If there's only one value but multiple namespaces are needed, it reuses that value 34 | func parseCommaSeparatedEnv(envVar string, numNamespaces int) []string { 35 | value := os.Getenv(envVar) 36 | if value == "" { 37 | return make([]string, numNamespaces) 38 | } 39 | 40 | values := strings.Split(value, ",") 41 | for i, v := range values { 42 | values[i] = strings.TrimSpace(v) 43 | } 44 | 45 | // If we have fewer values than namespaces, repeat the last value 46 | if len(values) < numNamespaces { 47 | lastValue := values[len(values)-1] 48 | for len(values) < numNamespaces { 49 | values = append(values, lastValue) 50 | } 51 | } 52 | 53 | return values 54 | } 55 | 56 | func main() { 57 | flag.Usage = func() { 58 | fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [flags] [workflow input] ...\n", os.Args[0]) 59 | flag.PrintDefaults() 60 | } 61 | 62 | flag.Parse() 63 | 64 | if _, err := maxprocs.Set(); err != nil { 65 | log.Printf("WARNING: failed to set GOMAXPROCS: %v.\n", err) 66 | } 67 | 68 | namespaces := *sNamespace 69 | envNamespace := os.Getenv("TEMPORAL_NAMESPACE") 70 | if envNamespace != "" && envNamespace != "default" { 71 | namespaces = envNamespace 72 | } 73 | 74 | // Parse comma-separated namespaces 75 | namespaceList := strings.Split(namespaces, ",") 76 | for i, ns := range namespaceList { 77 | namespaceList[i] = strings.TrimSpace(ns) 78 | } 79 | 80 | log.Printf("Using namespaces: %v", namespaceList) 81 | 82 | // Parse comma-separated configuration values 83 | grpcEndpoints := parseCommaSeparatedEnv("TEMPORAL_GRPC_ENDPOINT", len(namespaceList)) 84 | tlsKeyPaths := parseCommaSeparatedEnv("TEMPORAL_TLS_KEY", len(namespaceList)) 85 | tlsCertPaths := parseCommaSeparatedEnv("TEMPORAL_TLS_CERT", len(namespaceList)) 86 | tlsCaPaths := parseCommaSeparatedEnv("TEMPORAL_TLS_CA", len(namespaceList)) 87 | 88 | // Create clients for each namespace 89 | clients := make([]client.Client, len(namespaceList)) 90 | for i, namespace := range namespaceList { 91 | clientOptions := client.Options{ 92 | HostPort: grpcEndpoints[i], 93 | Namespace: namespace, 94 | Logger: NewNopLogger(), 95 | } 96 | 97 | tlsKeyPath := tlsKeyPaths[i] 98 | tlsCertPath := tlsCertPaths[i] 99 | tlsCaPath := tlsCaPaths[i] 100 | 101 | if tlsKeyPath != "" && tlsCertPath != "" { 102 | tlsConfig := tls.Config{} 103 | 104 | cert, err := tls.LoadX509KeyPair(tlsCertPath, tlsKeyPath) 105 | if err != nil { 106 | log.Fatalf("Unable to create key pair for TLS for namespace %s: %v", namespace, err) 107 | } 108 | 109 | var tlsCaPool *x509.CertPool 110 | if tlsCaPath != "" { 111 | tlsCaPool = x509.NewCertPool() 112 | b, err := os.ReadFile(tlsCaPath) 113 | if err != nil { 114 | log.Fatalf("Failed reading server CA for namespace %s: %v", namespace, err) 115 | } else if !tlsCaPool.AppendCertsFromPEM(b) { 116 | log.Fatalf("Server CA PEM file invalid for namespace %s", namespace) 117 | } 118 | } 119 | 120 | tlsConfig.Certificates = []tls.Certificate{cert} 121 | tlsConfig.RootCAs = tlsCaPool 122 | 123 | if os.Getenv("TEMPORAL_TLS_DISABLE_HOST_VERIFICATION") != "" { 124 | tlsConfig.InsecureSkipVerify = true 125 | } 126 | 127 | clientOptions.ConnectionOptions.TLS = &tlsConfig 128 | } 129 | 130 | if os.Getenv("PROMETHEUS_ENDPOINT") != "" { 131 | clientOptions.MetricsHandler = sdktally.NewMetricsHandler(newPrometheusScope(prometheus.Configuration{ 132 | ListenAddress: os.Getenv("PROMETHEUS_ENDPOINT"), 133 | TimerType: "histogram", 134 | })) 135 | } 136 | 137 | c, err := client.Dial(clientOptions) 138 | if err != nil { 139 | log.Fatalf("Unable to create client for namespace %s (endpoint: %s): %v", namespace, grpcEndpoints[i], err) 140 | } 141 | clients[i] = c 142 | log.Printf("Created client for namespace: %s (endpoint: %s)", namespace, grpcEndpoints[i]) 143 | } 144 | 145 | // Ensure all clients are closed on exit 146 | defer func() { 147 | for _, c := range clients { 148 | c.Close() 149 | } 150 | }() 151 | 152 | var input []interface{} 153 | for _, a := range flag.Args() { 154 | var i interface{} 155 | err := json.Unmarshal([]byte(a), &i) 156 | if err != nil { 157 | log.Fatalln("Unable to parse input", err) 158 | } 159 | input = append(input, i) 160 | } 161 | 162 | pool := pond.New(*nWorfklows, 0) 163 | 164 | // Counter for rotating among clients 165 | var clientCounter uint64 166 | 167 | var starter func() (client.WorkflowRun, error) 168 | 169 | if *sSignalType != "" { 170 | starter = func() (client.WorkflowRun, error) { 171 | // Rotate among clients 172 | clientIndex := atomic.AddUint64(&clientCounter, 1) % uint64(len(clients)) 173 | c := clients[clientIndex] 174 | 175 | wID := uuid.New() 176 | return c.SignalWithStartWorkflow( 177 | context.Background(), 178 | wID, 179 | *sSignalType, 180 | nil, 181 | client.StartWorkflowOptions{ 182 | ID: wID, 183 | TaskQueue: *sTaskQueue, 184 | }, 185 | *sWorkflow, 186 | input..., 187 | ) 188 | } 189 | } else { 190 | starter = func() (client.WorkflowRun, error) { 191 | // Rotate among clients 192 | clientIndex := atomic.AddUint64(&clientCounter, 1) % uint64(len(clients)) 193 | c := clients[clientIndex] 194 | 195 | return c.ExecuteWorkflow( 196 | context.Background(), 197 | client.StartWorkflowOptions{ 198 | TaskQueue: *sTaskQueue, 199 | }, 200 | *sWorkflow, 201 | input..., 202 | ) 203 | } 204 | } 205 | 206 | go (func() { 207 | for { 208 | pool.Submit(func() { 209 | wf, err := starter() 210 | if err != nil { 211 | log.Println("Unable to start workflow", err) 212 | return 213 | } 214 | 215 | if *bWait { 216 | err = wf.Get(context.Background(), nil) 217 | if err != nil { 218 | log.Println("Workflow failed", err) 219 | return 220 | } 221 | } 222 | }) 223 | } 224 | })() 225 | 226 | var lastCompleted uint64 227 | lastCheck := time.Now() 228 | 229 | for { 230 | rate := float64(pool.CompletedTasks()-lastCompleted) / time.Since(lastCheck).Seconds() 231 | 232 | fmt.Printf("Concurrent: %d Workflows: %d Rate: %f\n", pool.RunningWorkers(), pool.CompletedTasks(), rate) 233 | 234 | lastCheck = time.Now() 235 | lastCompleted = pool.CompletedTasks() 236 | 237 | time.Sleep(10 * time.Second) 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /cmd/runner/metrics.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | prom "github.com/prometheus/client_golang/prometheus" 8 | "github.com/uber-go/tally/v4" 9 | "github.com/uber-go/tally/v4/prometheus" 10 | sdktally "go.temporal.io/sdk/contrib/tally" 11 | ) 12 | 13 | func newPrometheusScope(c prometheus.Configuration) tally.Scope { 14 | reporter, err := c.NewReporter( 15 | prometheus.ConfigurationOptions{ 16 | Registry: prom.NewRegistry(), 17 | OnError: func(err error) { 18 | log.Println("error in prometheus reporter", err) 19 | }, 20 | }, 21 | ) 22 | if err != nil { 23 | log.Fatalln("error creating prometheus reporter", err) 24 | } 25 | scopeOpts := tally.ScopeOptions{ 26 | CachedReporter: reporter, 27 | Separator: prometheus.DefaultSeparator, 28 | SanitizeOptions: &sdktally.PrometheusSanitizeOptions, 29 | } 30 | scope, _ := tally.NewRootScope(scopeOpts, time.Second) 31 | 32 | log.Println("prometheus metrics scope created") 33 | return scope 34 | } 35 | -------------------------------------------------------------------------------- /cmd/worker/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "log" 7 | "os" 8 | "strconv" 9 | "strings" 10 | "sync" 11 | 12 | "github.com/temporalio/benchmark-workers/activities" 13 | "github.com/temporalio/benchmark-workers/workflows" 14 | "github.com/uber-go/tally/v4/prometheus" 15 | sdktally "go.temporal.io/sdk/contrib/tally" 16 | "go.uber.org/automaxprocs/maxprocs" 17 | 18 | "go.temporal.io/sdk/activity" 19 | "go.temporal.io/sdk/client" 20 | "go.temporal.io/sdk/worker" 21 | "go.temporal.io/sdk/workflow" 22 | ) 23 | 24 | // parseCommaSeparatedEnv parses a comma-separated environment variable and returns a slice 25 | // If there's only one value but multiple namespaces are needed, it reuses that value 26 | func parseCommaSeparatedEnv(envVar string, numNamespaces int) []string { 27 | value := os.Getenv(envVar) 28 | if value == "" { 29 | return make([]string, numNamespaces) 30 | } 31 | 32 | values := strings.Split(value, ",") 33 | for i, v := range values { 34 | values[i] = strings.TrimSpace(v) 35 | } 36 | 37 | // If we have fewer values than namespaces, repeat the last value 38 | if len(values) < numNamespaces { 39 | lastValue := values[len(values)-1] 40 | for len(values) < numNamespaces { 41 | values = append(values, lastValue) 42 | } 43 | } 44 | 45 | return values 46 | } 47 | 48 | func main() { 49 | if _, err := maxprocs.Set(); err != nil { 50 | log.Printf("WARNING: failed to set GOMAXPROCS: %v.\n", err) 51 | } 52 | 53 | namespaces := os.Getenv("TEMPORAL_NAMESPACE") 54 | if namespaces == "" { 55 | namespaces = "default" 56 | } 57 | 58 | // Parse comma-separated namespaces 59 | namespaceList := strings.Split(namespaces, ",") 60 | for i, ns := range namespaceList { 61 | namespaceList[i] = strings.TrimSpace(ns) 62 | } 63 | 64 | log.Printf("Creating workers for namespaces: %v", namespaceList) 65 | 66 | taskQueue := os.Getenv("TEMPORAL_TASK_QUEUE") 67 | if taskQueue == "" { 68 | taskQueue = "benchmark" 69 | } 70 | 71 | // Parse comma-separated configuration values 72 | grpcEndpoints := parseCommaSeparatedEnv("TEMPORAL_GRPC_ENDPOINT", len(namespaceList)) 73 | tlsKeyPaths := parseCommaSeparatedEnv("TEMPORAL_TLS_KEY", len(namespaceList)) 74 | tlsCertPaths := parseCommaSeparatedEnv("TEMPORAL_TLS_CERT", len(namespaceList)) 75 | tlsCaPaths := parseCommaSeparatedEnv("TEMPORAL_TLS_CA", len(namespaceList)) 76 | 77 | // Create shared metrics handler if Prometheus is enabled 78 | var metricsHandler client.MetricsHandler 79 | if os.Getenv("PROMETHEUS_ENDPOINT") != "" { 80 | metricsHandler = sdktally.NewMetricsHandler(newPrometheusScope(prometheus.Configuration{ 81 | ListenAddress: os.Getenv("PROMETHEUS_ENDPOINT"), 82 | TimerType: "histogram", 83 | })) 84 | } 85 | 86 | // Create workers for each namespace 87 | var wg sync.WaitGroup 88 | workers := make([]worker.Worker, len(namespaceList)) 89 | clients := make([]client.Client, len(namespaceList)) 90 | 91 | for i, namespace := range namespaceList { 92 | clientOptions := client.Options{ 93 | HostPort: grpcEndpoints[i], 94 | Namespace: namespace, 95 | } 96 | 97 | tlsKeyPath := tlsKeyPaths[i] 98 | tlsCertPath := tlsCertPaths[i] 99 | tlsCaPath := tlsCaPaths[i] 100 | 101 | if tlsKeyPath != "" && tlsCertPath != "" { 102 | tlsConfig := tls.Config{} 103 | 104 | cert, err := tls.LoadX509KeyPair(tlsCertPath, tlsKeyPath) 105 | if err != nil { 106 | log.Fatalf("Unable to create key pair for TLS for namespace %s: %v", namespace, err) 107 | } 108 | 109 | var tlsCaPool *x509.CertPool 110 | if tlsCaPath != "" { 111 | tlsCaPool = x509.NewCertPool() 112 | b, err := os.ReadFile(tlsCaPath) 113 | if err != nil { 114 | log.Fatalf("Failed reading server CA for namespace %s: %v", namespace, err) 115 | } else if !tlsCaPool.AppendCertsFromPEM(b) { 116 | log.Fatalf("Server CA PEM file invalid for namespace %s", namespace) 117 | } 118 | } 119 | 120 | tlsConfig.Certificates = []tls.Certificate{cert} 121 | tlsConfig.RootCAs = tlsCaPool 122 | 123 | if os.Getenv("TEMPORAL_TLS_DISABLE_HOST_VERIFICATION") != "" { 124 | tlsConfig.InsecureSkipVerify = true 125 | } 126 | 127 | clientOptions.ConnectionOptions.TLS = &tlsConfig 128 | } 129 | 130 | if metricsHandler != nil { 131 | clientOptions.MetricsHandler = metricsHandler 132 | } 133 | 134 | c, err := client.Dial(clientOptions) 135 | if err != nil { 136 | log.Fatalf("Unable to create client for namespace %s (endpoint: %s): %v", namespace, grpcEndpoints[i], err) 137 | } 138 | clients[i] = c 139 | 140 | workerOptions := worker.Options{} 141 | 142 | if os.Getenv("TEMPORAL_WORKFLOW_TASK_POLLERS") != "" { 143 | pollers, err := strconv.Atoi(os.Getenv("TEMPORAL_WORKFLOW_TASK_POLLERS")) 144 | if err != nil { 145 | log.Fatalf("TEMPORAL_WORKFLOW_TASK_POLLERS is invalid: %v", err) 146 | } 147 | workerOptions.MaxConcurrentWorkflowTaskPollers = pollers 148 | } 149 | 150 | if os.Getenv("TEMPORAL_ACTIVITY_TASK_POLLERS") != "" { 151 | pollers, err := strconv.Atoi(os.Getenv("TEMPORAL_ACTIVITY_TASK_POLLERS")) 152 | if err != nil { 153 | log.Fatalf("TEMPORAL_ACTIVITY_TASK_POLLERS is invalid: %v", err) 154 | } 155 | workerOptions.MaxConcurrentActivityTaskPollers = pollers 156 | } 157 | 158 | // TODO: Support more worker options 159 | 160 | w := worker.New(c, taskQueue, workerOptions) 161 | 162 | w.RegisterWorkflowWithOptions(workflows.ExecuteActivityWorkflow, workflow.RegisterOptions{Name: "ExecuteActivity"}) 163 | w.RegisterWorkflowWithOptions(workflows.ReceiveSignalWorkflow, workflow.RegisterOptions{Name: "ReceiveSignal"}) 164 | w.RegisterWorkflowWithOptions(workflows.DSLWorkflow, workflow.RegisterOptions{Name: "DSL"}) 165 | w.RegisterActivityWithOptions(activities.SleepActivity, activity.RegisterOptions{Name: "Sleep"}) 166 | w.RegisterActivityWithOptions(activities.EchoActivity, activity.RegisterOptions{Name: "Echo"}) 167 | 168 | workers[i] = w 169 | log.Printf("Created worker for namespace: %s (endpoint: %s)", namespace, grpcEndpoints[i]) 170 | } 171 | 172 | // Ensure all clients are closed on exit 173 | defer func() { 174 | for _, c := range clients { 175 | c.Close() 176 | } 177 | }() 178 | 179 | // Start all workers concurrently 180 | for i, w := range workers { 181 | wg.Add(1) 182 | go func(w worker.Worker, namespace string) { 183 | defer wg.Done() 184 | log.Printf("Starting worker for namespace: %s", namespace) 185 | err := w.Run(worker.InterruptCh()) 186 | if err != nil { 187 | log.Printf("Worker for namespace %s failed: %v", namespace, err) 188 | } 189 | }(w, namespaceList[i]) 190 | } 191 | 192 | // Wait for all workers to complete 193 | wg.Wait() 194 | } 195 | -------------------------------------------------------------------------------- /cmd/worker/metrics.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | prom "github.com/prometheus/client_golang/prometheus" 8 | "github.com/uber-go/tally/v4" 9 | "github.com/uber-go/tally/v4/prometheus" 10 | sdktally "go.temporal.io/sdk/contrib/tally" 11 | ) 12 | 13 | func newPrometheusScope(c prometheus.Configuration) tally.Scope { 14 | reporter, err := c.NewReporter( 15 | prometheus.ConfigurationOptions{ 16 | Registry: prom.NewRegistry(), 17 | OnError: func(err error) { 18 | log.Println("error in prometheus reporter", err) 19 | }, 20 | }, 21 | ) 22 | if err != nil { 23 | log.Fatalln("error creating prometheus reporter", err) 24 | } 25 | scopeOpts := tally.ScopeOptions{ 26 | CachedReporter: reporter, 27 | Separator: prometheus.DefaultSeparator, 28 | SanitizeOptions: &sdktally.PrometheusSanitizeOptions, 29 | } 30 | scope, _ := tally.NewRootScope(scopeOpts, time.Second) 31 | 32 | log.Println("prometheus metrics scope created") 33 | return scope 34 | } 35 | -------------------------------------------------------------------------------- /deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | app: benchmark 6 | component: workers 7 | name: benchmark-workers 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: benchmark 13 | component: workers 14 | template: 15 | metadata: 16 | labels: 17 | app: benchmark 18 | component: workers 19 | spec: 20 | containers: 21 | - image: ghcr.io/temporalio/benchmark-workers:latest 22 | imagePullPolicy: Always 23 | name: benchmark-workers 24 | env: 25 | - name: TEMPORAL_GRPC_ENDPOINT 26 | value: "temporal-frontend.temporal:7233" 27 | - name: TEMPORAL_NAMESPACE 28 | value: "default" 29 | - name: TEMPORAL_TASK_QUEUE 30 | value: "benchmark" 31 | - name: TEMPORAL_WORKFLOW_TASK_POLLERS 32 | value: "16" 33 | - name: TEMPORAL_ACTIVITY_TASK_POLLERS 34 | value: "8" 35 | restartPolicy: Always 36 | --- 37 | apiVersion: apps/v1 38 | kind: Deployment 39 | metadata: 40 | labels: 41 | app: benchmark 42 | component: soak-test 43 | name: benchmark-soak-test 44 | spec: 45 | replicas: 1 46 | selector: 47 | matchLabels: 48 | app: benchmark 49 | component: soak-test 50 | template: 51 | metadata: 52 | labels: 53 | app: benchmark 54 | component: soak-test 55 | spec: 56 | containers: 57 | - image: ghcr.io/temporalio/benchmark-workers:latest 58 | imagePullPolicy: Always 59 | name: benchmark-soak-test 60 | env: 61 | - name: TEMPORAL_GRPC_ENDPOINT 62 | value: "temporal-frontend.temporal:7233" 63 | - name: TEMPORAL_NAMESPACE 64 | value: "default" 65 | - name: TEMPORAL_TASK_QUEUE 66 | value: "benchmark" 67 | - name: CONCURRENT_WORKFLOWS 68 | value: "10" 69 | command: ["runner", "-w", "-c", "$(CONCURRENT_WORKFLOWS)", "-t", "ExecuteActivity", '{ "Count": 3, "Activity": "Echo", "Input": { "Message": "test" } }'] 70 | restartPolicy: Always 71 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/temporalio/benchmark-workers 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/alitto/pond v1.8.3 7 | github.com/pborman/uuid v1.2.1 8 | github.com/prometheus/client_golang v1.14.0 9 | github.com/uber-go/tally/v4 v4.1.3 10 | go.temporal.io/sdk v1.25.1 11 | go.temporal.io/sdk/contrib/tally v0.2.0 12 | go.uber.org/automaxprocs v1.5.2 13 | ) 14 | 15 | require ( 16 | github.com/beorn7/perks v1.0.1 // indirect 17 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 18 | github.com/davecgh/go-spew v1.1.1 // indirect 19 | github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect 20 | github.com/gogo/googleapis v1.4.1 // indirect 21 | github.com/gogo/protobuf v1.3.2 // indirect 22 | github.com/gogo/status v1.1.1 // indirect 23 | github.com/golang/mock v1.6.0 // indirect 24 | github.com/golang/protobuf v1.5.3 // indirect 25 | github.com/google/uuid v1.3.0 // indirect 26 | github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect 27 | github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect 28 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 29 | github.com/pkg/errors v0.9.1 // indirect 30 | github.com/pmezard/go-difflib v1.0.0 // indirect 31 | github.com/prometheus/client_model v0.3.0 // indirect 32 | github.com/prometheus/common v0.37.0 // indirect 33 | github.com/prometheus/procfs v0.8.0 // indirect 34 | github.com/robfig/cron v1.2.0 // indirect 35 | github.com/stretchr/objx v0.5.0 // indirect 36 | github.com/stretchr/testify v1.8.4 // indirect 37 | github.com/twmb/murmur3 v1.1.6 // indirect 38 | go.temporal.io/api v1.24.0 // indirect 39 | go.uber.org/atomic v1.10.0 // indirect 40 | golang.org/x/net v0.23.0 // indirect 41 | golang.org/x/sys v0.18.0 // indirect 42 | golang.org/x/text v0.14.0 // indirect 43 | golang.org/x/time v0.3.0 // indirect 44 | google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878 // indirect 45 | google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878 // indirect 46 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 // indirect 47 | google.golang.org/grpc v1.57.0 // indirect 48 | google.golang.org/protobuf v1.31.0 // indirect 49 | gopkg.in/yaml.v3 v3.0.1 // indirect 50 | ) 51 | -------------------------------------------------------------------------------- /workflows/workflow.go: -------------------------------------------------------------------------------- 1 | package workflows 2 | 3 | import ( 4 | "time" 5 | 6 | "go.temporal.io/sdk/workflow" 7 | ) 8 | 9 | type ExecuteActivityWorkflowInput struct { 10 | Count int 11 | Activity string 12 | Input interface{} 13 | } 14 | 15 | type ReceiveSignalWorkflowInput struct { 16 | Count int 17 | Name string 18 | } 19 | 20 | // DSL step: either an activity or a child workflow (which is always this workflow) 21 | type DSLStep struct { 22 | Activity string `json:"a,omitempty"` 23 | Input interface{} `json:"i,omitempty"` 24 | Child []DSLStep `json:"c,omitempty"` 25 | Repeat int `json:"r,omitempty"` 26 | } 27 | 28 | func ExecuteActivityWorkflow(ctx workflow.Context, input ExecuteActivityWorkflowInput) error { 29 | ao := workflow.ActivityOptions{ 30 | StartToCloseTimeout: 1 * time.Minute, 31 | } 32 | ctx = workflow.WithActivityOptions(ctx, ao) 33 | 34 | for i := 0; i < input.Count; i++ { 35 | err := workflow.ExecuteActivity(ctx, input.Activity, input.Input).Get(ctx, nil) 36 | if err != nil { 37 | return err 38 | } 39 | } 40 | 41 | return nil 42 | } 43 | 44 | func ReceiveSignalWorkflow(ctx workflow.Context, input ReceiveSignalWorkflowInput) error { 45 | ch := workflow.GetSignalChannel(ctx, input.Name) 46 | 47 | for i := 0; i < input.Count; i++ { 48 | var data interface{} 49 | 50 | ch.Receive(ctx, &data) 51 | } 52 | 53 | return nil 54 | } 55 | 56 | // DSLWorkflow executes a list of DSLStep instructions. 57 | func DSLWorkflow(ctx workflow.Context, steps []DSLStep) error { 58 | ao := workflow.ActivityOptions{ 59 | StartToCloseTimeout: 1 * time.Minute, 60 | } 61 | ctx = workflow.WithActivityOptions(ctx, ao) 62 | 63 | for _, step := range steps { 64 | repeat := step.Repeat 65 | if repeat <= 0 { 66 | repeat = 1 67 | } 68 | for i := 0; i < repeat; i++ { 69 | if step.Activity != "" { 70 | if err := workflow.ExecuteActivity(ctx, step.Activity, step.Input).Get(ctx, nil); err != nil { 71 | return err 72 | } 73 | } 74 | if len(step.Child) > 0 { 75 | if err := workflow.ExecuteChildWorkflow(ctx, DSLWorkflow, step.Child).Get(ctx, nil); err != nil { 76 | return err 77 | } 78 | } 79 | } 80 | } 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /workflows/workflow_test.go: -------------------------------------------------------------------------------- 1 | package workflows 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/temporalio/benchmark-workers/activities" 8 | 9 | "github.com/stretchr/testify/mock" 10 | "github.com/stretchr/testify/require" 11 | "go.temporal.io/sdk/activity" 12 | "go.temporal.io/sdk/testsuite" 13 | ) 14 | 15 | func TestDSLWorkflow(t *testing.T) { 16 | ts := &testsuite.WorkflowTestSuite{} 17 | env := ts.NewTestWorkflowEnvironment() 18 | 19 | env.RegisterActivityWithOptions(activities.EchoActivity, activity.RegisterOptions{Name: "Echo"}) 20 | var echoCount int 21 | env.OnActivity("Echo", mock.Anything, mock.Anything).Return(func(ctx context.Context, input activities.EchoActivityInput) (string, error) { 22 | echoCount++ 23 | return input.Message, nil 24 | }) 25 | 26 | steps := []DSLStep{ 27 | {Activity: "Echo", Input: map[string]interface{}{"Message": "test"}, Repeat: 3}, 28 | {Child: []DSLStep{ 29 | {Activity: "Echo", Input: map[string]interface{}{"Message": "test"}, Repeat: 3}, 30 | }}, 31 | } 32 | 33 | env.ExecuteWorkflow(DSLWorkflow, steps) 34 | 35 | require.True(t, env.IsWorkflowCompleted()) 36 | require.NoError(t, env.GetWorkflowError()) 37 | require.Equal(t, 6, echoCount, "Echo activity should be called 6 times") 38 | } 39 | --------------------------------------------------------------------------------