├── .ci ├── jobs │ ├── apm-hey-test-benchmark.yml │ ├── apm-server.yml │ └── defaults.yml ├── scheduled-benchmark.groovy └── scripts │ ├── run-bench-in-docker.sh │ ├── run-test.sh │ └── unit-test.sh ├── .dockerignore ├── .gitignore ├── LICENSE.txt ├── README.md ├── benchmark └── benchmark.go ├── docker-compose.yml ├── docker ├── Dockerfile └── Dockerfile-bench ├── es ├── api.go └── es.go ├── go.mod ├── go.sum ├── main.go ├── main_test.go ├── models ├── input.go └── report.go ├── server └── server.go └── worker ├── logger.go ├── result.go ├── run.go ├── tracer.go └── work.go /.ci/jobs/apm-hey-test-benchmark.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - job: 3 | name: apm-server/apm-hey-test-benchmark 4 | display-name: Hey apm test benchmark pipeline scheduled daily 5 | description: Hey apm test benchmark pipeline scheduled daily from Monday to Friday 6 | view: APM-CI 7 | project-type: pipeline 8 | pipeline-scm: 9 | script-path: .ci/scheduled-benchmark.groovy 10 | scm: 11 | - git: 12 | url: git@github.com:elastic/hey-apm.git 13 | refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/* 14 | wipe-workspace: 'True' 15 | name: origin 16 | shallow-clone: true 17 | credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba 18 | reference-repo: /var/lib/jenkins/.git-references/hey-apm.git 19 | branches: 20 | - main 21 | triggers: 22 | - timed: 'H H(3-5) * * *' 23 | -------------------------------------------------------------------------------- /.ci/jobs/apm-server.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - job: 3 | name: apm-server 4 | project-type: folder 5 | -------------------------------------------------------------------------------- /.ci/jobs/defaults.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ##### GLOBAL METADATA 4 | 5 | - meta: 6 | cluster: apm-ci 7 | 8 | ##### JOB DEFAULTS 9 | 10 | - job: 11 | logrotate: 12 | numToKeep: 300 13 | publishers: 14 | - email: 15 | recipients: infra-root+build@elastic.co 16 | prune-dead-branches: true 17 | -------------------------------------------------------------------------------- /.ci/scheduled-benchmark.groovy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env groovy 2 | 3 | @Library('apm@current') _ 4 | 5 | pipeline { 6 | agent { label 'linux && immutable' } 7 | environment { 8 | BASE_DIR = 'src/github.com/elastic/hey-apm' 9 | JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba" 10 | GO_VERSION = "${params.GO_VERSION}" 11 | STACK_VERSION = "${params.STACK_VERSION}" 12 | APM_DOCKER_IMAGE = "${params.APM_DOCKER_IMAGE}" 13 | NOTIFY_TO = credentials('notify-to') 14 | JOB_GCS_BUCKET = credentials('gcs-bucket') 15 | BENCHMARK_SECRET = 'secret/apm-team/ci/benchmark-cloud' 16 | DOCKER_SECRET = 'secret/apm-team/ci/docker-registry/prod' 17 | DOCKER_REGISTRY = 'docker.elastic.co' 18 | } 19 | options { 20 | timeout(time: 1, unit: 'HOURS') 21 | buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) 22 | timestamps() 23 | ansiColor('xterm') 24 | disableResume() 25 | durabilityHint('PERFORMANCE_OPTIMIZED') 26 | } 27 | triggers { 28 | cron('H H(3-5) * * *') 29 | } 30 | parameters { 31 | string(name: 'GO_VERSION', defaultValue: '1.14', description: 'Go version to use.') 32 | string(name: 'STACK_VERSION', defaultValue: '8.0.0-SNAPSHOT', description: 'Stack version Git branch/tag to use.') 33 | string(name: 'APM_DOCKER_IMAGE', defaultValue: 'docker.elastic.co/apm/apm-server', description: 'The docker image to be used.') 34 | } 35 | stages { 36 | stage('Initializing'){ 37 | options { skipDefaultCheckout() } 38 | environment { 39 | PATH = "${env.PATH}:${env.WORKSPACE}/bin" 40 | HOME = "${env.WORKSPACE}" 41 | GOPATH = "${env.WORKSPACE}" 42 | } 43 | stages { 44 | /** 45 | Checkout the code and stash it, to use it on other stages. 46 | */ 47 | stage('Checkout') { 48 | steps { 49 | deleteDir() 50 | gitCheckout(basedir: env.BASE_DIR, repo: 'git@github.com:elastic/hey-apm.git', 51 | branch: 'main', credentialsId: env.JOB_GIT_CREDENTIALS) 52 | stash allowEmpty: true, name: 'source', useDefaultExcludes: false 53 | } 54 | } 55 | /** 56 | Unit tests. 57 | */ 58 | stage('Test') { 59 | steps { 60 | deleteDir() 61 | unstash 'source' 62 | dir("${BASE_DIR}"){ 63 | sh "./.ci/scripts/unit-test.sh ${GO_VERSION}" 64 | } 65 | } 66 | post { 67 | always { 68 | coverageReport("${BASE_DIR}/build/coverage") 69 | junit(allowEmptyResults: true, 70 | keepLongStdio: true, 71 | testResults: "${BASE_DIR}/build/*.xml") 72 | } 73 | } 74 | } 75 | /** 76 | APM server benchmark. 77 | */ 78 | stage('Benchmark') { 79 | agent { label 'metal' } 80 | steps { 81 | deleteDir() 82 | unstash 'source' 83 | dockerLogin(secret: env.DOCKER_SECRET, registry: env.DOCKER_REGISTRY) 84 | script { 85 | dir(BASE_DIR){ 86 | sendBenchmarks.prepareAndRun(secret: env.BENCHMARK_SECRET, url_var: 'ES_URL', 87 | user_var: 'ES_USER', pass_var: 'ES_PASS') { 88 | sh '.ci/scripts/run-bench-in-docker.sh' 89 | } 90 | } 91 | } 92 | } 93 | post { 94 | always { 95 | archiveArtifacts "${BASE_DIR}/build/environment.txt" 96 | deleteDir() 97 | } 98 | } 99 | } 100 | } 101 | } 102 | } 103 | post { 104 | always { 105 | notifyBuildResult() 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /.ci/scripts/run-bench-in-docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -eo pipefail 3 | set +x 4 | 5 | function finish { 6 | set +e 7 | mkdir -p build 8 | { 9 | docker-compose version 10 | docker system info 11 | docker ps -a 12 | docker-compose logs apm-server validate-es-url hey-apm 13 | docker inspect --format "{{json .State }}" apm-server validate-es-url hey-apm 14 | } > build/environment.txt 15 | # Ensure all the sensitive details are obfuscated 16 | sed -i.bck -e "s#${ES_USER}#********#g" -e "s#${ES_PASS}#********#g" -e "s#${ES_URL}#********#g" build/environment.txt 17 | rm build/*.bck || true 18 | docker-compose down -v 19 | # To avoid running twice the same function and therefore override the environment.txt file. 20 | trap - INT QUIT TERM EXIT 21 | set -e 22 | } 23 | 24 | trap finish EXIT INT TERM 25 | 26 | echo "Ensure the docker context is fresh" 27 | docker-compose down -v || true ## We don't want to fail yet 28 | 29 | echo "Validate whether the ES_URL is reachable" 30 | curl --user "${ES_USER}:${ES_PASS}" "${ES_URL}" 31 | 32 | echo "Report ES stack health" 33 | curl -s --user "${ES_USER}:${ES_PASS}" "${ES_URL}/_cluster/health" 34 | 35 | echo "Launch test" 36 | STACK_VERSION=${STACK_VERSION} \ 37 | ES_URL=${ES_URL} \ 38 | ES_USER=${ES_USER} \ 39 | ES_PASS=${ES_PASS} \ 40 | USER_ID="$(id -u):$(id -g)" docker-compose up \ 41 | --no-color \ 42 | --exit-code-from hey-apm \ 43 | --build \ 44 | --remove-orphans \ 45 | --abort-on-container-exit \ 46 | hey-apm 47 | -------------------------------------------------------------------------------- /.ci/scripts/run-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Stress testing the given Go version and APM server 3 | # 4 | # NOTE: It's required to be launched inside the root of the project. 5 | # 6 | # Usage: ./.ci/scripts/run-test.sh 1.14 /src/apm-server 7 | # 8 | # Requirements 9 | #  env variable CLOUD_ADDR 10 | #  env variable CLOUD_USERNAME 11 | #  env variable CLOUD_PASSWORD 12 | # 13 | set -euo pipefail 14 | 15 | RED='\033[31;49m' 16 | GREEN='\033[32;49m' 17 | NC='\033[0m' # No Color 18 | 19 | GO_VERSION=${1:?Please specify the Go version} 20 | APM_SERVER_DIR=${2:?Please specify the path pointing to the APM server source code} 21 | 22 | echo "Setup Go ${GO_VERSION}" 23 | GOPATH=$(pwd)/build 24 | export PATH=$PATH:$GOPATH/bin 25 | eval "$(gvm "${GO_VERSION}")" 26 | 27 | export COV_DIR="build/coverage" 28 | export COV_FILE="${COV_DIR}/hey-apm-stress-test.cov" 29 | export OUT_FILE="build/stress-test.out" 30 | mkdir -p "${COV_DIR}" 31 | 32 | echo "Installing hey-apm dependencies for Jenkins..." 33 | go get -v -u github.com/t-yuki/gocover-cobertura 34 | go get -v -u github.com/jstemmer/go-junit-report 35 | 36 | echo "Running apm-server stress tests..." 37 | (ELASTICSEARCH_URL=$CLOUD_ADDR \ 38 | ELASTICSEARCH_USR=$CLOUD_USERNAME \ 39 | ELASTICSEARCH_PWD=$CLOUD_PASSWORD \ 40 | go test -timeout 2h -v github.com/elastic/hey-apm/server/client 2>&1 | tee ${OUT_FILE}) \ 41 | && echo -e "${GREEN}Tests PASSED${NC}" || echo -e "${RED}Tests FAILED${NC}" 42 | 43 | go-junit-report < ${OUT_FILE} > build/junit-hey-apm-stress-test-report.xml 44 | -------------------------------------------------------------------------------- /.ci/scripts/unit-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Test the given Go version. 4 | # 5 | # NOTE: It's required to be launched inside the root of the project. 6 | # 7 | # Usage: ./.ci/scripts/unit-test.sh 1.14 8 | # 9 | set -exuo pipefail 10 | 11 | RED='\033[31;49m' 12 | GREEN='\033[32;49m' 13 | NC='\033[0m' # No Color 14 | 15 | GO_VERSION=${1:?Please specify the Go version} 16 | 17 | echo "Setup Go ${GO_VERSION}" 18 | GOPATH=$(pwd)/build 19 | export PATH=$PATH:$GOPATH/bin 20 | eval "$(gvm "${GO_VERSION}")" 21 | env | sort 22 | 23 | export COV_DIR="build/coverage" 24 | export COV_FILE="${COV_DIR}/hey-apm.cov" 25 | export OUT_FILE="build/test-report.out" 26 | mkdir -p "${COV_DIR}" 27 | 28 | echo "Installing hey-apm dependencies for Jenkins..." 29 | go get -v -u github.com/t-yuki/gocover-cobertura 30 | go get -v -u github.com/jstemmer/go-junit-report 31 | 32 | echo "Running unit tests..." 33 | (SKIP_EXTERNAL=1 SKIP_STRESS=1 go test -v ./... -coverprofile="${COV_FILE}" 2>&1 | tee ${OUT_FILE}) \ 34 | && echo -e "${GREEN}Tests PASSED${NC}" || echo -e "${RED}Tests FAILED${NC}" 35 | go-junit-report < ${OUT_FILE} > build/junit-hey-apm-report.xml 36 | 37 | echo "Running cobertura" 38 | go tool cover -html="${COV_FILE}" -o "${COV_DIR}/coverage-hey-apm-report.html" \ 39 | && echo -e "${GREEN}Tests PASSED${NC}" || echo -e "${RED}Tests FAILED${NC}" 40 | 41 | gocover-cobertura < "${COV_FILE}" > "${COV_DIR}/coverage-hey-apm-report.xml" 42 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | vendor/ 3 | hey-apm 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | hey-apm 3 | build 4 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018 Elasticsearch BV 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://apm-ci.elastic.co/buildStatus/icon?job=apm-server/apm-hey-test-mbp/main)](https://apm-ci.elastic.co/job/apm-server/job/apm-hey-test-mbp/job/main) 2 | 3 | # Overview 4 | 5 | hey-apm is a basic load generation tool for apm-server simulating different workloads. 6 | Back in the intake V1 days it was based on [hey](https://github.com/rakyll/hey), 7 | but now it uses the Go APM agent to generate events. 8 | 9 | hey-apm generates performance reports that can be indexed in Elasticsearch. 10 | It can be used manually or automatically (ie. in a CI environment) 11 | 12 | # Requirements 13 | 14 | hey-apm requires go modules support. Tested with go1.14.4. 15 | 16 | # Install 17 | 18 | ``` 19 | go get github.com/elastic/hey-apm 20 | ``` 21 | 22 | # Docker build 23 | 24 | ``` 25 | docker build -t hey-apm -f docker/Dockerfile . 26 | ``` 27 | 28 | ### Usage 29 | 30 | Run `./hey-apm -help` or see `main.go` 31 | 32 | # CI 33 | 34 | The `Jenkinsfile` triggers sequentially: 35 | 36 | - `.ci/scripts/unit-test.sh` 37 | - `.ci/scripts/run-bench-in-docker.sh` 38 | 39 | ## Requirements 40 | - [gvm](https://github.com/andrewkroh/gvm) 41 | 42 | ## Run scripts locally 43 | 44 | ```bash 45 | ./.ci/scripts/unit-test.sh 1.14 46 | ``` 47 | 48 | # How to run locally the hey-apm using a docker-compose services 49 | 50 | Run `.ci/scripts/run-bench-in-docker.sh` 51 | 52 | ## Configure the ES stack version 53 | 54 | Run `ELASTIC_STACK= .ci/scripts/run-bench-in-docker.sh` 55 | 56 | ## Configure the docker image for the ES stack 57 | 58 | Run `APM_DOCKER_IMAGE= .ci/scripts/run-bench-in-docker.sh` 59 | 60 | ## Configure the ES stack where to send the metrics to 61 | 62 | Run `ELASTIC_STACK= ES_URL= ES_USER= ES_PASS= .ci/scripts/run-bench-in-docker.sh` 63 | 64 | # Known issues 65 | 66 | * A single Go agent (as hey-apm uses) can't push enough load to overwhelm the apm-server, 67 | as it will drop data too conservatively for benchmarking purposes. 68 | -------------------------------------------------------------------------------- /benchmark/benchmark.go: -------------------------------------------------------------------------------- 1 | package benchmark 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "math" 9 | "time" 10 | 11 | "github.com/pkg/errors" 12 | 13 | "github.com/elastic/hey-apm/es" 14 | "github.com/elastic/hey-apm/models" 15 | "github.com/elastic/hey-apm/worker" 16 | ) 17 | 18 | const ( 19 | cool = time.Second * 60 20 | warm = time.Second * 60 21 | ) 22 | 23 | // Run executes a benchmark test with different workloads against a running apm-server, 24 | // and it checks that are no regressions by comparing it with previous benchmark results 25 | // executed with the same workload. 26 | // 27 | // Regression checks accept an error margin and are not aware of apm-server versions, only URLs. 28 | // apm-server must be started independently with -E apm-server.expvar.enabled=true 29 | func Run(ctx context.Context, input models.Input) error { 30 | conn, err := es.NewConnection(input.ElasticsearchUrl, input.ElasticsearchAuth) 31 | if err != nil { 32 | return errors.Wrap(err, "Elasticsearch not reachable, won't be able to index a report") 33 | } 34 | 35 | log.Printf("Deleting previous APM event documents...") 36 | deleted, err := es.DeleteAPMEvents(conn) 37 | if err != nil { 38 | return err 39 | } 40 | log.Printf("Deleted %d documents", deleted) 41 | 42 | if err := warmUp(ctx, input); err != nil { 43 | return err 44 | } 45 | 46 | tests := defineTests(input) 47 | reports, err := tests.run(ctx) 48 | if err != nil { 49 | return err 50 | } 51 | if err := verifyReports(reports, conn, input.RegressionMargin, input.RegressionDays); err != nil { 52 | return err 53 | } 54 | return nil 55 | } 56 | 57 | func defineTests(input models.Input) tests { 58 | var t tests 59 | t.add("transactions only", input.WithTransactions(math.MaxInt32, time.Millisecond*5)) 60 | t.add("small transactions", input.WithTransactions(math.MaxInt32, time.Millisecond*5).WithSpans(10)) 61 | t.add("large transactions", input.WithTransactions(math.MaxInt32, time.Millisecond*5).WithSpans(40)) 62 | t.add("small errors only", input.WithErrors(math.MaxInt32, time.Millisecond).WithFrames(10)) 63 | t.add("very large errors only", input.WithErrors(math.MaxInt32, time.Millisecond).WithFrames(500)) 64 | t.add("transactions only very high load", input.WithTransactions(math.MaxInt32, time.Microsecond*100)) 65 | t.add("transactions, spans and errors high load", input.WithTransactions(math.MaxInt32, time.Millisecond*5).WithSpans(10).WithErrors(math.MaxInt32, time.Millisecond).WithFrames(50)) 66 | return t 67 | } 68 | 69 | type test struct { 70 | name string 71 | input models.Input 72 | } 73 | 74 | type tests []test 75 | 76 | func (t *tests) add(name string, input models.Input) { 77 | *t = append(*t, test{name: name, input: input}) 78 | } 79 | 80 | func (t *tests) run(ctx context.Context) ([]models.Report, error) { 81 | reports := make([]models.Report, len(*t)) 82 | for i, test := range *t { 83 | log.Printf("running benchmark %q", test.name) 84 | report, err := worker.Run(ctx, test.input, test.name, nil /*stop*/) 85 | if err != nil { 86 | return nil, err 87 | } 88 | if err := coolDown(ctx); err != nil { 89 | return nil, err 90 | } 91 | reports[i] = report 92 | } 93 | return reports, nil 94 | } 95 | 96 | func verifyReports(reports []models.Report, conn es.Connection, margin float64, days string) error { 97 | var lastErr error 98 | for _, report := range reports { 99 | if err := verify(conn, report, margin, days); err != nil { 100 | fmt.Println(err) 101 | lastErr = err 102 | } 103 | } 104 | return lastErr 105 | } 106 | 107 | // warmUp sends a moderate load to apm-server without saving a report. 108 | func warmUp(ctx context.Context, input models.Input) error { 109 | input = input.WithErrors(math.MaxInt16, time.Millisecond) 110 | input.RunTimeout = warm 111 | input.SkipIndexReport = true 112 | log.Printf("warming up %.1f seconds...", warm.Seconds()) 113 | if _, err := worker.Run(ctx, input, "warm up", nil); err != nil { 114 | return err 115 | } 116 | return coolDown(ctx) 117 | } 118 | 119 | // coolDown waits an arbitrary time for events in elasticsearch be flushed, heap be freed, etc. 120 | func coolDown(ctx context.Context) error { 121 | log.Printf("cooling down %.1f seconds... ", cool.Seconds()) 122 | timer := time.NewTimer(cool) 123 | defer timer.Stop() 124 | select { 125 | case <-ctx.Done(): 126 | return ctx.Err() 127 | case <-timer.C: 128 | return nil 129 | } 130 | } 131 | 132 | // verify asserts there are no performance regressions for a given workload. 133 | // 134 | // compares the given report with saved reports with the same input indexed in the last specified days 135 | // returns an error if connection can't be established, 136 | // or performance decreased by a margin larger than specified 137 | func verify(conn es.Connection, report models.Report, margin float64, days string) error { 138 | if report.EventsIndexed < 100 { 139 | return fmt.Errorf("not enough events indexed: %d", report.EventsIndexed) 140 | } 141 | 142 | filters := []map[string]interface{}{{ 143 | "range": map[string]interface{}{ 144 | "@timestamp": map[string]interface{}{ 145 | "gte": fmt.Sprintf("now-%sd/d", days), 146 | "lt": "now", 147 | }, 148 | }, 149 | }} 150 | 151 | // Convert input to a JSON map, to filter on the previous results for matching inputs. 152 | inputMap := make(map[string]interface{}) 153 | encodedInput, err := json.Marshal(report.Input) 154 | if err != nil { 155 | return err 156 | } 157 | if err := json.Unmarshal(encodedInput, &inputMap); err != nil { 158 | return err 159 | } 160 | for k, v := range inputMap { 161 | filters = append(filters, map[string]interface{}{ 162 | "match": map[string]interface{}{k: v}, 163 | }) 164 | } 165 | 166 | savedReports, fetchErr := es.FetchReports(conn, map[string]interface{}{ 167 | "query": map[string]interface{}{ 168 | "bool": map[string]interface{}{ 169 | "must": filters, 170 | }, 171 | }, 172 | }) 173 | if fetchErr != nil { 174 | return fetchErr 175 | } 176 | 177 | var accPerformance, count float64 178 | for _, sr := range savedReports { 179 | if report.ReportId != sr.ReportId { 180 | accPerformance += sr.Performance() 181 | count += 1.0 182 | } 183 | } 184 | avgPerformance := accPerformance / count 185 | if avgPerformance > report.Performance()*margin { 186 | return newRegression(report, avgPerformance) 187 | } 188 | return nil 189 | } 190 | 191 | func newRegression(r models.Report, threshold float64) error { 192 | return errors.New(fmt.Sprintf( 193 | `test report with doc id %s was expected to show same or better performance than average of %.2f, however %.2f is significantly lower`, 194 | r.ReportId, threshold, r.Performance(), 195 | )) 196 | } 197 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2.1" 2 | services: 3 | apm-server: 4 | image: ${APM_DOCKER_IMAGE:-docker.elastic.co/apm/apm-server}:${STACK_VERSION:-8.0.0-SNAPSHOT} 5 | ports: 6 | - "127.0.0.1:8201:8200" 7 | - "127.0.0.1:6061:6060" 8 | command: > 9 | apm-server -e 10 | -E monitoring.enabled=true 11 | -E apm-server.expvar.enabled=true 12 | -E apm-server.instrumentation.enabled=true 13 | -E output.elasticsearch.hosts=["${ES_URL}"] 14 | -E output.elasticsearch.username=${ES_USER} 15 | -E output.elasticsearch.password=${ES_PASS} 16 | environment: 17 | - ES_URL=${ES_URL} 18 | - ES_USER=${ES_USER} 19 | - ES_PASS=${ES_PASS} 20 | cap_drop: 21 | - ALL 22 | cap_add: 23 | - CHOWN 24 | - DAC_OVERRIDE 25 | - SETGID 26 | - SETUID 27 | logging: 28 | driver: 'json-file' 29 | options: 30 | max-size: '2m' 31 | max-file: '5' 32 | healthcheck: 33 | test: ["CMD", "curl", "--write-out", "'HTTP %{http_code}'", "--silent", "--output", "/dev/null", "http://apm-server:8200/"] 34 | retries: 10 35 | interval: 10s 36 | 37 | hey-apm: 38 | build: 39 | context: ./ 40 | dockerfile: docker/Dockerfile-bench 41 | working_dir: /app 42 | command: > 43 | /hey-apm -bench -run 5m -rm 1.2 -apm-url http://apm-server:8200 -es-url ${ES_URL} -es-auth "${ES_USER}:${ES_PASS}" -apm-es-url ${ES_URL} -apm-es-auth "${ES_USER}:${ES_PASS}" 44 | environment: 45 | - ES_URL=${ES_URL} 46 | - ES_USER=${ES_USER} 47 | - ES_PASS=${ES_PASS} 48 | volumes: 49 | - ${PWD}:/app 50 | user: ${USER_ID} 51 | mem_limit: 200m 52 | depends_on: 53 | apm-server: 54 | condition: service_healthy 55 | validate-es-url: 56 | condition: service_started 57 | 58 | validate-es-url: 59 | image: pstauffer/curl 60 | command: curl --user "${ES_USER}:${ES_PASS}" "${ES_URL}" 61 | environment: 62 | - ES_URL=${ES_URL} 63 | - ES_USER=${ES_USER} 64 | - ES_PASS=${ES_PASS} 65 | 66 | volumes: 67 | esdata: 68 | driver: local 69 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # from .. run: docker build -t hey-apm -f docker/Dockerfile . 2 | FROM golang:1.14 3 | RUN useradd hey 4 | 5 | WORKDIR /build 6 | COPY go.* ./ 7 | RUN go mod download 8 | COPY . . 9 | RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o hey-apm . 10 | 11 | FROM scratch 12 | MAINTAINER Elastic APM Team 13 | 14 | # https://github.com/golang/go/blob/release-branch.go1.14/src/crypto/x509/root_linux.go 15 | COPY --from=0 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 16 | COPY --from=0 /etc/passwd /etc/passwd 17 | COPY --from=0 /build/hey-apm /hey-apm 18 | 19 | USER hey 20 | ENTRYPOINT ["/hey-apm"] 21 | CMD ["-apm-url", "http://apm-server:8200", "-es-url", "http://elasticsearch:9200"] 22 | -------------------------------------------------------------------------------- /docker/Dockerfile-bench: -------------------------------------------------------------------------------- 1 | # from .. run: docker build -t hey-apm -f docker/Dockerfile . 2 | FROM golang:1.14 3 | RUN useradd hey 4 | 5 | WORKDIR /build 6 | COPY go.* ./ 7 | RUN go mod download 8 | COPY . . 9 | RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o hey-apm . 10 | 11 | FROM scratch 12 | MAINTAINER Elastic APM Team 13 | 14 | # https://github.com/golang/go/blob/release-branch.go1.14/src/crypto/x509/root_linux.go 15 | COPY --from=0 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 16 | COPY --from=0 /etc/passwd /etc/passwd 17 | COPY --from=0 /build/hey-apm /hey-apm 18 | 19 | USER hey 20 | -------------------------------------------------------------------------------- /es/api.go: -------------------------------------------------------------------------------- 1 | package es 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/elastic/go-elasticsearch/v7" 9 | "github.com/elastic/go-elasticsearch/v7/esutil" 10 | "github.com/elastic/hey-apm/models" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | const ( 15 | reportingIndex = "hey-bench" 16 | local = "http://localhost:9200" 17 | ) 18 | 19 | // Connection holds an elasticsearch client plus URL and credentials strings 20 | type Connection struct { 21 | *elasticsearch.Client 22 | Url string 23 | username string 24 | password string 25 | } 26 | 27 | // NewConnection returns a client connected to an ElasticSearch node with given `params` 28 | // "local" is short for http://localhost:9200 29 | func NewConnection(url, auth string) (Connection, error) { 30 | if url == "local" { 31 | url = local 32 | } 33 | 34 | // Split "username:password" 35 | // 36 | // TODO(axw) consider removing the separate "auth" option to 37 | // reduce options, and instead require userinfo to be included 38 | // in the URL. 39 | username, password := auth, "" 40 | if sep := strings.IndexRune(auth, ':'); sep >= 0 { 41 | username, password = auth[:sep], auth[sep+1:] 42 | } 43 | 44 | cfg := elasticsearch.Config{ 45 | Addresses: []string{url}, 46 | Username: username, 47 | Password: password, 48 | } 49 | 50 | client, err := elasticsearch.NewClient(cfg) 51 | return Connection{client, url, username, password}, err 52 | } 53 | 54 | // IndexReport saves in elasticsearch a performance report. 55 | func IndexReport(conn Connection, report models.Report) error { 56 | resp, err := conn.Index(reportingIndex, esutil.NewJSONReader(report), 57 | conn.Index.WithRefresh("true"), 58 | conn.Index.WithDocumentID(report.ReportId), 59 | ) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | if resp.IsError() { 65 | return errors.New(resp.String()) 66 | } 67 | return nil 68 | } 69 | 70 | // FetchReports retrieves performance reports from elasticsearch. 71 | func FetchReports(conn Connection, body interface{}) ([]models.Report, error) { 72 | resp, err := conn.Search( 73 | conn.Search.WithIndex(reportingIndex), 74 | conn.Search.WithSort("@timestamp:desc"), 75 | conn.Search.WithBody(esutil.NewJSONReader(body)), 76 | ) 77 | if err != nil { 78 | return nil, err 79 | } 80 | 81 | if resp.IsError() { 82 | return nil, errors.New(resp.String()) 83 | } 84 | 85 | parsed := SearchResult{} 86 | err = json.NewDecoder(resp.Body).Decode(&parsed) 87 | 88 | ret := make([]models.Report, len(parsed.Hits.Hits)) 89 | 90 | for idx, hit := range parsed.Hits.Hits { 91 | hit.Source.ReportId = hit.Id 92 | ret[idx] = hit.Source 93 | } 94 | 95 | return ret, err 96 | } 97 | 98 | // Count returns the number of documents in the given index, excluding 99 | // those related to self-instrumentation. 100 | func Count(conn Connection, index, eventType string) uint64 { 101 | res, err := conn.Count( 102 | conn.Count.WithIndex(index), 103 | conn.Count.WithBody(strings.NewReader(fmt.Sprintf(` 104 | { 105 | "query": { 106 | "bool": { 107 | "filter": { 108 | "term": { 109 | "processor.event": "%s" 110 | } 111 | }, 112 | "must_not": { 113 | "term": { 114 | "service.name": { 115 | "value": "apm-server" 116 | } 117 | } 118 | } 119 | } 120 | } 121 | }`[1:], eventType))), 122 | ) 123 | if err != nil { 124 | return 0 125 | } 126 | var m map[string]interface{} 127 | json.NewDecoder(res.Body).Decode(&m) 128 | if ct, ok := m["count"]; ok && ct != nil { 129 | return uint64(m["count"].(float64)) 130 | } 131 | return 0 132 | } 133 | 134 | func DeleteAPMEvents(conn Connection) (int, error) { 135 | body := map[string]interface{}{ 136 | "query": map[string]interface{}{ 137 | "bool": map[string]interface{}{ 138 | "must_not": []map[string]interface{}{ 139 | { 140 | "term": map[string]interface{}{ 141 | "service.name": map[string]interface{}{ 142 | "value": "apm-server", 143 | }, 144 | }, 145 | }, 146 | }, 147 | }, 148 | }, 149 | } 150 | resp, err := conn.DeleteByQuery( 151 | []string{".ds-traces-apm*", ".ds-metrics-apm*", ".ds-logs-apm*"}, 152 | esutil.NewJSONReader(body), 153 | conn.DeleteByQuery.WithExpandWildcards("all"), 154 | ) 155 | if err != nil { 156 | return -1, err 157 | } 158 | defer resp.Body.Close() 159 | 160 | if resp.IsError() { 161 | return -1, errors.New(fmt.Sprintf("%s: %s", resp.Status(), resp.String())) 162 | } 163 | 164 | var result struct { 165 | Deleted int `json:"deleted"` 166 | } 167 | if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { 168 | return -1, err 169 | } 170 | return result.Deleted, nil 171 | } 172 | -------------------------------------------------------------------------------- /es/es.go: -------------------------------------------------------------------------------- 1 | package es 2 | 3 | import "github.com/elastic/hey-apm/models" 4 | 5 | type SearchResult struct { 6 | Hits Hits `json:"hits"` 7 | } 8 | 9 | type Hits struct { 10 | Hits []ActualHit `json:"hits"` 11 | } 12 | 13 | type ActualHit struct { 14 | Id string `json:"_id"` 15 | Source models.Report `json:"_source"` 16 | } 17 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/elastic/hey-apm 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/dustin/go-humanize v1.0.0 7 | github.com/elastic/go-elasticsearch/v7 v7.8.0 8 | github.com/elastic/go-sysinfo v1.4.0 // indirect 9 | github.com/elastic/go-windows v1.0.1 // indirect 10 | github.com/pkg/errors v0.9.1 11 | github.com/prometheus/procfs v0.1.3 // indirect 12 | github.com/stretchr/testify v1.4.0 13 | go.elastic.co/apm v1.8.1-0.20200904000055-489947bc48c1 14 | go.elastic.co/fastjson v1.1.0 // indirect 15 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e 16 | golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed // indirect 17 | howett.net/plist v0.0.0-20200419221736-3b63eb3a43b5 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= 2 | github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 3 | github.com/cucumber/godog v0.8.1 h1:lVb+X41I4YDreE+ibZ50bdXmySxgRviYFgKY6Aw4XE8= 4 | github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAElF6hxnA= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= 9 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 10 | github.com/elastic/go-elasticsearch/v7 v7.8.0 h1:M9D55OK13IEgg51Jb57mZgseag1AsncwAUn4C6j1vlc= 11 | github.com/elastic/go-elasticsearch/v7 v7.8.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= 12 | github.com/elastic/go-sysinfo v1.1.1 h1:ZVlaLDyhVkDfjwPGU55CQRCRolNpc7P0BbyhhQZQmMI= 13 | github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= 14 | github.com/elastic/go-sysinfo v1.3.0 h1:eb2XFGTMlSwG/yyU9Y8jVAYLIzU2sFzWXwo2gmetyrE= 15 | github.com/elastic/go-sysinfo v1.3.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= 16 | github.com/elastic/go-sysinfo v1.4.0 h1:LUnK6TNOuy8JEByuDzTAQH3iQ6bIywy55+Z+QlKNSWk= 17 | github.com/elastic/go-sysinfo v1.4.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= 18 | github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= 19 | github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= 20 | github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0= 21 | github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= 22 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 23 | github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= 24 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 25 | github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 26 | github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= 27 | github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= 28 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 29 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 30 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 31 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 32 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 33 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 34 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 35 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 36 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 37 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 38 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 39 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 40 | github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 41 | github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE= 42 | github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= 43 | github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= 44 | github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 45 | github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= 46 | github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 47 | github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis= 48 | github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4= 49 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 50 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 51 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 52 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 53 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 54 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 55 | go.elastic.co/apm v1.8.0 h1:AWEKpHwRal0yCMd4K8Oxy1HAa7xid+xq1yy+XjgoVU0= 56 | go.elastic.co/apm v1.8.0/go.mod h1:tCw6CkOJgkWnzEthFN9HUP1uL3Gjc/Ur6m7gRPLaoH0= 57 | go.elastic.co/apm v1.8.1-0.20200904000055-489947bc48c1 h1:NuH+QfytaWQJwm/N0U8b0Zb2q9Rqfd2j2qSY26ZJYOc= 58 | go.elastic.co/apm v1.8.1-0.20200904000055-489947bc48c1/go.mod h1:qoOSi09pnzJDh5fKnfY7bPmQgl8yl2tULdOu03xhui0= 59 | go.elastic.co/fastjson v1.0.0 h1:ooXV/ABvf+tBul26jcVViPT3sBir0PvXgibYB1IQQzg= 60 | go.elastic.co/fastjson v1.0.0/go.mod h1:PmeUOMMtLHQr9ZS9J9owrAVg0FkaZDRZJEFTTGHtchs= 61 | go.elastic.co/fastjson v1.1.0 h1:3MrGBWWVIxe/xvsbpghtkFoPciPhOCmjsR/HfwEeQR4= 62 | go.elastic.co/fastjson v1.1.0/go.mod h1:boNGISWMjQsUPy/t6yqt2/1Wx4YNPSe+mZjlyw9vKKI= 63 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 64 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 65 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 66 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 67 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 68 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 69 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 70 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 71 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= 72 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 73 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 74 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 75 | golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 76 | golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 77 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e h1:9vRrk9YW2BTzLP0VCB9ZDjU4cPqkg+IDWL7XgxA1yxQ= 78 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 79 | golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 80 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= 81 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 82 | golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed h1:J22ig1FUekjjkmZUM7pTKixYm8DvrYsvrBZdunYeIuQ= 83 | golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 84 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 85 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 86 | golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 87 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 88 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 89 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 90 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 91 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 92 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 93 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 94 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 95 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 96 | howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= 97 | howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= 98 | howett.net/plist v0.0.0-20200225050739-77e249a2e2ba h1:HiEs/6jQFMHpFqsdPBAk3ieVcsSS8IV+D93f43UuDPo= 99 | howett.net/plist v0.0.0-20200225050739-77e249a2e2ba/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= 100 | howett.net/plist v0.0.0-20200419221736-3b63eb3a43b5 h1:AQkaJpH+/FmqRjmXZPELom5zIERYZfwTjnHpfoVMQEc= 101 | howett.net/plist v0.0.0-20200419221736-3b63eb3a43b5/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= 102 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "math" 9 | "math/rand" 10 | "os" 11 | "os/signal" 12 | "strconv" 13 | "time" 14 | 15 | "go.elastic.co/apm" 16 | "golang.org/x/sync/errgroup" 17 | 18 | "github.com/elastic/hey-apm/benchmark" 19 | "github.com/elastic/hey-apm/models" 20 | "github.com/elastic/hey-apm/worker" 21 | ) 22 | 23 | func init() { 24 | apm.DefaultTracer.Close() 25 | rand.Seed(1000) 26 | } 27 | 28 | func main() { 29 | if err := Main(); err != nil { 30 | log.Fatal(err) 31 | } 32 | } 33 | 34 | func Main() error { 35 | signalC := make(chan os.Signal, 1) 36 | signal.Notify(signalC, os.Interrupt) 37 | input := parseFlags() 38 | if input.IsBenchmark { 39 | ctx, cancel := context.WithCancel(context.Background()) 40 | defer cancel() 41 | go func() { 42 | // Ctrl+C when running benchmarks causes them to be 43 | // aborted, as the results are not meaningful for 44 | // comparison. 45 | defer cancel() 46 | <-signalC 47 | log.Printf("Interrupt signal received, aborting benchmarks...") 48 | }() 49 | if err := benchmark.Run(ctx, input); err != nil { 50 | return err 51 | } 52 | return nil 53 | } 54 | 55 | stopChan := make(chan struct{}) 56 | go func() { 57 | // Ctrl+C when running load generation gracefully stops the 58 | // workers and prints the statistics. 59 | defer close(stopChan) 60 | <-signalC 61 | log.Printf("Interrupt signal received, stopping load generator...") 62 | }() 63 | return runWorkers(input, stopChan) 64 | } 65 | 66 | func runWorkers(input models.Input, stop <-chan struct{}) error { 67 | g, ctx := errgroup.WithContext(context.Background()) 68 | for i := 0; i < input.Instances; i++ { 69 | idx := i 70 | g.Go(func() error { 71 | randomDelay := time.Duration(rand.Intn(input.DelayMillis)) * time.Millisecond 72 | fmt.Println(fmt.Sprintf("--- Starting instance (%v) in %v milliseconds", idx, randomDelay)) 73 | time.Sleep(randomDelay) 74 | _, err := worker.Run(ctx, input, "", stop) 75 | return err 76 | }) 77 | } 78 | return g.Wait() 79 | } 80 | 81 | func parseFlags() models.Input { 82 | // run options 83 | runTimeout := flag.Duration("run", 30*time.Second, "stop run after this duration") 84 | flushTimeout := flag.Duration("flush", 10*time.Second, "wait timeout for agent flush") 85 | seed := flag.Int64("seed", time.Now().Unix(), "random seed") 86 | instances := flag.Int("instances", 1, "number of concurrent instances to create load (only if -bench is not passed)") 87 | delayMillis := flag.Int("delay", 1000, "max delay in milliseconds per worker to start (only if -bench is not passed)") 88 | 89 | // convenience for https://www.elastic.co/guide/en/apm/agent/go/current/configuration.html 90 | serviceName := os.Getenv("ELASTIC_APM_SERVICE_NAME") 91 | if serviceName == "" { 92 | serviceName = *flag.String("service-name", "hey-service", "service name") // ELASTIC_APM_SERVICE_NAME 93 | } 94 | // apm-server options 95 | apmServerSecret := flag.String("apm-secret", "", "apm server secret token") // ELASTIC_APM_SECRET_TOKEN 96 | apmServerAPIKey := flag.String("api-key", "", "APM API yey") 97 | apmServerUrl := flag.String("apm-url", "http://localhost:8200", "apm server url") // ELASTIC_APM_SERVER_URL 98 | 99 | elasticsearchUrl := flag.String("es-url", "http://localhost:9200", "elasticsearch url for reporting") 100 | elasticsearchAuth := flag.String("es-auth", "", "elasticsearch username:password reporting") 101 | 102 | apmElasticsearchUrl := flag.String("apm-es-url", "http://localhost:9200", "elasticsearch output host for apm-server under load") 103 | apmElasticsearchAuth := flag.String("apm-es-auth", "", "elasticsearch output username:password for apm-server under load") 104 | 105 | isBench := flag.Bool("bench", false, "execute a benchmark with fixed parameters") 106 | regressionMargin := flag.Float64("rm", 1.1, "margin of acceptable performance decrease to not consider a regression (only in combination with -bench)") 107 | regressionDays := flag.String("rd", "7", "number of days back to check for regressions (only in combination with -bench)") 108 | 109 | // payload options 110 | errorLimit := flag.Int("e", math.MaxInt64, "max errors to generate (only if -bench is not passed)") 111 | errorFrequency := flag.Duration("ef", 1*time.Nanosecond, "error frequency. "+ 112 | "generate errors up to once in this duration (only if -bench is not passed)") 113 | errorFrameMaxLimit := flag.Int("ex", 10, "max error frames to per error (only if -bench is not passed)") 114 | errorFrameMinLimit := flag.Int("em", 0, "max error frames to per error (only if -bench is not passed)") 115 | spanMaxLimit := flag.Int("sx", 10, "max spans to per transaction (only if -bench is not passed)") 116 | spanMinLimit := flag.Int("sm", 1, "min spans to per transaction (only if -bench is not passed)") 117 | transactionLimit := flag.Int("t", math.MaxInt64, "max transactions to generate (only if -bench is not passed)") 118 | transactionFrequency := flag.Duration("tf", 1*time.Nanosecond, "transaction frequency. "+ 119 | "generate transactions up to once in this duration (only if -bench is not passed)") 120 | flag.Parse() 121 | 122 | if *spanMaxLimit < *spanMinLimit { 123 | spanMaxLimit = spanMinLimit 124 | } 125 | 126 | rand.Seed(*seed) 127 | 128 | input := models.Input{ 129 | IsBenchmark: *isBench, 130 | ApmServerUrl: *apmServerUrl, 131 | ApmServerSecret: *apmServerSecret, 132 | APIKey: *apmServerAPIKey, 133 | ElasticsearchUrl: *elasticsearchUrl, 134 | ElasticsearchAuth: *elasticsearchAuth, 135 | ApmElasticsearchUrl: *apmElasticsearchUrl, 136 | ApmElasticsearchAuth: *apmElasticsearchAuth, 137 | ServiceName: serviceName, 138 | RunTimeout: *runTimeout, 139 | FlushTimeout: *flushTimeout, 140 | Instances: *instances, 141 | DelayMillis: *delayMillis, 142 | } 143 | 144 | if *isBench { 145 | if _, err := strconv.Atoi(*regressionDays); err != nil { 146 | panic(err) 147 | } 148 | input.RegressionDays = *regressionDays 149 | input.RegressionMargin = *regressionMargin 150 | return input 151 | } 152 | 153 | input.TransactionFrequency = *transactionFrequency 154 | input.TransactionLimit = *transactionLimit 155 | input.SpanMaxLimit = *spanMaxLimit 156 | input.SpanMinLimit = *spanMinLimit 157 | input.ErrorFrequency = *errorFrequency 158 | input.ErrorLimit = *errorLimit 159 | input.ErrorFrameMaxLimit = *errorFrameMaxLimit 160 | input.ErrorFrameMinLimit = *errorFrameMinLimit 161 | 162 | return input 163 | } 164 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "reflect" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | // All JSON fields in Input are required, with zero values being meaningful. 15 | func TestDefaultInput(t *testing.T) { 16 | expectedZeroValues := []string{ 17 | "transaction_generation_limit", 18 | "transaction_generation_frequency", 19 | "spans_generated_max_limit", 20 | "spans_generated_min_limit", 21 | "error_generation_limit", 22 | "error_generation_frequency", 23 | "error_generation_frames_max_limit", 24 | "error_generation_frames_min_limit", 25 | } 26 | expectedZeroValuesMap := make(map[string]bool) 27 | for _, k := range expectedZeroValues { 28 | expectedZeroValuesMap[k] = true 29 | } 30 | 31 | os.Args[1] = "-bench" 32 | input := parseFlags() 33 | assert.True(t, input.IsBenchmark) 34 | 35 | encodedInput, err := json.Marshal(input) 36 | require.NoError(t, err) 37 | inputMap := make(map[string]interface{}) 38 | require.NoError(t, json.Unmarshal(encodedInput, &inputMap)) 39 | 40 | for k, v := range inputMap { 41 | if expectedZeroValuesMap[k] { 42 | continue 43 | } 44 | // any zero values not in `expectedZeroValues` should have been omitted 45 | r := reflect.ValueOf(v) 46 | assert.False(t, r.IsZero(), fmt.Sprintf("field %s has zero value %v", k, v)) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /models/input.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Input holds all the parameters given to a load test work. 8 | // Most parameters describe a workload pattern, others are required to create performance reports. 9 | type Input struct { 10 | 11 | // Whether or not this object will be processed by the `benchmark` package 12 | IsBenchmark bool `json:"-"` 13 | // Number of days to look back for regressions (only if IsBenchmark is true) 14 | RegressionDays string `json:"-"` 15 | // Acceptable performance decrease without being considered as regressions, as a percentage 16 | // (only if IsBenchmark is true) 17 | RegressionMargin float64 `json:"-"` 18 | 19 | // URL of the APM Server under test 20 | ApmServerUrl string `json:"apm_url"` 21 | // Secret token of the APM Server under test 22 | ApmServerSecret string `json:"-"` 23 | // API Key for communication between APM Server and the Go agent 24 | APIKey string `json:"-"` 25 | // If true, it will index the performance report of a run in ElasticSearch 26 | SkipIndexReport bool `json:"-"` 27 | // URL of the Elasticsearch instance used for indexing the performance report 28 | ElasticsearchUrl string `json:"-"` 29 | // of the Elasticsearch instance used for indexing the performance report 30 | ElasticsearchAuth string `json:"-"` 31 | // URL of the Elasticsearch instance used by APM Server 32 | ApmElasticsearchUrl string `json:"elastic_url,omitempty"` 33 | // of the Elasticsearch instance used by APM Server 34 | ApmElasticsearchAuth string `json:"-"` 35 | // Service name passed to the tracer 36 | ServiceName string `json:"service_name,omitempty"` 37 | 38 | // Run timeout of the performance test (ends the test when reached) 39 | RunTimeout time.Duration `json:"run_timeout"` 40 | // Timeout for flushing the workload to APM Server 41 | FlushTimeout time.Duration `json:"flush_timeout"` 42 | // Number of Instances that are creating load 43 | Instances int `json:"instances"` 44 | // DelayMillis is the maximum amount of milliseconds to wait per instance before starting it, 45 | // can be used to add some randomness for producing load 46 | DelayMillis int `json:"delay_millis"` 47 | // Frequency at which the tracer will generate transactions 48 | TransactionFrequency time.Duration `json:"transaction_generation_frequency"` 49 | // Maximum number of transactions to push to the APM Server (ends the test when reached) 50 | TransactionLimit int `json:"transaction_generation_limit"` 51 | // Maximum number of spans per transaction 52 | SpanMaxLimit int `json:"spans_generated_max_limit"` 53 | // Minimum number of spans per transaction 54 | SpanMinLimit int `json:"spans_generated_min_limit"` 55 | // Frequency at which the tracer will generate errors 56 | ErrorFrequency time.Duration `json:"error_generation_frequency"` 57 | // Maximum number of errors to push to the APM Server (ends the test when reached) 58 | ErrorLimit int `json:"error_generation_limit"` 59 | // Maximum number of stacktrace frames per error 60 | ErrorFrameMaxLimit int `json:"error_generation_frames_max_limit"` 61 | // Minimum number of stacktrace frames per error 62 | ErrorFrameMinLimit int `json:"error_generation_frames_min_limit"` 63 | } 64 | 65 | func (in Input) WithErrors(limit int, freq time.Duration) Input { 66 | in.ErrorLimit = limit 67 | in.ErrorFrequency = freq 68 | return in 69 | } 70 | 71 | func (in Input) WithFrames(f int) Input { 72 | in.ErrorFrameMaxLimit = f 73 | in.ErrorFrameMinLimit = f 74 | return in 75 | } 76 | 77 | func (in Input) WithTransactions(limit int, freq time.Duration) Input { 78 | in.TransactionLimit = limit 79 | in.TransactionFrequency = freq 80 | return in 81 | } 82 | 83 | func (in Input) WithSpans(s int) Input { 84 | in.SpanMaxLimit = s 85 | in.SpanMinLimit = s 86 | return in 87 | } 88 | -------------------------------------------------------------------------------- /models/report.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | const GITRFC = "Mon, 2 Jan 2006 15:04:05 -0700" 8 | 9 | // Report holds performance statistics generated by a load test work. 10 | type Report struct { 11 | 12 | // Input arguments to the load test work 13 | Input 14 | 15 | // Elasticsearch doc id 16 | ReportId string `json:"report_id"` 17 | // see GITRFC 18 | ReportDate string `json:"report_date"` 19 | // hey-apm host 20 | ReporterHost string `json:"reporter_host"` 21 | // like reportDate, but better for querying ES and sorting 22 | Timestamp time.Time `json:"@timestamp"` 23 | // any arbitrary strings set by the user, meant to filter results 24 | Labels []string `json:"labels,omitempty"` 25 | // name of the test run 26 | TestName string `json:"test_name,omitempty"` 27 | // apm-server release version or build sha 28 | ApmVersion string `json:"apm_version,omitempty"` 29 | // commit SHA 30 | ApmBuild string `json:"apm_build,omitempty"` 31 | // commit date 32 | ApmBuildDate time.Time `json:"apm_build_date,omitempty"` 33 | // list of settings apm-server has been started with 34 | // some are explicitly omitted (eg passwords) 35 | // only captured options passed with -E when expvar is enabled 36 | ApmSettings map[string]string `json:"apm_settings,omitempty"` 37 | 38 | // total elapsed (timeout + flush) 39 | Elapsed float64 `json:"elapsed"` 40 | 41 | // number of total requests to apm-server 42 | Requests uint64 `json:"requests"` 43 | // number of total failed requests 44 | FailedRequests uint64 `json:"failed_requests"` 45 | 46 | // TODO 47 | // total number of responses 48 | // Responses uint64 `json:"responses"` 49 | // total number of responses 50 | // Responses202 uint64 `json:"responses_202"` 51 | // total number of responses 52 | // Responses4XX uint64 `json:"responses_4xx"` 53 | // total number of responses 54 | // Responses5XX uint64 `json:"responses_5xx"` 55 | // 202 / total 56 | // ResponseSuccessRatio *float64 `json:"response_success_ratio"` 57 | // 58 | // number of stacktrace frames per error 59 | // ErrorFrames int `json:"error_frames"` 60 | 61 | // number of errors generated 62 | ErrorsGenerated uint64 `json:"errors_generated"` 63 | // number of errors sent to apm-server 64 | ErrorsSent uint64 `json:"errors_sent"` 65 | // number of errors indexed in Elasticsearch 66 | ErrorsIndexed uint64 `json:"errors_indexed"` 67 | 68 | // number of transactions generated (as per user input) 69 | TransactionsGenerated uint64 `json:"transactions_generated"` 70 | // number of transactions sent to apm-server 71 | TransactionsSent uint64 `json:"transactions_sent"` 72 | // number of transactions indexed in Elasticsearch 73 | TransactionsIndexed uint64 `json:"transactions_indexed"` 74 | // TODO 75 | // number of stacktrace frames per span 76 | // SpanFrames int `json:"span_frames"` 77 | // number of generated spans 78 | SpansGenerated uint64 `json:"spans_generated"` 79 | // number of spans sent to apm-server 80 | SpansSent uint64 `json:"spans_sent"` 81 | // number of spans indexed in Elasticsearch 82 | SpansIndexed uint64 `json:"spans_indexed"` 83 | 84 | // total generated 85 | EventsGenerated uint64 `json:"events_generated"` 86 | // total sent 87 | EventsSent uint64 `json:"events_sent"` 88 | // total accepted 89 | EventsAccepted uint64 `json:"events_accepted"` 90 | // total indexed 91 | EventsIndexed uint64 `json:"events_indexed"` 92 | 93 | // total memory allocated in bytes 94 | TotalAlloc *uint64 `json:"total_alloc,omitempty"` 95 | // total memory allocated in the heap, in bytes 96 | HeapAlloc *uint64 `json:"heap_alloc,omitempty"` 97 | // total number of mallocs 98 | Mallocs *uint64 `json:"mallocs,omitempty"` 99 | // number of GC runs 100 | NumGC *uint64 `json:"num_gc,omitempty"` 101 | } 102 | 103 | func (r Report) date() time.Time { 104 | t, _ := time.Parse(GITRFC, r.ReportDate) 105 | return t 106 | } 107 | 108 | func (r Report) Performance() float64 { 109 | if r.Elapsed > 0 { 110 | return float64(r.EventsIndexed) / r.Elapsed 111 | } 112 | return 0 113 | } 114 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | errs "errors" 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | "net/url" 12 | "strings" 13 | "text/tabwriter" 14 | "time" 15 | 16 | "github.com/dustin/go-humanize" 17 | "github.com/pkg/errors" 18 | 19 | "github.com/elastic/hey-apm/es" 20 | ) 21 | 22 | type Status struct { 23 | Metrics *ExpvarMetrics 24 | SpanIndexCount uint64 25 | TransactionIndexCount uint64 26 | ErrorIndexCount uint64 27 | } 28 | 29 | // GetStatus returns apm-server info and memory stats, plus elasticsearch counts of apm documents. 30 | func GetStatus(logger *log.Logger, secret, url string, connection es.Connection) Status { 31 | status := Status{} 32 | 33 | metrics, err := QueryExpvar(secret, url) 34 | if err == nil { 35 | status.Metrics = &metrics 36 | } else { 37 | logger.Println(err.Error()) 38 | } 39 | status.SpanIndexCount = es.Count(connection, "traces-apm*", "span") 40 | status.TransactionIndexCount = es.Count(connection, "traces-apm*", "transaction") 41 | status.ErrorIndexCount = es.Count(connection, "logs-apm*", "error") 42 | return status 43 | } 44 | 45 | type Info struct { 46 | BuildDate time.Time `json:"build_date"` 47 | BuildSha string `json:"build_sha"` 48 | Version string `json:"version"` 49 | } 50 | 51 | type Cmdline []string 52 | 53 | type ExpvarMetrics struct { 54 | Cmdline Cmdline `json:"cmdline"` 55 | Memstats Memstats `json:"memstats"` 56 | LibbeatMetrics 57 | } 58 | 59 | type LibbeatMetrics struct { 60 | OutputEventsActive *int64 `json:"libbeat.output.events.active"` 61 | PipelineEventsActive *int64 `json:"libbeat.pipeline.events.active"` 62 | } 63 | 64 | type Memstats struct { 65 | TotalAlloc uint64 `json:"TotalAlloc"` 66 | HeapAlloc uint64 `json:"HeapAlloc"` 67 | Mallocs uint64 `json:"Mallocs"` 68 | NumGC uint64 `json:"NumGC"` 69 | TotalAllocDiff uint64 70 | } 71 | 72 | // Sub subtracts some memory stats from another 73 | func (ms Memstats) Sub(ms2 Memstats) Memstats { 74 | return Memstats{ 75 | TotalAlloc: ms.TotalAlloc, 76 | HeapAlloc: ms.HeapAlloc, 77 | TotalAllocDiff: ms.TotalAlloc - ms2.TotalAlloc, 78 | Mallocs: ms.Mallocs - ms2.Mallocs, 79 | NumGC: ms.NumGC - ms2.NumGC, 80 | } 81 | } 82 | 83 | func (ms Memstats) String() string { 84 | var buf bytes.Buffer 85 | tw := tabwriter.NewWriter(&buf, 20, 8, 0, '.', 0) 86 | fmt.Fprintf(tw, "heap \t %s\n", humanize.Bytes(ms.HeapAlloc)) 87 | fmt.Fprintf(tw, "total allocated \t %s\n", humanize.Bytes(ms.TotalAllocDiff)) 88 | fmt.Fprintf(tw, "mallocs \t %d\n", ms.Mallocs) 89 | fmt.Fprintf(tw, "num GC \t %d\n", ms.NumGC) 90 | tw.Flush() 91 | return buf.String() 92 | } 93 | 94 | func (info Info) String() string { 95 | if info.Version == "" { 96 | return "unknown apm-server version" 97 | } 98 | return fmt.Sprintf("apm-server version %s built on %d %s [%s]", 99 | info.Version, info.BuildDate.Day(), info.BuildDate.Month().String(), info.BuildSha[:7]) 100 | } 101 | 102 | // Parse returns all the -E arguments passed to an apm-server except passwords 103 | func (cmd Cmdline) Parse() map[string]string { 104 | ret := make(map[string]string) 105 | var lookup bool 106 | for _, arg := range cmd { 107 | switch { 108 | case arg == "-E": 109 | lookup = true 110 | case lookup: 111 | lookup = false 112 | sep := strings.IndexRune(arg, '=') 113 | if sep < 0 { 114 | continue 115 | } 116 | k, v := arg[:sep], arg[sep+1:] 117 | if !strings.Contains(strings.ToLower(k), "password") { 118 | ret[k] = v 119 | } 120 | } 121 | } 122 | return ret 123 | } 124 | 125 | // QueryInfo sends a request to an apm-server health-check endpoint and parses the result. 126 | func QueryInfo(secret, url string) (Info, error) { 127 | body, err := request(secret, url) 128 | info := Info{} 129 | if err == nil { 130 | err = json.Unmarshal(body, &info) 131 | } 132 | return info, err 133 | } 134 | 135 | // QueryExpvar sends a request to an apm-server /debug/vars endpoint and parses the result. 136 | func QueryExpvar(secret, raw string) (ExpvarMetrics, error) { 137 | u, _ := url.Parse(raw) 138 | u.Path = "/debug/vars" 139 | body, err := request(secret, u.String()) 140 | metrics := ExpvarMetrics{} 141 | if err == nil { 142 | err = json.Unmarshal(body, &metrics) 143 | } 144 | return metrics, errors.Wrap(err, fmt.Sprintf("error querying %s, ensure to start apm-server"+ 145 | " with -E apm-server.expvar.enabled=true", u.Path)) 146 | } 147 | 148 | func request(secret, url string) ([]byte, error) { 149 | req, _ := http.NewRequest("GET", url, nil) 150 | if secret != "" { 151 | req.Header.Set("Authorization", "Beater "+secret) 152 | } 153 | req.Header.Set("Accept", "application/json") 154 | 155 | client := &http.Client{} 156 | resp, err := client.Do(req) 157 | 158 | if err != nil { 159 | return nil, err 160 | } 161 | defer resp.Body.Close() 162 | 163 | body, _ := ioutil.ReadAll(resp.Body) 164 | if resp.StatusCode >= 300 { 165 | return body, errs.New("server status not OK: " + resp.Status) 166 | } 167 | return body, err 168 | } 169 | -------------------------------------------------------------------------------- /worker/logger.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import "log" 4 | 5 | type apmLogger struct { 6 | *log.Logger 7 | } 8 | 9 | func (l *apmLogger) Debugf(format string, args ...interface{}) { 10 | l.Printf("[debug] "+format, args...) 11 | } 12 | 13 | func (l *apmLogger) Errorf(format string, args ...interface{}) { 14 | l.Printf("[error] "+format, args...) 15 | } 16 | 17 | func newApmLogger(logger *log.Logger) *apmLogger { 18 | return &apmLogger{ 19 | Logger: logger, 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /worker/result.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "text/tabwriter" 7 | "time" 8 | 9 | "go.elastic.co/apm" 10 | ) 11 | 12 | // Result holds stats captured from a Go agent plus timing information. 13 | type Result struct { 14 | apm.TracerStats 15 | TransportStats 16 | Start time.Time 17 | End time.Time 18 | Flushed time.Time 19 | } 20 | 21 | func (r Result) EventsGenerated() uint64 { 22 | sent := r.EventsSent() 23 | return sent + r.ErrorsDropped + r.ErrorsDropped + r.TransactionsDropped 24 | } 25 | 26 | func (r Result) EventsSent() uint64 { 27 | return r.ErrorsSent + r.SpansSent + r.TransactionsSent 28 | } 29 | 30 | func (r Result) ElapsedSeconds() float64 { 31 | return r.Flushed.Sub(r.Start).Seconds() 32 | } 33 | 34 | func (r Result) String() string { 35 | var buf bytes.Buffer 36 | tw := tabwriter.NewWriter(&buf, 30, 8, 0, '.', 0) 37 | add := func(key, format string, value interface{}) { 38 | fmt.Fprintf(tw, "%s \t "+format+"\n", key, value) 39 | } 40 | 41 | add("transactions sent", "%d", r.TransactionsSent) 42 | add("transactions dropped", "%d", r.TransactionsDropped) 43 | if total := r.TransactionsSent + r.TransactionsDropped; total > 0 { 44 | add(" - success %", "%.2f", 100*float64(r.TransactionsSent)/float64(total)) 45 | } 46 | 47 | add("spans sent", "%d", r.SpansSent) 48 | add("spans dropped", "%d", r.SpansDropped) 49 | if total := r.SpansSent + r.SpansDropped; total > 0 { 50 | add(" - success %", "%.2f", 100*float64(r.SpansSent)/float64(total)) 51 | } 52 | if r.TransactionsSent > 0 { 53 | add("spans sent per transaction", "%.2f", float64(r.SpansSent)/float64(r.TransactionsSent)) 54 | } 55 | 56 | add("errors sent", "%d", r.ErrorsSent) 57 | add("errors dropped", "%d", r.ErrorsDropped) 58 | if total := r.ErrorsSent + r.ErrorsDropped; total > 0 { 59 | add(" - success %", "%.2f", 100*float64(r.ErrorsSent)/float64(total)) 60 | } 61 | 62 | if elapsedSeconds := r.ElapsedSeconds(); elapsedSeconds > 0 { 63 | eventsSent := r.EventsSent() 64 | add("total events sent", "%d", eventsSent) 65 | add(" - per second", "%.2f", float64(eventsSent)/elapsedSeconds) 66 | if total := r.EventsGenerated(); total > 0 { 67 | add(" - success %", "%.2f", 100*float64(eventsSent)/float64(total)) 68 | } 69 | add(" - accepted", "%d", r.EventsAccepted) 70 | add(" - per second", "%.2f", float64(r.EventsAccepted)/elapsedSeconds) 71 | } 72 | add("total requests", "%d", r.NumRequests) 73 | add("failed", "%d", r.Errors.SendStream) 74 | if len(r.UniqueErrors) > 0 { 75 | add("server errors", "%d", r.UniqueErrors) 76 | } 77 | 78 | tw.Flush() 79 | return buf.String() 80 | } 81 | -------------------------------------------------------------------------------- /worker/run.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "math/rand" 8 | "os" 9 | "time" 10 | 11 | "github.com/pkg/errors" 12 | 13 | "github.com/elastic/hey-apm/es" 14 | "github.com/elastic/hey-apm/models" 15 | "github.com/elastic/hey-apm/server" 16 | ) 17 | 18 | const quiesceTimeout = 5 * time.Minute 19 | 20 | // Run executes a load test work with the given input, prints the results, 21 | // indexes a performance report, and returns it along any error. 22 | // 23 | // If the context is cancelled, the worker exits with the context's error. 24 | // If the stop channel is signalled, the worker exits gracefully with no error. 25 | func Run(ctx context.Context, input models.Input, testName string, stop <-chan struct{}) (models.Report, error) { 26 | testNode, err := es.NewConnection(input.ApmElasticsearchUrl, input.ApmElasticsearchAuth) 27 | if err != nil { 28 | return models.Report{}, errors.Wrap(err, "Elasticsearch used by APM Server not known or reachable") 29 | } 30 | 31 | worker, err := newWorker(input, stop) 32 | if err != nil { 33 | return models.Report{}, err 34 | } 35 | logger := worker.logger.Logger 36 | initialStatus := server.GetStatus(logger, input.ApmServerSecret, input.ApmServerUrl, testNode) 37 | 38 | result, err := worker.work(ctx) 39 | if err != nil { 40 | logger.Println(err.Error()) 41 | return models.Report{}, err 42 | } 43 | logger.Printf("%s elapsed since event generation completed", result.Flushed.Sub(result.End)) 44 | fmt.Println(result) 45 | 46 | // Wait for apm-server to quiesce before proceeding. 47 | var finalStatus server.Status 48 | deadline := time.Now().Add(quiesceTimeout) 49 | for { 50 | finalStatus = server.GetStatus(logger, input.ApmServerSecret, input.ApmServerUrl, testNode) 51 | if finalStatus.Metrics == nil { 52 | logger.Print("expvar endpoint not available, returning") 53 | break 54 | } 55 | outputActiveEvents := derefInt64(finalStatus.Metrics.LibbeatMetrics.OutputEventsActive, 0) 56 | pipelineActiveEvents := derefInt64(finalStatus.Metrics.LibbeatMetrics.PipelineEventsActive, 0) 57 | if outputActiveEvents == 0 && pipelineActiveEvents == 0 { 58 | break 59 | } 60 | if !deadline.After(time.Now()) { 61 | logger.Printf( 62 | "giving up waiting for active events to be processed: %d output, %d pipeline", 63 | outputActiveEvents, pipelineActiveEvents, 64 | ) 65 | break 66 | } 67 | logger.Printf( 68 | "waiting for active events to be processed: %d output, %d pipeline", 69 | outputActiveEvents, pipelineActiveEvents, 70 | ) 71 | time.Sleep(time.Second) 72 | } 73 | report := createReport(input, testName, result, initialStatus, finalStatus) 74 | 75 | if input.SkipIndexReport { 76 | return report, err 77 | } 78 | 79 | if input.ElasticsearchUrl == "" { 80 | logger.Println("es-url unset: not indexing report") 81 | } else { 82 | reportNode, _ := es.NewConnection(input.ElasticsearchUrl, input.ElasticsearchAuth) 83 | if err = es.IndexReport(reportNode, report); err != nil { 84 | logger.Println(err.Error()) 85 | } else { 86 | logger.Println("report indexed with document Id " + report.ReportId) 87 | } 88 | } 89 | return report, err 90 | } 91 | 92 | func derefInt64(v *int64, d int64) int64 { 93 | if v != nil { 94 | return *v 95 | } 96 | return d 97 | } 98 | 99 | // newWorker returns a new worker with with a workload defined by the input. 100 | func newWorker(input models.Input, stop <-chan struct{}) (*worker, error) { 101 | logger := newApmLogger(log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lshortfile)) 102 | tracer, err := newTracer(logger, input.ApmServerUrl, input.ApmServerSecret, input.APIKey, input.ServiceName, input.SpanMaxLimit) 103 | if err != nil { 104 | return nil, err 105 | } 106 | return &worker{ 107 | stop: stop, 108 | logger: logger, 109 | tracer: tracer, 110 | RunTimeout: input.RunTimeout, 111 | FlushTimeout: input.FlushTimeout, 112 | 113 | TransactionFrequency: input.TransactionFrequency, 114 | TransactionLimit: input.TransactionLimit, 115 | SpanMinLimit: input.SpanMinLimit, 116 | SpanMaxLimit: input.SpanMaxLimit, 117 | 118 | ErrorFrequency: input.ErrorFrequency, 119 | ErrorLimit: input.ErrorLimit, 120 | ErrorFrameMinLimit: input.ErrorFrameMinLimit, 121 | ErrorFrameMaxLimit: input.ErrorFrameMaxLimit, 122 | }, nil 123 | } 124 | 125 | func createReport(input models.Input, testName string, result Result, initialStatus, finalStatus server.Status) models.Report { 126 | this, _ := os.Hostname() 127 | r := models.Report{ 128 | Input: input, 129 | 130 | ReportId: shortId(), 131 | ReportDate: time.Now().Format(models.GITRFC), 132 | ReporterHost: this, 133 | TestName: testName, 134 | 135 | Timestamp: time.Now(), 136 | Elapsed: result.Flushed.Sub(result.Start).Seconds(), 137 | 138 | Requests: result.NumRequests, 139 | FailedRequests: result.Errors.SendStream, 140 | 141 | ErrorsGenerated: result.ErrorsSent + result.ErrorsDropped, 142 | ErrorsSent: result.ErrorsSent, 143 | ErrorsIndexed: finalStatus.ErrorIndexCount - initialStatus.ErrorIndexCount, 144 | 145 | TransactionsGenerated: result.TransactionsSent + result.TransactionsDropped, 146 | TransactionsSent: result.TransactionsSent, 147 | TransactionsIndexed: finalStatus.TransactionIndexCount - initialStatus.TransactionIndexCount, 148 | 149 | SpansGenerated: result.SpansSent + result.SpansDropped, 150 | SpansSent: result.SpansSent, 151 | SpansIndexed: finalStatus.SpanIndexCount - initialStatus.SpanIndexCount, 152 | 153 | EventsAccepted: result.EventsAccepted, 154 | } 155 | r.EventsGenerated = r.TransactionsGenerated + r.SpansGenerated + r.ErrorsGenerated 156 | r.EventsSent = r.TransactionsSent + r.SpansSent + r.ErrorsSent 157 | r.EventsIndexed = r.TransactionsIndexed + r.SpansIndexed + r.ErrorsIndexed 158 | 159 | info, ierr := server.QueryInfo(input.ApmServerSecret, input.ApmServerUrl) 160 | if ierr == nil { 161 | fmt.Println(info) 162 | 163 | r.ApmBuild = info.BuildSha 164 | r.ApmBuildDate = info.BuildDate 165 | r.ApmVersion = info.Version 166 | } 167 | 168 | if initialStatus.Metrics != nil && finalStatus.Metrics != nil { 169 | memstats := finalStatus.Metrics.Memstats.Sub(initialStatus.Metrics.Memstats) 170 | fmt.Println(memstats) 171 | 172 | r.TotalAlloc = &memstats.TotalAlloc 173 | r.HeapAlloc = &memstats.HeapAlloc 174 | r.Mallocs = &memstats.Mallocs 175 | r.NumGC = &memstats.NumGC 176 | 177 | r.ApmSettings = initialStatus.Metrics.Cmdline.Parse() 178 | } 179 | 180 | return r 181 | } 182 | 183 | // shortId returns a short docId for elasticsearch documents. It is not an UUID 184 | func shortId() string { 185 | b := make([]byte, 16) 186 | rand.Read(b) 187 | return fmt.Sprintf("%x", b[0:4]) 188 | } 189 | -------------------------------------------------------------------------------- /worker/tracer.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io/ioutil" 7 | "net/http" 8 | "net/url" 9 | "sync" 10 | "time" 11 | 12 | "go.elastic.co/apm" 13 | apmtransport "go.elastic.co/apm/transport" 14 | ) 15 | 16 | type tracer struct { 17 | *apm.Tracer 18 | roundTripper *roundTripperWrapper 19 | } 20 | 21 | func (t *tracer) TransportStats() TransportStats { 22 | t.roundTripper.statsMu.RLock() 23 | defer t.roundTripper.statsMu.RUnlock() 24 | return t.roundTripper.stats 25 | } 26 | 27 | // TransportStats are captured by reading apm-server responses. 28 | type TransportStats struct { 29 | EventsAccepted uint64 30 | UniqueErrors []string 31 | NumRequests uint64 32 | } 33 | 34 | // newTracer returns a wrapper with a new Go agent instance and its transport stats. 35 | func newTracer( 36 | logger apm.Logger, 37 | serverURL, serverSecret, apiKey, serviceName string, 38 | maxSpans int, 39 | ) (*tracer, error) { 40 | 41 | // Ensure that each tracer uses an independent transport. 42 | transport, err := apmtransport.NewHTTPTransport() 43 | if err != nil { 44 | return nil, err 45 | } 46 | transport.SetUserAgent("hey-apm") 47 | if apiKey != "" { 48 | transport.SetAPIKey(apiKey) 49 | } else if serverSecret != "" { 50 | transport.SetSecretToken(serverSecret) 51 | } 52 | if serverURL != "" { 53 | u, err := url.Parse(serverURL) 54 | if err != nil { 55 | panic(err) 56 | } 57 | transport.SetServerURL(u) 58 | } 59 | roundTripper := &roundTripperWrapper{ 60 | roundTripper: transport.Client.Transport, 61 | logger: logger, 62 | uniqueErrors: make(map[string]struct{}), 63 | } 64 | transport.Client.Transport = roundTripper 65 | 66 | goTracer, err := apm.NewTracerOptions(apm.TracerOptions{ 67 | ServiceName: serviceName, 68 | Transport: transport, 69 | }) 70 | if err != nil { 71 | return nil, err 72 | } 73 | goTracer.SetLogger(logger) 74 | goTracer.SetMetricsInterval(0) // disable metrics 75 | goTracer.SetSpanFramesMinDuration(1 * time.Nanosecond) 76 | goTracer.SetMaxSpans(maxSpans) 77 | return &tracer{Tracer: goTracer, roundTripper: roundTripper}, nil 78 | } 79 | 80 | type roundTripperWrapper struct { 81 | roundTripper http.RoundTripper 82 | logger apm.Logger 83 | 84 | statsMu sync.RWMutex 85 | stats TransportStats 86 | uniqueErrors map[string]struct{} 87 | } 88 | 89 | func (rt *roundTripperWrapper) RoundTrip(req *http.Request) (*http.Response, error) { 90 | switch req.URL.Path { 91 | case "/intake/v2/events", "/intake/v2/rum/events": 92 | default: 93 | return rt.roundTripper.RoundTrip(req) 94 | } 95 | 96 | q := req.URL.Query() 97 | q.Set("verbose", "") 98 | req.URL.RawQuery = q.Encode() 99 | 100 | resp, err := rt.roundTripper.RoundTrip(req) 101 | if err != nil { 102 | // Number of *failed* requests is tracked by the Go Agent. 103 | rt.statsMu.Lock() 104 | rt.stats.NumRequests++ 105 | rt.statsMu.Unlock() 106 | return resp, err 107 | } 108 | defer resp.Body.Close() 109 | 110 | rt.statsMu.Lock() 111 | defer rt.statsMu.Unlock() 112 | rt.stats.NumRequests++ 113 | 114 | if resp.Body != http.NoBody { 115 | if data, rerr := ioutil.ReadAll(resp.Body); rerr == nil { 116 | resp.Body = ioutil.NopCloser(bytes.NewReader(data)) 117 | 118 | var response intakeResponse 119 | if err := json.Unmarshal(data, &response); err != nil { 120 | rt.logger.Errorf("failed to decode response: %s", err) 121 | } else { 122 | rt.stats.EventsAccepted += response.Accepted 123 | for _, e := range response.Errors { 124 | if _, ok := rt.uniqueErrors[e.Message]; !ok { 125 | rt.uniqueErrors[e.Message] = struct{}{} 126 | rt.stats.UniqueErrors = append(rt.stats.UniqueErrors, e.Message) 127 | } 128 | } 129 | } 130 | } 131 | } 132 | return resp, err 133 | } 134 | 135 | type intakeResponse struct { 136 | Accepted uint64 137 | Errors []struct { 138 | Message string 139 | Document string 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /worker/work.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math/rand" 7 | "strconv" 8 | "time" 9 | 10 | "go.elastic.co/apm" 11 | "go.elastic.co/apm/stacktrace" 12 | ) 13 | 14 | type worker struct { 15 | stop <-chan struct{} // graceful shutdown 16 | logger *apmLogger 17 | tracer *tracer 18 | 19 | ErrorFrequency time.Duration 20 | ErrorLimit int 21 | ErrorFrameMinLimit int 22 | ErrorFrameMaxLimit int 23 | 24 | TransactionFrequency time.Duration 25 | TransactionLimit int 26 | SpanMinLimit int 27 | SpanMaxLimit int 28 | 29 | RunTimeout time.Duration 30 | FlushTimeout time.Duration 31 | } 32 | 33 | // work uses the Go agent API to generate events and send them to apm-server. 34 | func (w *worker) work(ctx context.Context) (Result, error) { 35 | var runTimerC <-chan time.Time 36 | if w.RunTimeout > 0 { 37 | runTimer := time.NewTimer(w.RunTimeout) 38 | defer runTimer.Stop() 39 | runTimerC = runTimer.C 40 | } 41 | 42 | var errorTicker, transactionTicker maybeTicker 43 | if w.ErrorFrequency > 0 && w.ErrorLimit > 0 { 44 | errorTicker.Start(w.ErrorFrequency) 45 | defer errorTicker.Stop() 46 | } 47 | if w.TransactionFrequency > 0 && w.TransactionLimit > 0 { 48 | transactionTicker.Start(w.TransactionFrequency) 49 | defer transactionTicker.Stop() 50 | } 51 | 52 | result := Result{Start: time.Now()} 53 | var done bool 54 | for !done { 55 | select { 56 | case <-ctx.Done(): 57 | return Result{}, ctx.Err() 58 | case <-w.stop: 59 | done = true 60 | case <-runTimerC: 61 | done = true 62 | case <-errorTicker.C: 63 | w.sendError() 64 | w.ErrorLimit-- 65 | if w.ErrorLimit == 0 { 66 | errorTicker.Stop() 67 | } 68 | case <-transactionTicker.C: 69 | w.sendTransaction() 70 | w.TransactionLimit-- 71 | if w.TransactionLimit == 0 { 72 | transactionTicker.Stop() 73 | } 74 | } 75 | } 76 | 77 | result.End = time.Now() 78 | w.flush() 79 | result.Flushed = time.Now() 80 | result.TracerStats = w.tracer.Stats() 81 | result.TransportStats = w.tracer.TransportStats() 82 | return result, nil 83 | } 84 | 85 | func (w *worker) sendError() { 86 | err := &generatedErr{frames: randRange(w.ErrorFrameMinLimit, w.ErrorFrameMaxLimit)} 87 | w.tracer.NewError(err).Send() 88 | } 89 | 90 | func (w *worker) sendTransaction() { 91 | tx := w.tracer.StartTransaction("generated", "gen") 92 | defer tx.End() 93 | spanCount := randRange(w.SpanMinLimit, w.SpanMaxLimit) 94 | sendSpans(tx, spanCount) 95 | tx.Context.SetTag("spans", strconv.Itoa(spanCount)) 96 | } 97 | 98 | func sendSpans(tx *apm.Transaction, n int) { 99 | // Send spans in a separate goroutine, to ensure we keep 100 | // the number of stack frames stable despite changes to 101 | // hey-apm. 102 | done := make(chan struct{}) 103 | go func() { 104 | defer close(done) 105 | for i := 0; i < n; i++ { 106 | span := tx.StartSpan("I'm a span", "gen.era.ted", nil) 107 | resource := "service-1" 108 | if n % 2 == 0 { 109 | resource = "service-2" 110 | } 111 | span.Context.SetDestinationService(apm.DestinationServiceSpanContext{ 112 | Name: resource, 113 | Resource: resource, 114 | }) 115 | span.Duration = time.Duration(rand.Intn(int(10 * time.Millisecond))) 116 | span.End() 117 | } 118 | }() 119 | <-done 120 | } 121 | 122 | func randRange(min, max int) int { 123 | return min + rand.Intn(max-min+1) 124 | } 125 | 126 | // flush ensures that the entire workload defined is pushed to the apm-server, within the worker timeout limit. 127 | func (w *worker) flush() { 128 | defer w.tracer.Close() 129 | 130 | ctx := context.Background() 131 | if w.FlushTimeout > 0 { 132 | var cancel context.CancelFunc 133 | ctx, cancel = context.WithTimeout(ctx, w.FlushTimeout) 134 | defer cancel() 135 | } 136 | w.tracer.Flush(ctx.Done()) 137 | if ctx.Err() != nil { 138 | w.logger.Errorf("timed out waiting for flush to complete") 139 | } 140 | } 141 | 142 | type generatedErr struct { 143 | frames int 144 | } 145 | 146 | func (e *generatedErr) Error() string { 147 | plural := "s" 148 | if e.frames == 1 { 149 | plural = "" 150 | } 151 | return fmt.Sprintf("Generated error with %d stacktrace frame%s", e.frames, plural) 152 | } 153 | 154 | // must be public for apm agent to use it - https://www.elastic.co/guide/en/apm/agent/go/current/api.html#error-api 155 | func (e *generatedErr) StackTrace() []stacktrace.Frame { 156 | st := make([]stacktrace.Frame, e.frames) 157 | for i := 0; i < e.frames; i++ { 158 | st[i] = stacktrace.Frame{ 159 | File: "fake.go", 160 | Function: "oops", 161 | Line: i + 100, 162 | } 163 | } 164 | return st 165 | } 166 | 167 | type maybeTicker struct { 168 | ticker *time.Ticker 169 | C <-chan time.Time 170 | } 171 | 172 | func (t *maybeTicker) Start(d time.Duration) { 173 | t.ticker = time.NewTicker(d) 174 | t.C = t.ticker.C 175 | } 176 | 177 | func (t *maybeTicker) Stop() { 178 | t.ticker.Stop() 179 | t.C = nil 180 | } 181 | --------------------------------------------------------------------------------