├── .github ├── dependabot.yml └── workflows │ └── pull_request.yml ├── .gitignore ├── CODEOWNERS ├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── main.go ├── mise.lock ├── mise.toml ├── observability └── observability_endpoint.go ├── testhelpers ├── compose │ └── compose.go ├── kubernetes │ └── kubernetes.go ├── prometheus │ └── responses │ │ ├── models.go │ │ └── response_helpers.go ├── remote │ └── remote_observability_endpoint.go ├── requests │ └── http.go └── tempo │ └── responses │ ├── models.go │ ├── response_helpers.go │ ├── response_helpers_test.go │ └── testdata │ └── trace_by_id.json └── yaml ├── README.md ├── docker-compose-docker-lgtm-template.yml ├── docker-compose-include-base.yml ├── generator.go ├── generator_test.go ├── logs.go ├── logs_test.go ├── metrics.go ├── model.go ├── profiles.go ├── runner.go ├── testcase.go ├── testcase_test.go ├── testdata ├── .oatsignore ├── docker-compose-addition.yaml ├── docker-compose-expected.yaml ├── docker-compose-template.yaml ├── foo │ └── oats.yaml ├── loki_response.json ├── oats-merged.yaml └── oats-template.yaml └── traces.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: weekly 7 | - package-ecosystem: gomod 8 | directory: "/" 9 | schedule: 10 | interval: daily 11 | -------------------------------------------------------------------------------- /.github/workflows/pull_request.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Pull request checks 3 | 4 | on: [pull_request] 5 | 6 | permissions: {} 7 | 8 | jobs: 9 | test: 10 | runs-on: ubuntu-24.04 11 | steps: 12 | - name: Check out 13 | with: 14 | persist-credentials: false 15 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 16 | - uses: jdx/mise-action@c94f0bf9e520b150e34c017db785461f7e71c5fb # v2.2.2 17 | - name: Run tests 18 | run: mise run check 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Vim swap files 2 | *.swp 3 | 4 | **/*.test 5 | .idea/ 6 | **/*.log 7 | build/ 8 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # https://help.github.com/articles/about-codeowners/ 2 | # https://git-scm.com/docs/gitignore#_pattern_format 3 | 4 | * @grcevski @rlankfo @zeitlinger 5 | 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenTelemetry Acceptance Tests (OATs) 2 | 3 | OpenTelemetry Acceptance Tests (OATs), or OATs for short, is a test framework for OpenTelemetry. 4 | 5 | - Declarative tests written in YAML 6 | - Supported signals: traces, logs, metrics 7 | - Full round-trip testing: from the application to the observability stack 8 | - Data is stored in the LGTM stack ([Loki], [Grafana], [Tempo], [Prometheus], [OpenTelemetry Collector]) 9 | - Data is queried using LogQL, PromQL, and TraceQL 10 | - All data is sent to the observability stack via OTLP - so OATs can also be used with other observability stacks 11 | - End-to-end testing 12 | - Docker Compose with the [docker-otel-lgtm] image 13 | - Kubernetes with the [docker-otel-lgtm] and [k3d] 14 | 15 | ## Installation 16 | 17 | 1. Install the `oats` binary: 18 | 19 | ```sh 20 | go install github.com/grafana/oats@latest 21 | ``` 22 | 23 | 2. You can confirm it was installed with: 24 | 25 | ```sh 26 | ❯ ls $GOPATH/bin 27 | oats 28 | ``` 29 | 30 | ## Getting Started 31 | 32 | > You can use the test cases in [prom_client_java](https://github.com/prometheus/client_java/tree/main/examples/example-exporter-opentelemetry/oats-tests) as a reference. 33 | > The [GitHub action](https://github.com/prometheus/client_java/blob/main/.github/workflows/acceptance-tests.yml) 34 | > uses a [script](https://github.com/prometheus/client_java/blob/main/scripts/run-acceptance-tests.sh) to run the tests. 35 | 36 | 1. Create a folder `oats-tests` for the following files 37 | 2. Create `Dockerfile` to build the application you want to test 38 | ```Dockerfile 39 | FROM eclipse-temurin:21-jre 40 | COPY target/example-exporter-opentelemetry.jar ./app.jar 41 | ENTRYPOINT [ "java", "-jar", "./app.jar" ] 42 | ``` 43 | 3. Create `docker-compose.yaml` to start the application and any dependencies 44 | ```yaml 45 | version: '3.4' 46 | 47 | services: 48 | java: 49 | build: 50 | dockerfile: Dockerfile 51 | environment: 52 | OTEL_SERVICE_NAME: "rolldice" 53 | OTEL_EXPORTER_OTLP_ENDPOINT: http://lgtm:4318 54 | OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf 55 | OTEL_METRIC_EXPORT_INTERVAL: "5000" # so we don't have to wait 60s for metrics 56 | ``` 57 | 4. Create `oats.yaml` with the test cases 58 | ```yaml 59 | # OATs is an acceptance testing framework for OpenTelemetry - https://github.com/grafana/oats 60 | docker-compose: 61 | files: 62 | - ./docker-compose.yaml 63 | expected: 64 | metrics: 65 | - promql: 'uptime_seconds_total{}' 66 | value: '>= 0' 67 | ``` 68 | 5. Run the tests: 69 | ```sh 70 | oats /path/to/oats-tests/oats.yaml 71 | ``` 72 | 73 | ## Running OATs Directly 74 | 75 | OATs can be run directly using the command-line interface: 76 | 77 | ```sh 78 | # Basic usage 79 | go run main.go /path/to/oats-tests/oats.yaml 80 | 81 | # With flags 82 | go run main.go --timeout=1m --lgtm-version=latest --manual-debug=false /path/to/oats-tests/oats.yaml 83 | ``` 84 | 85 | ## Running multiple tests 86 | 87 | It can run multiple tests: 88 | 89 | ```sh 90 | oats /path/to/repo 91 | ``` 92 | 93 | This will search all subdirectories for test files. The tests are defined in `oats*.yaml` files. 94 | 95 | ## Flags 96 | 97 | The following flags are available: 98 | 99 | - `-timeout`: Set the timeout for test cases (default: 30s) 100 | - `-lgtm-version`: Specify the version of [docker-otel-lgtm] to use (default: "latest") 101 | - `-manual-debug`: Enable debug mode to keep containers running (default: false) 102 | - `-lgtm-log-all`: Enable logging for all containers (default: false) 103 | - `-lgtm-log-grafana`: Enable logging for Grafana (default: false) 104 | - `-lgtm-log-loki`: Enable logging for Loki (default: false) 105 | - `-lgtm-log-tempo`: Enable logging for Tempo (default: false) 106 | - `-lgtm-log-prometheus`: Enable logging for Prometheus (default: false) 107 | - `-lgtm-log-pyroscope`: Enable logging for Pyroscope (default: false) 108 | - `-lgtm-log-otel-collector`: Enable logging for OpenTelemetry Collector (default: false) 109 | 110 | ## Run OATs in GitHub Actions 111 | 112 | Here's an [script](https://github.com/grafana/docker-otel-lgtm/blob/main/scripts/run-acceptance-tests.sh) that is used 113 | from GitHub Actions. It uses [mise](https://mise.jdx.dev/) to install OATs, but you also 114 | [install OATs directly](#installation). 115 | 116 | ## Test Case Syntax 117 | 118 | > You can use any file name that matches `oats*.yaml` (e.g. `oats-test.yaml`), that doesn't end in `-template.yaml`. 119 | > `oats-template.yaml` is reserved for template files, which are used in the `include` section. 120 | 121 | The syntax is a bit similar to https://github.com/kubeshop/tracetest 122 | 123 | This is an example: 124 | 125 | ```yaml 126 | include: 127 | - ../oats-template.yaml 128 | docker-compose: 129 | file: ../docker-compose.yaml 130 | input: 131 | - path: /stock 132 | status: 200 (expected status code, 200 is the default) 133 | interval: 500ms # interval between requests to the input URL 134 | expected: 135 | traces: 136 | - traceql: '{ name =~ "SELECT .*product"}' 137 | spans: 138 | - name: 'regex:SELECT .*' 139 | attributes: 140 | db.system: h2 141 | logs: 142 | - logql: '{exporter = "OTLP"}' 143 | contains: 144 | - 'hello LGTM' 145 | metrics: 146 | - promql: 'db_client_connections_max{pool_name="HikariPool-1"}' 147 | value: "== 10" 148 | ``` 149 | 150 | ### Query traces 151 | 152 | Each entry in the `traces` array is a test case for traces. 153 | 154 | ```yaml 155 | expected: 156 | traces: 157 | - traceql: '{ name =~ "SELECT .*product"}' 158 | spans: 159 | - name: 'regex:SELECT .*' # regex match 160 | attributes: 161 | db.system: h2 162 | allow-duplicates: true # allow multiple spans with the same attributes 163 | ``` 164 | 165 | ### Query logs 166 | 167 | Each entry in the `logs` array is a test case for logs. 168 | 169 | ```yaml 170 | expected: 171 | logs: 172 | - logql: '{service_name="rolldice"} |~ `Anonymous player is rolling the dice.*`' 173 | equals: 'Anonymous player is rolling the dice' 174 | attributes: 175 | service_name: rolldice 176 | attribute-regexp: 177 | container_id: ".*" 178 | no-extra-attributes: true # fail if there are extra attributes 179 | - logql: '{service_name="rolldice"} |~ `Anonymous player is rolling the dice.*`' 180 | regexp: 'Anonymous player is .*' 181 | ``` 182 | 183 | ### Query metrics 184 | 185 | ```yaml 186 | expected: 187 | metrics: 188 | - promql: 'db_client_connections_max{pool_name="HikariPool-1"}' 189 | value: "== 10" 190 | ``` 191 | 192 | ### Matrix of test cases 193 | 194 | Matrix tests are useful to test different configurations of the same application, 195 | e.g. with different settings of the otel collector or different flags in the application. 196 | 197 | ```yaml 198 | matrix: 199 | - name: default 200 | docker-compose: 201 | files: 202 | - ./docker-compose.oats.yml 203 | - name: self-contained 204 | docker-compose: 205 | files: 206 | - ./docker-compose.self-contained.oats.yml 207 | - name: net8 208 | docker-compose: 209 | files: 210 | - ./docker-compose.net8.oats.yml 211 | ``` 212 | 213 | You can then make test cases depend on the matrix name: 214 | 215 | ```yaml 216 | expected: 217 | metrics: 218 | - promql: 'db_client_connections_max{pool_name="HikariPool-1"}' 219 | value: "== 10" 220 | matrix-condition: default 221 | ``` 222 | 223 | `matrix-condition` is a regex that is applied to the matrix name. 224 | 225 | ## Docker Compose 226 | 227 | Describes the docker-compose file(s) to use for the test. 228 | The files typically define the instrumented application you want to test and optionally some dependencies, 229 | e.g. a database server to send requests to. 230 | You don't need (and shouldn't have) to define the observability stack (e.g. Prometheus, Grafana, etc.), 231 | because this is provided by the test framework (and may test different versions of the observability stack, 232 | e.g. OTel Collector and Grafana Alloy). 233 | 234 | This docker-compose file is relative to the `oats.yaml` file. 235 | 236 | ## Kubernetes 237 | 238 | A local Kubernetes cluster can be used to test the application in a Kubernetes environment rather than in docker-compose. 239 | This is useful to test the application in a more realistic environment - and when you want to test Kubernetes specific features. 240 | 241 | Describes the Kubernetes manifest(s) to use for the test. 242 | 243 | ```yaml 244 | kubernetes: 245 | dir: k8s 246 | app-service: dice 247 | app-docker-file: Dockerfile 248 | app-docker-context: .. 249 | app-docker-tag: dice:1.1-SNAPSHOT 250 | app-docker-port: 8080 251 | ``` 252 | 253 | 254 | [Tempo]: https://github.com/grafana/tempo 255 | [OpenTelemetry Collector]: https://opentelemetry.io/docs/collector/ 256 | [Prometheus]: https://prometheus.io/ 257 | [Grafana]: https://grafana.com/ 258 | [Loki]: https://github.com/grafana/loki 259 | [docker-otel-lgtm]: https://github.com/grafana/docker-otel-lgtm/ 260 | [k3d]: https://k3d.io/ 261 | 262 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/grafana/oats 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/onsi/gomega v1.37.0 7 | github.com/stretchr/testify v1.10.0 8 | go.opentelemetry.io/collector/pdata v1.33.0 9 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 10 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 11 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 12 | go.opentelemetry.io/otel/sdk v1.36.0 13 | gopkg.in/yaml.v3 v3.0.1 14 | ) 15 | 16 | require ( 17 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 18 | github.com/cenkalti/backoff/v5 v5.0.2 // indirect 19 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 20 | github.com/go-logr/logr v1.4.2 // indirect 21 | github.com/go-logr/stdr v1.2.2 // indirect 22 | github.com/gogo/protobuf v1.3.2 // indirect 23 | github.com/google/go-cmp v0.7.0 // indirect 24 | github.com/google/pprof v0.0.0-20250208200701-d0013a598941 // indirect 25 | github.com/google/uuid v1.6.0 // indirect 26 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect 27 | github.com/json-iterator/go v1.1.12 // indirect 28 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 29 | github.com/modern-go/reflect2 v1.0.2 // indirect 30 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 31 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 32 | go.opentelemetry.io/otel v1.36.0 // indirect 33 | go.opentelemetry.io/otel/metric v1.36.0 // indirect 34 | go.opentelemetry.io/otel/trace v1.36.0 // indirect 35 | go.opentelemetry.io/proto/otlp v1.6.0 // indirect 36 | go.uber.org/multierr v1.11.0 // indirect 37 | golang.org/x/net v0.40.0 // indirect 38 | golang.org/x/sys v0.33.0 // indirect 39 | golang.org/x/text v0.25.0 // indirect 40 | google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 // indirect 41 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect 42 | google.golang.org/grpc v1.72.1 // indirect 43 | google.golang.org/protobuf v1.36.6 // indirect 44 | ) 45 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= 2 | github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= 3 | github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= 4 | github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 8 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 10 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 11 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 12 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 13 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 14 | github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= 15 | github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= 16 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 17 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 18 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 19 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 20 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 21 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 22 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 23 | github.com/google/pprof v0.0.0-20250208200701-d0013a598941 h1:43XjGa6toxLpeksjcxs1jIoIyr+vUfOqY2c6HB4bpoc= 24 | github.com/google/pprof v0.0.0-20250208200701-d0013a598941/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= 25 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 26 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 27 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= 28 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= 29 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 30 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 31 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 32 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 33 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 34 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 35 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 36 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 37 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 38 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 39 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 40 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 41 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 42 | github.com/onsi/ginkgo/v2 v2.23.3 h1:edHxnszytJ4lD9D5Jjc4tiDkPBZ3siDeJJkUZJJVkp0= 43 | github.com/onsi/ginkgo/v2 v2.23.3/go.mod h1:zXTP6xIp3U8aVuXN8ENK9IXRaTjFnpVB9mGmaSRvxnM= 44 | github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= 45 | github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= 46 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 47 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 48 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 49 | github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= 50 | github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= 51 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 52 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 53 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 54 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 55 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 56 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 57 | go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= 58 | go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= 59 | go.opentelemetry.io/collector/pdata v1.33.0 h1:PuqiZzdCoBJo9NmMzuYfzazpeFZyLmbDVcRrvb497lg= 60 | go.opentelemetry.io/collector/pdata v1.33.0/go.mod h1:TDvbHuvIK+g6xqu1gxtz8ti4pB2x1WcBpjFob5KfleU= 61 | go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= 62 | go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= 63 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= 64 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= 65 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 h1:JgtbA0xkWHnTmYk7YusopJFX6uleBmAuZ8n05NEh8nQ= 66 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0/go.mod h1:179AK5aar5R3eS9FucPy6rggvU0g52cvKId8pv4+v0c= 67 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0 h1:xJ2qHD0C1BeYVTLLR9sX12+Qb95kfeD/byKj6Ky1pXg= 68 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.35.0/go.mod h1:u5BF1xyjstDowA1R5QAO9JHzqK+ublenEW/dyqTjBVk= 69 | go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= 70 | go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= 71 | go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= 72 | go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= 73 | go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= 74 | go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= 75 | go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= 76 | go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= 77 | go.opentelemetry.io/proto/otlp v1.6.0 h1:jQjP+AQyTf+Fe7OKj/MfkDrmK4MNVtw2NpXsf9fefDI= 78 | go.opentelemetry.io/proto/otlp v1.6.0/go.mod h1:cicgGehlFuNdgZkcALOCh3VE6K/u2tAjzlRhDwmVpZc= 79 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 80 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 81 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 82 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 83 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 84 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 85 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 86 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 87 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 88 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 89 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 90 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 91 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 92 | golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= 93 | golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= 94 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 95 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 96 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 97 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 98 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 99 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 100 | golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= 101 | golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 102 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 103 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 104 | golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= 105 | golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= 106 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 107 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 108 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 109 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 110 | golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= 111 | golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= 112 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 113 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 114 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 115 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 116 | google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237 h1:Kog3KlB4xevJlAcbbbzPfRG0+X9fdoGM+UBRKVz6Wr0= 117 | google.golang.org/genproto/googleapis/api v0.0.0-20250519155744-55703ea1f237/go.mod h1:ezi0AVyMKDWy5xAncvjLWH7UcLBB5n7y2fQ8MzjJcto= 118 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34= 119 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= 120 | google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= 121 | google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= 122 | google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 123 | google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 124 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 125 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 126 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 127 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 128 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 129 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "flag" 6 | "fmt" 7 | "github.com/grafana/oats/yaml" 8 | "github.com/onsi/gomega" 9 | "log/slog" 10 | "time" 11 | ) 12 | 13 | func main() { 14 | err := run() 15 | if err != nil { 16 | panic(err) 17 | } 18 | } 19 | 20 | func run() error { 21 | lgtmVersion := flag.String("lgtm-version", "latest", "version of https://github.com/grafana/docker-otel-lgtm") 22 | 23 | logAll := flag.Bool("lgtm-log-all", false, "enable logging for all LGTM components") 24 | logGrafana := flag.Bool("lgtm-log-grafana", false, "enable logging for Grafana") 25 | logPrometheus := flag.Bool("lgtm-log-prometheus", false, "enable logging for Prometheus") 26 | logLoki := flag.Bool("lgtm-log-loki", false, "enable logging for Loki") 27 | logTempo := flag.Bool("lgtm-log-tempo", false, "enable logging for Tempo") 28 | logPyroscope := flag.Bool("lgtm-log-pyroscope", false, "enable logging for Pyroscope") 29 | logCollector := flag.Bool("lgtm-log-collector", false, "enable logging for OTel Collector") 30 | 31 | timeout := flag.Duration("timeout", 30*time.Second, "timeout for the test case") 32 | manualDebug := flag.Bool("manual-debug", false, "debug mode") 33 | flag.Parse() 34 | 35 | if flag.NArg() != 1 { 36 | return errors.New("you must pass a path to the test case yaml file") 37 | } 38 | 39 | logSettings := make(map[string]bool) 40 | logSettings["ENABLE_LOGS_ALL"] = *logAll 41 | logSettings["ENABLE_LOGS_GRAFANA"] = *logGrafana 42 | logSettings["ENABLE_LOGS_PROMETHEUS"] = *logPrometheus 43 | logSettings["ENABLE_LOGS_LOKI"] = *logLoki 44 | logSettings["ENABLE_LOGS_TEMPO"] = *logTempo 45 | logSettings["ENABLE_LOGS_PYROSCOPE"] = *logPyroscope 46 | logSettings["ENABLE_LOGS_OTELCOL"] = *logCollector 47 | 48 | gomega.RegisterFailHandler(func(message string, callerSkip ...int) { 49 | panic(message) 50 | }) 51 | 52 | cases, base := yaml.ReadTestCases(flag.Arg(0)) 53 | if len(cases) == 0 { 54 | return fmt.Errorf("no cases found in %s", base) 55 | } 56 | for _, testCase := range cases { 57 | slog.Info("test case found", "test", testCase.Name) 58 | } 59 | 60 | for _, c := range cases { 61 | c.LgtmVersion = *lgtmVersion 62 | c.LgtmLogSettings = logSettings 63 | c.Timeout = *timeout 64 | c.ManualDebug = *manualDebug 65 | yaml.RunTestCase(c) 66 | } 67 | 68 | slog.Info("all test cases passed") 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /mise.lock: -------------------------------------------------------------------------------- 1 | [tools.go] 2 | version = "1.24.3" 3 | backend = "core:go" 4 | 5 | [tools.go.checksums] 6 | "go1.24.3.linux-amd64.tar.gz" = "sha256:3333f6ea53afa971e9078895eaa4ac7204a8c6b5c68c10e6bc9a33e8e391bdd8" 7 | 8 | [tools.golangci-lint] 9 | version = "2.1.6" 10 | backend = "aqua:golangci/golangci-lint" 11 | 12 | [tools.golangci-lint.checksums] 13 | "golangci-lint-2.1.6-linux-amd64.tar.gz" = "sha256:e55e0eb515936c0fbd178bce504798a9bd2f0b191e5e357768b18fd5415ee541" 14 | -------------------------------------------------------------------------------- /mise.toml: -------------------------------------------------------------------------------- 1 | [tools] 2 | go = "latest" 3 | golangci-lint = "latest" 4 | 5 | [tasks.test] 6 | description = "Run tests" 7 | run = "go test $(go list ./...)" 8 | 9 | [tasks.fmt] 10 | description = "Format code" 11 | run = "gofmt -w ." 12 | 13 | [tasks.lint] 14 | description = "Lint code" 15 | run = "golangci-lint run" 16 | 17 | [tasks.deps] 18 | description = "Update dependencies" 19 | run = "go mod tidy" 20 | 21 | [tasks.check] 22 | description = "Run all checks" 23 | depends = ['lint', 'test'] 24 | 25 | [tasks.build] 26 | description = "Build the project" 27 | run = "go build" 28 | -------------------------------------------------------------------------------- /observability/observability_endpoint.go: -------------------------------------------------------------------------------- 1 | package observability 2 | 3 | import ( 4 | "context" 5 | 6 | "go.opentelemetry.io/otel/sdk/resource" 7 | "go.opentelemetry.io/otel/sdk/trace" 8 | ) 9 | 10 | type Endpoint interface { 11 | // Traces 12 | TracerProvider(context.Context, *resource.Resource) (*trace.TracerProvider, error) 13 | GetTraceByID(context.Context, string) ([]byte, error) 14 | SearchTags(context.Context, map[string]string) ([]byte, error) 15 | 16 | // Metrics 17 | RunPromQL(string) ([]byte, error) 18 | 19 | Start(context.Context) error 20 | Stop(context.Context) error 21 | SearchComposeLogs(string) (bool, error) 22 | } 23 | -------------------------------------------------------------------------------- /testhelpers/compose/compose.go: -------------------------------------------------------------------------------- 1 | // Package docker provides some helpers to manage docker-compose clusters from the test suites 2 | package compose 3 | 4 | import ( 5 | "bufio" 6 | "context" 7 | "errors" 8 | "fmt" 9 | "github.com/grafana/oats/testhelpers/remote" 10 | "io" 11 | "log/slog" 12 | "os" 13 | "os/exec" 14 | "path" 15 | "strings" 16 | "sync" 17 | ) 18 | 19 | type Compose struct { 20 | Command string 21 | DefaultArgs []string 22 | Path string 23 | Env []string 24 | } 25 | 26 | func defaultEnv() []string { 27 | return os.Environ() 28 | } 29 | 30 | func Suite(composeFile string) (*Compose, error) { 31 | command := "docker" 32 | defaultArgs := []string{"compose"} 33 | 34 | return &Compose{ 35 | Command: command, 36 | DefaultArgs: defaultArgs, 37 | Path: path.Join(composeFile), 38 | Env: defaultEnv(), 39 | }, nil 40 | } 41 | 42 | func (c *Compose) Up() error { 43 | //networks accumulate over time and can cause issues with the tests 44 | err := c.runDocker(newCommand("network", "prune", "-f", "--filter", "until=5m").withCompose(false)) 45 | if err != nil { 46 | return fmt.Errorf("failed to prune docker networks: %w", err) 47 | } 48 | 49 | return c.runDocker(newCommand("up", "--build", "--detach", "--force-recreate").withBackground(true)) 50 | } 51 | 52 | func (c *Compose) LogToStdout() error { 53 | return c.runDocker(newCommand("logs")) 54 | } 55 | 56 | func (c *Compose) LogsToConsumer(logConsumer func(io.ReadCloser, *sync.WaitGroup)) error { 57 | return c.runDocker(newCommand("logs").withLogConsumer(logConsumer)) 58 | } 59 | 60 | func (c *Compose) Stop() error { 61 | return c.runDocker(newCommand("stop")) 62 | } 63 | 64 | func (c *Compose) Remove() error { 65 | return c.runDocker(newCommand("rm", "-f")) 66 | } 67 | 68 | func (c *Compose) runDocker(cc command) error { 69 | var cmdArgs []string 70 | if cc.compose { 71 | cmdArgs = c.DefaultArgs 72 | cmdArgs = append(cmdArgs, "-f", c.Path) 73 | } 74 | cmdArgs = append(cmdArgs, cc.args...) 75 | cmd := exec.Command(c.Command, cmdArgs...) 76 | cmd.Env = c.Env 77 | if cc.logConsumer != nil { 78 | stdout, _ := cmd.StdoutPipe() 79 | cmd.Stderr = cmd.Stdout 80 | wg := sync.WaitGroup{} 81 | wg.Add(1) 82 | go cc.logConsumer(stdout, &wg) 83 | 84 | err := cmd.Start() 85 | if err != nil { 86 | return fmt.Errorf("failed to start docker command: %w", err) 87 | } 88 | wg.Wait() 89 | } else if cc.background { 90 | slog.Info("Running", "command", cmd.String(), "dir", c.Path) 91 | stdout, _ := cmd.StdoutPipe() 92 | cmd.Stderr = cmd.Stdout 93 | go func() { 94 | reader := bufio.NewReader(stdout) 95 | line, err := reader.ReadString('\n') 96 | for err == nil { 97 | slog.Info(line) 98 | line, err = reader.ReadString('\n') 99 | } 100 | }() 101 | 102 | err := cmd.Start() 103 | if err != nil { 104 | return fmt.Errorf("failed to start docker command: %w", err) 105 | } 106 | } else { 107 | slog.Info("Running", "command", cmd.String(), "dir", c.Path) 108 | cmd.Stdout = os.Stdout 109 | cmd.Stderr = os.Stderr 110 | err := cmd.Run() 111 | if err != nil { 112 | return fmt.Errorf("failed to run docker command: %w", err) 113 | } 114 | } 115 | return nil 116 | } 117 | 118 | func (c *Compose) Close() error { 119 | var errs []string 120 | if err := c.LogToStdout(); err != nil { 121 | errs = append(errs, err.Error()) 122 | } 123 | if err := c.Stop(); err != nil { 124 | errs = append(errs, err.Error()) 125 | } 126 | if err := c.Remove(); err != nil { 127 | errs = append(errs, err.Error()) 128 | } 129 | if len(errs) == 0 { 130 | return nil 131 | } 132 | return errors.New(strings.Join(errs, " / ")) 133 | } 134 | 135 | func NewEndpoint(composeFilePath string, ports remote.PortsConfig) *remote.Endpoint { 136 | var compose *Compose 137 | return remote.NewEndpoint(ports, func(ctx context.Context) error { 138 | var err error 139 | 140 | if composeFilePath == "" { 141 | return fmt.Errorf("composeFilePath cannot be empty") 142 | } 143 | 144 | compose, err = Suite(composeFilePath) 145 | if err != nil { 146 | return err 147 | } 148 | err = compose.Up() 149 | 150 | return err 151 | }, func(ctx context.Context) error { 152 | return compose.Close() 153 | }, 154 | func(f func(io.ReadCloser, *sync.WaitGroup)) error { 155 | return compose.LogsToConsumer(f) 156 | }, 157 | ) 158 | } 159 | 160 | type command struct { 161 | background bool 162 | compose bool 163 | logConsumer func(io.ReadCloser, *sync.WaitGroup) 164 | args []string 165 | } 166 | 167 | func newCommand( 168 | args ...string) command { 169 | return command{ 170 | args: args, 171 | compose: true, 172 | } 173 | } 174 | 175 | func (c command) withBackground(background bool) command { 176 | c.background = background 177 | return c 178 | } 179 | 180 | func (c command) withCompose(compose bool) command { 181 | c.compose = compose 182 | return c 183 | } 184 | 185 | func (c command) withLogConsumer(logConsumer func(io.ReadCloser, *sync.WaitGroup)) command { 186 | c.logConsumer = logConsumer 187 | return c 188 | } 189 | -------------------------------------------------------------------------------- /testhelpers/kubernetes/kubernetes.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/grafana/oats/testhelpers/remote" 7 | "io" 8 | "log/slog" 9 | "os" 10 | "os/exec" 11 | "sync" 12 | ) 13 | 14 | type Kubernetes struct { 15 | Dir string `yaml:"dir"` 16 | AppService string `yaml:"app-service"` 17 | AppDockerFile string `yaml:"app-docker-file"` 18 | AppDockerContext string `yaml:"app-docker-context"` 19 | AppDockerTag string `yaml:"app-docker-tag"` 20 | AppDockerPort int `yaml:"app-docker-port"` 21 | ImportImages []string `yaml:"import-images"` 22 | } 23 | 24 | func NewEndpoint(model *Kubernetes, ports remote.PortsConfig, testName string, dir string) *remote.Endpoint { 25 | var killList []*os.Process 26 | run := func(cmd *exec.Cmd, background bool) error { 27 | slog.Info("running", "command", cmd.String(), "dir", dir) 28 | cmd.Stdout = os.Stdout 29 | cmd.Stderr = os.Stderr 30 | cmd.Dir = dir 31 | if background { 32 | err := cmd.Start() 33 | if err != nil { 34 | return err 35 | } 36 | killList = append(killList, cmd.Process) 37 | return nil 38 | } 39 | return cmd.Run() 40 | } 41 | return remote.NewEndpoint(ports, func(ctx context.Context) error { 42 | return start(model, ports, testName, run) 43 | }, func(ctx context.Context) error { 44 | for _, p := range killList { 45 | err := p.Kill() 46 | if err != nil { 47 | return err 48 | } 49 | } 50 | return run(exec.Command("k3d", "cluster", "delete", testName), false) 51 | }, 52 | func(f func(io.ReadCloser, *sync.WaitGroup)) error { 53 | panic("not implemented for kubernetes") 54 | }, 55 | ) 56 | } 57 | 58 | func start(model *Kubernetes, ports remote.PortsConfig, testName string, run func(cmd *exec.Cmd, background bool) error) error { 59 | portForward := func(localPort int, remotePort int) error { 60 | cmd := exec.Command("kubectl", "port-forward", "service/lgtm", fmt.Sprintf("%d:%d", localPort, remotePort)) 61 | return run(cmd, true) 62 | } 63 | 64 | if model.AppDockerContext == "" { 65 | model.AppDockerContext = "." 66 | } 67 | 68 | err := run(exec.Command("docker", "build", "-f", model.AppDockerFile, "-t", model.AppDockerTag, model.AppDockerContext), false) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | cluster := testName 74 | if len(cluster) > 32 { 75 | cluster = cluster[(len(cluster))-32:] 76 | } 77 | 78 | err = run(exec.Command("k3d", "cluster", "list", cluster), false) 79 | if err == nil { 80 | slog.Info("cluster already exists - deleting", "name", cluster) 81 | err = run(exec.Command("k3d", "cluster", "delete", cluster), false) 82 | if err != nil { 83 | return err 84 | } 85 | } 86 | 87 | err = run(exec.Command("k3d", "cluster", "create", cluster), false) 88 | if err != nil { 89 | return err 90 | } 91 | importImages := []string{model.AppDockerTag} 92 | importImages = append(importImages, model.ImportImages...) 93 | for _, image := range importImages { 94 | err = run(exec.Command("k3d", "image", "import", "-c", cluster, image), false) 95 | if err != nil { 96 | return err 97 | } 98 | } 99 | err = run(exec.Command("kubectl", "apply", "-f", model.Dir), false) 100 | if err != nil { 101 | return err 102 | } 103 | err = run(exec.Command("kubectl", "wait", "--timeout=5m", "--for=condition=ready", "pod", "-l", "app=lgtm"), false) 104 | if err != nil { 105 | return err 106 | } 107 | err = run(exec.Command("kubectl", "port-forward", "service/"+model.AppService, fmt.Sprintf("%d:8080", model.AppDockerPort)), true) 108 | if err != nil { 109 | return err 110 | } 111 | 112 | err = portForward(ports.LokiHttpPort, 3100) 113 | if err != nil { 114 | return err 115 | } 116 | err = portForward(ports.PrometheusHTTPPort, 9090) 117 | if err != nil { 118 | return err 119 | } 120 | err = portForward(ports.TempoHTTPPort, 3200) 121 | if err != nil { 122 | return err 123 | } 124 | err = portForward(ports.PyroscopeHttpPort, 4040) 125 | if err != nil { 126 | return err 127 | } 128 | return nil 129 | } 130 | -------------------------------------------------------------------------------- /testhelpers/prometheus/responses/models.go: -------------------------------------------------------------------------------- 1 | package responses 2 | 3 | type PrometheusQueryResult struct { 4 | Status string `json:"status"` 5 | Data Data `json:"data"` 6 | } 7 | 8 | type Data struct { 9 | Result []Result `json:"result"` 10 | ResultType string `json:"resultType"` 11 | } 12 | 13 | // Result structure assumes that resultType is always == "vector" 14 | type Result struct { 15 | Metric map[string]string `json:"metric"` 16 | Value []interface{} 17 | } 18 | -------------------------------------------------------------------------------- /testhelpers/prometheus/responses/response_helpers.go: -------------------------------------------------------------------------------- 1 | package responses 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | ) 7 | 8 | func ParseQueryOutput(body []byte) ([]Result, error) { 9 | qr := PrometheusQueryResult{} 10 | if err := json.Unmarshal(body, &qr); err != nil { 11 | return nil, fmt.Errorf("decoding Prometheus response: %w", err) 12 | } 13 | 14 | return qr.Data.Result, nil 15 | } 16 | -------------------------------------------------------------------------------- /testhelpers/remote/remote_observability_endpoint.go: -------------------------------------------------------------------------------- 1 | package remote 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "go.opentelemetry.io/otel/exporters/otlp/otlptrace" 8 | "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" 9 | "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" 10 | "go.opentelemetry.io/otel/sdk/resource" 11 | "go.opentelemetry.io/otel/sdk/trace" 12 | "io" 13 | "log/slog" 14 | "net/http" 15 | "net/url" 16 | "strings" 17 | "sync" 18 | ) 19 | 20 | type PortsConfig struct { 21 | TracesGRPCPort int 22 | TracesHTTPPort int 23 | TempoHTTPPort int 24 | MimirHTTPPort int 25 | PrometheusHTTPPort int 26 | LokiHttpPort int 27 | PyroscopeHttpPort int 28 | } 29 | 30 | type Endpoint struct { 31 | ports PortsConfig 32 | start func(context.Context) error 33 | stop func(context.Context) error 34 | logReader func(func(io.ReadCloser, *sync.WaitGroup)) error 35 | } 36 | 37 | func NewEndpoint(ports PortsConfig, start func(context.Context) error, stop func(context.Context) error, logReader func(func(io.ReadCloser, *sync.WaitGroup)) error) *Endpoint { 38 | return &Endpoint{ 39 | ports: ports, 40 | start: start, 41 | stop: stop, 42 | logReader: logReader, 43 | } 44 | } 45 | 46 | func (e *Endpoint) TracerProvider(ctx context.Context, r *resource.Resource) (*trace.TracerProvider, error) { 47 | var exporter *otlptrace.Exporter 48 | var err error 49 | 50 | if e.ports.TracesGRPCPort != 0 { 51 | exporter, err = otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure(), otlptracegrpc.WithEndpoint(fmt.Sprintf("localhost:%d", e.ports.TracesGRPCPort))) 52 | if err != nil { 53 | return nil, err 54 | } 55 | } else if e.ports.TracesHTTPPort != 0 { 56 | exporter, err = otlptracehttp.New(ctx, otlptracehttp.WithInsecure(), otlptracehttp.WithEndpoint(fmt.Sprintf("localhost:%d/v1/traces", e.ports.TracesHTTPPort))) 57 | if err != nil { 58 | return nil, err 59 | } 60 | } 61 | 62 | if ctx.Err() != nil { 63 | return nil, ctx.Err() 64 | } 65 | 66 | if exporter == nil { 67 | return nil, fmt.Errorf("unknown exporter format, specify an OTel trace GRPC or HTTP port") 68 | } 69 | 70 | traceProvider := trace.NewTracerProvider( 71 | trace.WithBatcher(exporter), 72 | trace.WithResource(r), 73 | ) 74 | 75 | return traceProvider, nil 76 | } 77 | 78 | func (e *Endpoint) makeGetRequest(url string) ([]byte, error) { 79 | resp, getErr := http.Get(url) 80 | if getErr != nil { 81 | return nil, getErr 82 | } 83 | 84 | if resp.StatusCode != http.StatusOK { 85 | return nil, fmt.Errorf("expected HTTP status 200, but got: %d", resp.StatusCode) 86 | } 87 | 88 | defer func(Body io.ReadCloser) { 89 | _ = Body.Close() 90 | }(resp.Body) 91 | 92 | respBytes, err := io.ReadAll(resp.Body) 93 | if err != nil { 94 | return nil, err 95 | } 96 | 97 | return respBytes, nil 98 | } 99 | 100 | func (e *Endpoint) GetTraceByID(ctx context.Context, id string) ([]byte, error) { 101 | if ctx.Err() != nil { 102 | return nil, ctx.Err() 103 | } 104 | 105 | return e.makeGetRequest(fmt.Sprintf("http://localhost:%d/api/traces/%s", e.ports.TempoHTTPPort, id)) 106 | } 107 | 108 | func (e *Endpoint) SearchTempo(ctx context.Context, query string) ([]byte, error) { 109 | if ctx.Err() != nil { 110 | return nil, ctx.Err() 111 | } 112 | 113 | return e.makeGetRequest(fmt.Sprintf("http://localhost:%d/api/search?q=%s", e.ports.TempoHTTPPort, url.QueryEscape(query))) 114 | } 115 | 116 | func (e *Endpoint) SearchTags(ctx context.Context, tags map[string]string) ([]byte, error) { 117 | if ctx.Err() != nil { 118 | return nil, ctx.Err() 119 | } 120 | 121 | var tb strings.Builder 122 | 123 | for tag, val := range tags { 124 | if tb.Len() != 0 { 125 | tb.WriteString("&") 126 | } 127 | s := tag + "=" + val 128 | tb.WriteString(url.QueryEscape(s)) 129 | } 130 | 131 | return e.makeGetRequest(fmt.Sprintf("http://localhost:%d/api/search?tags=%s", e.ports.TempoHTTPPort, tb.String())) 132 | } 133 | 134 | func (e *Endpoint) RunPromQL(promQL string) ([]byte, error) { 135 | var u string 136 | if e.ports.MimirHTTPPort != 0 { 137 | u = fmt.Sprintf("http://localhost:%d/prometheus/api/v1/query?query=%s", e.ports.MimirHTTPPort, url.PathEscape(promQL)) 138 | } else if e.ports.PrometheusHTTPPort != 0 { 139 | u = fmt.Sprintf("http://localhost:%d/api/v1/query?query=%s", e.ports.PrometheusHTTPPort, url.PathEscape(promQL)) 140 | } else { 141 | return nil, fmt.Errorf("to run PromQL you must configure a MimirHTTPPort or a PrometheusHTTPPort") 142 | } 143 | 144 | resp, err := http.Get(u) 145 | if err != nil { 146 | return nil, fmt.Errorf("querying prometheus: %w", err) 147 | } 148 | 149 | defer func(Body io.ReadCloser) { 150 | _ = Body.Close() 151 | }(resp.Body) 152 | 153 | body, err := io.ReadAll(resp.Body) 154 | if err != nil { 155 | return nil, fmt.Errorf("can't read response body: %w", err) 156 | } 157 | 158 | return body, nil 159 | } 160 | 161 | func (e *Endpoint) SearchLoki(query string) ([]byte, error) { 162 | if e.ports.LokiHttpPort == 0 { 163 | return nil, fmt.Errorf("to search Loki you must configure a LokiHttpPort") 164 | } 165 | 166 | u := fmt.Sprintf("http://localhost:%d/loki/api/v1/query_range?since=5m&limit=1&query=%s", e.ports.LokiHttpPort, url.PathEscape(query)) 167 | 168 | resp, err := http.Get(u) 169 | if err != nil { 170 | return nil, fmt.Errorf("querying loki: %w", err) 171 | } 172 | 173 | defer func(Body io.ReadCloser) { 174 | _ = Body.Close() 175 | }(resp.Body) 176 | 177 | body, err := io.ReadAll(resp.Body) 178 | if err != nil { 179 | return nil, fmt.Errorf("can't read response body: %w", err) 180 | } 181 | 182 | return body, nil 183 | } 184 | 185 | func (e *Endpoint) SearchPyroscope(query string) ([]byte, error) { 186 | if e.ports.PyroscopeHttpPort == 0 { 187 | return nil, fmt.Errorf("to search Pyroscope you must configure a PyroscopeHttpPort") 188 | } 189 | 190 | u := fmt.Sprintf("http://localhost:%d/pyroscope/render?from=from=now-1m&query=%s", e.ports.PyroscopeHttpPort, url.PathEscape(query)) 191 | 192 | resp, err := http.Get(u) 193 | if err != nil { 194 | return nil, fmt.Errorf("querying pyroscope: %w", err) 195 | } 196 | 197 | defer func(Body io.ReadCloser) { 198 | _ = Body.Close() 199 | }(resp.Body) 200 | 201 | body, err := io.ReadAll(resp.Body) 202 | if err != nil { 203 | return nil, fmt.Errorf("can't read response body: %w", err) 204 | } 205 | 206 | return body, nil 207 | } 208 | 209 | func (e *Endpoint) Start(ctx context.Context) error { 210 | return e.start(ctx) 211 | } 212 | 213 | func (e *Endpoint) Stop(ctx context.Context) error { 214 | return e.stop(ctx) 215 | } 216 | 217 | func (e *Endpoint) SearchComposeLogs(message string) (bool, error) { 218 | found := false 219 | slog.Info("searching compose logs", "message", message) 220 | err := e.logReader(func(pipe io.ReadCloser, wg *sync.WaitGroup) { 221 | reader := bufio.NewReader(pipe) 222 | line, err := reader.ReadString('\n') 223 | for err == nil { 224 | if strings.Contains(line, message) { 225 | found = true 226 | } 227 | line, err = reader.ReadString('\n') 228 | } 229 | wg.Done() 230 | }) 231 | if err != nil { 232 | return false, fmt.Errorf("error reading logs: %w", err) 233 | } 234 | return found, nil 235 | } 236 | -------------------------------------------------------------------------------- /testhelpers/requests/http.go: -------------------------------------------------------------------------------- 1 | package requests 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "net/http" 7 | ) 8 | 9 | var tr = &http.Transport{ 10 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 11 | } 12 | var testHTTPClient = &http.Client{Transport: tr} 13 | 14 | func doRequest(req *http.Request, statusCode int) error { 15 | req.Header.Set("Content-Type", "application/json") 16 | 17 | r, err := testHTTPClient.Do(req) 18 | 19 | if err != nil { 20 | return err 21 | } 22 | 23 | if r.StatusCode != statusCode { 24 | return fmt.Errorf("expected HTTP status %d, but got: %d", statusCode, r.StatusCode) 25 | } 26 | 27 | return nil 28 | } 29 | 30 | func DoHTTPGet(url string, statusCode int) error { 31 | req, err := http.NewRequest(http.MethodGet, url, nil) 32 | 33 | if err != nil { 34 | return err 35 | } 36 | 37 | return doRequest(req, statusCode) 38 | } 39 | -------------------------------------------------------------------------------- /testhelpers/tempo/responses/models.go: -------------------------------------------------------------------------------- 1 | package responses 2 | 3 | type TempoSearchResult struct { 4 | Traces []Trace `json:"traces"` 5 | } 6 | 7 | type Trace struct { 8 | TraceID string `json:"traceID"` 9 | RootServiceName string `json:"rootServiceName"` 10 | RootTraceName string `json:"rootTraceName"` 11 | StartTimeUnixNano string `json:"startTimeUnixNano"` 12 | DurationMs int `json:"durationMs"` 13 | } 14 | -------------------------------------------------------------------------------- /testhelpers/tempo/responses/response_helpers.go: -------------------------------------------------------------------------------- 1 | package responses 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/hex" 6 | "encoding/json" 7 | "fmt" 8 | "regexp" 9 | "strings" 10 | 11 | "go.opentelemetry.io/collector/pdata/pcommon" 12 | "go.opentelemetry.io/collector/pdata/ptrace" 13 | ) 14 | 15 | func MatchTraceAttribute(attributes pcommon.Map, attrType pcommon.ValueType, key, value string) error { 16 | att, found := attributes.Get(key) 17 | if !found { 18 | return fmt.Errorf("couldn't find attribute %s", key) 19 | } 20 | 21 | // We convert to strings anyway, if this check is here you can't match Int values in traces 22 | // valueType := att.Type() 23 | // if valueType != attrType { 24 | // return fmt.Errorf("value type for key %s is %s which doesn't match the expect type %s", key, valueType, attrType) 25 | // } 26 | 27 | if value != "" && !matcherMaybeRegex(value)(att.AsString()) { 28 | return fmt.Errorf("value for key %s is %s which doesn't match the expect value %s", key, att.AsString(), value) 29 | } 30 | return nil 31 | } 32 | 33 | type AttributeMatch struct { 34 | Key string 35 | Value string 36 | Type pcommon.ValueType 37 | } 38 | 39 | func ParseTraceDetails(body []byte) (ptrace.Traces, error) { 40 | body = fixIds(body, regexp.MustCompile(`"traceId":\s*"(.*?)"`), "traceId", 16) 41 | body = fixIds(body, regexp.MustCompile(`"spanId":\s*"(.*?)"`), "spanId", 8) 42 | body = fixIds(body, regexp.MustCompile(`"parentSpanId":\s*"(.*?)"`), "parentSpanId", 8) 43 | s := string(body) 44 | s = strings.ReplaceAll(s, `"batches"`, `"resourceSpans"`) 45 | body = []byte(s) 46 | 47 | unmarshaler := ptrace.JSONUnmarshaler{} 48 | return unmarshaler.UnmarshalTraces(body) 49 | } 50 | 51 | func fixIds(body []byte, re *regexp.Regexp, idName string, capacity int) []byte { 52 | return re.ReplaceAllFunc(body, func(b []byte) []byte { 53 | submatch := re.FindStringSubmatch(string(b)) 54 | dst := make([]byte, capacity) 55 | _, err := base64.StdEncoding.Decode(dst, []byte(submatch[1])) 56 | if err != nil { 57 | panic(err) 58 | } 59 | r := fmt.Sprintf("\"%s\": \"%s\"", idName, hex.EncodeToString(dst)) 60 | return []byte(r) 61 | }) 62 | } 63 | 64 | func ParseTempoSearchResult(body []byte) (TempoSearchResult, error) { 65 | var st TempoSearchResult 66 | err := json.Unmarshal(body, &st) 67 | 68 | return st, err 69 | } 70 | 71 | func FindSpans(td ptrace.Traces, name string) []ptrace.Span { 72 | spans, _ := FindSpansWithAttributes(td, name) 73 | return spans 74 | } 75 | func FindSpansWithAttributes(td ptrace.Traces, name string) ([]ptrace.Span, map[string]any) { 76 | m := matcherMaybeRegex(name) 77 | return FindSpansFunc(td, func(span *ptrace.Span) bool { 78 | return m(span.Name()) 79 | }) 80 | } 81 | 82 | func FindSpansFunc(td ptrace.Traces, pred func(*ptrace.Span) bool) ([]ptrace.Span, map[string]any) { 83 | var result []ptrace.Span 84 | atts := map[string]any{} 85 | resourceSpans := td.ResourceSpans() 86 | for i := 0; i < resourceSpans.Len(); i++ { 87 | resourceSpan := resourceSpans.At(i) 88 | scopeSpans := resourceSpan.ScopeSpans() 89 | for j := 0; j < scopeSpans.Len(); j++ { 90 | scopeSpan := scopeSpans.At(j) 91 | spans := scopeSpan.Spans() 92 | for k := 0; k < spans.Len(); k++ { 93 | span := spans.At(k) 94 | if pred(&span) { 95 | result = append(result, span) 96 | for k, v := range resourceSpan.Resource().Attributes().AsRaw() { 97 | atts[k] = v 98 | } 99 | scope := scopeSpan.Scope() 100 | for k, v := range scope.Attributes().AsRaw() { 101 | atts[k] = v 102 | } 103 | //this is how the scope name is shown in tempo 104 | atts["otel.library.name"] = scope.Name() 105 | atts["otel.library.version"] = scope.Version() 106 | } 107 | } 108 | } 109 | } 110 | return result, atts 111 | } 112 | 113 | func matcherMaybeRegex(want string) func(got string) bool { 114 | var re *regexp.Regexp 115 | if strings.HasPrefix(want, "regex:") { 116 | re = regexp.MustCompile(want[6:]) 117 | } 118 | 119 | return func(got string) bool { 120 | if re != nil { 121 | return re.MatchString(got) 122 | } 123 | return want == got 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /testhelpers/tempo/responses/response_helpers_test.go: -------------------------------------------------------------------------------- 1 | package responses 2 | 3 | import ( 4 | "github.com/stretchr/testify/require" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func TestParseTraceDetails(t *testing.T) { 10 | b, err := os.ReadFile("testdata/trace_by_id.json") 11 | require.NoError(t, err) 12 | details, err := ParseTraceDetails(b) 13 | require.NoError(t, err) 14 | spans := details.ResourceSpans() 15 | i := spans.Len() 16 | require.NotZero(t, i) 17 | require.NotEmpty(t, details) 18 | require.Len(t, FindSpans(details, "kafkaTopic publish"), 1) 19 | require.Len(t, FindSpans(details, "regex:.* publish"), 1) 20 | } 21 | -------------------------------------------------------------------------------- /testhelpers/tempo/responses/testdata/trace_by_id.json: -------------------------------------------------------------------------------- 1 | { 2 | "batches": [ 3 | { 4 | "resource": { 5 | "attributes": [ 6 | { 7 | "key": "container.id", 8 | "value": { 9 | "stringValue": "6bc40019e40bf63b842d99834ab4ed6826792a36fedb0b45754290816c4cfbe3" 10 | } 11 | }, 12 | { 13 | "key": "deployment.environment", 14 | "value": { 15 | "stringValue": "production" 16 | } 17 | }, 18 | { 19 | "key": "host.arch", 20 | "value": { 21 | "stringValue": "amd64" 22 | } 23 | }, 24 | { 25 | "key": "host.name", 26 | "value": { 27 | "stringValue": "nevla" 28 | } 29 | }, 30 | { 31 | "key": "os.description", 32 | "value": { 33 | "stringValue": "Linux 6.2.0-31-generic" 34 | } 35 | }, 36 | { 37 | "key": "os.type", 38 | "value": { 39 | "stringValue": "linux" 40 | } 41 | }, 42 | { 43 | "key": "process.command_args", 44 | "value": { 45 | "arrayValue": { 46 | "values": [ 47 | { 48 | "stringValue": "/opt/java/openjdk/bin/java" 49 | }, 50 | { 51 | "stringValue": "-javaagent:grafana-opentelemetry-java.jar" 52 | }, 53 | { 54 | "stringValue": "-jar" 55 | }, 56 | { 57 | "stringValue": "/app.jar" 58 | } 59 | ] 60 | } 61 | } 62 | }, 63 | { 64 | "key": "process.executable.path", 65 | "value": { 66 | "stringValue": "/opt/java/openjdk/bin/java" 67 | } 68 | }, 69 | { 70 | "key": "process.pid", 71 | "value": { 72 | "intValue": "1" 73 | } 74 | }, 75 | { 76 | "key": "process.runtime.description", 77 | "value": { 78 | "stringValue": "Eclipse Adoptium OpenJDK 64-Bit Server VM 17.0.8+7" 79 | } 80 | }, 81 | { 82 | "key": "process.runtime.name", 83 | "value": { 84 | "stringValue": "OpenJDK Runtime Environment" 85 | } 86 | }, 87 | { 88 | "key": "process.runtime.version", 89 | "value": { 90 | "stringValue": "17.0.8+7" 91 | } 92 | }, 93 | { 94 | "key": "service.instance.id", 95 | "value": { 96 | "stringValue": "7d722ae5-15ae-47c5-81c5-41842a9b98ac" 97 | } 98 | }, 99 | { 100 | "key": "service.namespace", 101 | "value": { 102 | "stringValue": "shop" 103 | } 104 | }, 105 | { 106 | "key": "service.version", 107 | "value": { 108 | "stringValue": "1.1" 109 | } 110 | }, 111 | { 112 | "key": "telemetry.auto.version", 113 | "value": { 114 | "stringValue": "1.29.0" 115 | } 116 | }, 117 | { 118 | "key": "telemetry.sdk.language", 119 | "value": { 120 | "stringValue": "java" 121 | } 122 | }, 123 | { 124 | "key": "telemetry.sdk.name", 125 | "value": { 126 | "stringValue": "grafana" 127 | } 128 | }, 129 | { 130 | "key": "telemetry.sdk.version", 131 | "value": { 132 | "stringValue": "0.1" 133 | } 134 | }, 135 | { 136 | "key": "service.name", 137 | "value": { 138 | "stringValue": "app" 139 | } 140 | } 141 | ] 142 | }, 143 | "scopeSpans": [ 144 | { 145 | "scope": { 146 | "name": "io.opentelemetry.tomcat-10.0", 147 | "version": "1.29.0-alpha" 148 | }, 149 | "spans": [ 150 | { 151 | "traceId": "B3vTDy+BVLWUjcBHJPif9Q==", 152 | "spanId": "P7TdbwfHmLc=", 153 | "name": "GET /stock", 154 | "kind": "SPAN_KIND_SERVER", 155 | "startTimeUnixNano": "1693573635765422206", 156 | "endTimeUnixNano": "1693573635766725356", 157 | "attributes": [ 158 | { 159 | "key": "user_agent.original", 160 | "value": { 161 | "stringValue": "Go-http-client/1.1" 162 | } 163 | }, 164 | { 165 | "key": "network.protocol.version", 166 | "value": { 167 | "stringValue": "1.1" 168 | } 169 | }, 170 | { 171 | "key": "http.response.body.size", 172 | "value": { 173 | "intValue": "13" 174 | } 175 | }, 176 | { 177 | "key": "client.socket.port", 178 | "value": { 179 | "intValue": "45600" 180 | } 181 | }, 182 | { 183 | "key": "url.scheme", 184 | "value": { 185 | "stringValue": "http" 186 | } 187 | }, 188 | { 189 | "key": "thread.name", 190 | "value": { 191 | "stringValue": "http-nio-8080-exec-4" 192 | } 193 | }, 194 | { 195 | "key": "url.path", 196 | "value": { 197 | "stringValue": "/stock" 198 | } 199 | }, 200 | { 201 | "key": "server.port", 202 | "value": { 203 | "intValue": "8080" 204 | } 205 | }, 206 | { 207 | "key": "server.address", 208 | "value": { 209 | "stringValue": "localhost" 210 | } 211 | }, 212 | { 213 | "key": "http.response.status_code", 214 | "value": { 215 | "intValue": "200" 216 | } 217 | }, 218 | { 219 | "key": "http.route", 220 | "value": { 221 | "stringValue": "/stock" 222 | } 223 | }, 224 | { 225 | "key": "http.request.method", 226 | "value": { 227 | "stringValue": "GET" 228 | } 229 | }, 230 | { 231 | "key": "thread.id", 232 | "value": { 233 | "intValue": "42" 234 | } 235 | }, 236 | { 237 | "key": "client.socket.address", 238 | "value": { 239 | "stringValue": "127.0.0.1" 240 | } 241 | }, 242 | { 243 | "key": "network.protocol.name", 244 | "value": { 245 | "stringValue": "http" 246 | } 247 | } 248 | ], 249 | "status": {} 250 | } 251 | ] 252 | } 253 | ] 254 | }, 255 | { 256 | "resource": { 257 | "attributes": [ 258 | { 259 | "key": "container.id", 260 | "value": { 261 | "stringValue": "6bc40019e40bf63b842d99834ab4ed6826792a36fedb0b45754290816c4cfbe3" 262 | } 263 | }, 264 | { 265 | "key": "deployment.environment", 266 | "value": { 267 | "stringValue": "production" 268 | } 269 | }, 270 | { 271 | "key": "host.arch", 272 | "value": { 273 | "stringValue": "amd64" 274 | } 275 | }, 276 | { 277 | "key": "host.name", 278 | "value": { 279 | "stringValue": "nevla" 280 | } 281 | }, 282 | { 283 | "key": "os.description", 284 | "value": { 285 | "stringValue": "Linux 6.2.0-31-generic" 286 | } 287 | }, 288 | { 289 | "key": "os.type", 290 | "value": { 291 | "stringValue": "linux" 292 | } 293 | }, 294 | { 295 | "key": "process.command_args", 296 | "value": { 297 | "arrayValue": { 298 | "values": [ 299 | { 300 | "stringValue": "/opt/java/openjdk/bin/java" 301 | }, 302 | { 303 | "stringValue": "-javaagent:grafana-opentelemetry-java.jar" 304 | }, 305 | { 306 | "stringValue": "-jar" 307 | }, 308 | { 309 | "stringValue": "/app.jar" 310 | } 311 | ] 312 | } 313 | } 314 | }, 315 | { 316 | "key": "process.executable.path", 317 | "value": { 318 | "stringValue": "/opt/java/openjdk/bin/java" 319 | } 320 | }, 321 | { 322 | "key": "process.pid", 323 | "value": { 324 | "intValue": "1" 325 | } 326 | }, 327 | { 328 | "key": "process.runtime.description", 329 | "value": { 330 | "stringValue": "Eclipse Adoptium OpenJDK 64-Bit Server VM 17.0.8+7" 331 | } 332 | }, 333 | { 334 | "key": "process.runtime.name", 335 | "value": { 336 | "stringValue": "OpenJDK Runtime Environment" 337 | } 338 | }, 339 | { 340 | "key": "process.runtime.version", 341 | "value": { 342 | "stringValue": "17.0.8+7" 343 | } 344 | }, 345 | { 346 | "key": "service.instance.id", 347 | "value": { 348 | "stringValue": "7d722ae5-15ae-47c5-81c5-41842a9b98ac" 349 | } 350 | }, 351 | { 352 | "key": "service.namespace", 353 | "value": { 354 | "stringValue": "shop" 355 | } 356 | }, 357 | { 358 | "key": "service.version", 359 | "value": { 360 | "stringValue": "1.1" 361 | } 362 | }, 363 | { 364 | "key": "telemetry.auto.version", 365 | "value": { 366 | "stringValue": "1.29.0" 367 | } 368 | }, 369 | { 370 | "key": "telemetry.sdk.language", 371 | "value": { 372 | "stringValue": "java" 373 | } 374 | }, 375 | { 376 | "key": "telemetry.sdk.name", 377 | "value": { 378 | "stringValue": "grafana" 379 | } 380 | }, 381 | { 382 | "key": "telemetry.sdk.version", 383 | "value": { 384 | "stringValue": "0.1" 385 | } 386 | }, 387 | { 388 | "key": "service.name", 389 | "value": { 390 | "stringValue": "app" 391 | } 392 | } 393 | ] 394 | }, 395 | "scopeSpans": [ 396 | { 397 | "scope": { 398 | "name": "io.opentelemetry.spring-webmvc-6.0", 399 | "version": "1.29.0-alpha" 400 | }, 401 | "spans": [ 402 | { 403 | "traceId": "B3vTDy+BVLWUjcBHJPif9Q==", 404 | "spanId": "2qIRtIJLOhY=", 405 | "parentSpanId": "P7TdbwfHmLc=", 406 | "name": "StockController.getStock", 407 | "kind": "SPAN_KIND_INTERNAL", 408 | "startTimeUnixNano": "1693573635765875566", 409 | "endTimeUnixNano": "1693573635766430797", 410 | "attributes": [ 411 | { 412 | "key": "thread.id", 413 | "value": { 414 | "intValue": "42" 415 | } 416 | }, 417 | { 418 | "key": "thread.name", 419 | "value": { 420 | "stringValue": "http-nio-8080-exec-4" 421 | } 422 | } 423 | ], 424 | "status": {} 425 | } 426 | ] 427 | } 428 | ] 429 | }, 430 | { 431 | "resource": { 432 | "attributes": [ 433 | { 434 | "key": "container.id", 435 | "value": { 436 | "stringValue": "6bc40019e40bf63b842d99834ab4ed6826792a36fedb0b45754290816c4cfbe3" 437 | } 438 | }, 439 | { 440 | "key": "deployment.environment", 441 | "value": { 442 | "stringValue": "production" 443 | } 444 | }, 445 | { 446 | "key": "host.arch", 447 | "value": { 448 | "stringValue": "amd64" 449 | } 450 | }, 451 | { 452 | "key": "host.name", 453 | "value": { 454 | "stringValue": "nevla" 455 | } 456 | }, 457 | { 458 | "key": "os.description", 459 | "value": { 460 | "stringValue": "Linux 6.2.0-31-generic" 461 | } 462 | }, 463 | { 464 | "key": "os.type", 465 | "value": { 466 | "stringValue": "linux" 467 | } 468 | }, 469 | { 470 | "key": "process.command_args", 471 | "value": { 472 | "arrayValue": { 473 | "values": [ 474 | { 475 | "stringValue": "/opt/java/openjdk/bin/java" 476 | }, 477 | { 478 | "stringValue": "-javaagent:grafana-opentelemetry-java.jar" 479 | }, 480 | { 481 | "stringValue": "-jar" 482 | }, 483 | { 484 | "stringValue": "/app.jar" 485 | } 486 | ] 487 | } 488 | } 489 | }, 490 | { 491 | "key": "process.executable.path", 492 | "value": { 493 | "stringValue": "/opt/java/openjdk/bin/java" 494 | } 495 | }, 496 | { 497 | "key": "process.pid", 498 | "value": { 499 | "intValue": "1" 500 | } 501 | }, 502 | { 503 | "key": "process.runtime.description", 504 | "value": { 505 | "stringValue": "Eclipse Adoptium OpenJDK 64-Bit Server VM 17.0.8+7" 506 | } 507 | }, 508 | { 509 | "key": "process.runtime.name", 510 | "value": { 511 | "stringValue": "OpenJDK Runtime Environment" 512 | } 513 | }, 514 | { 515 | "key": "process.runtime.version", 516 | "value": { 517 | "stringValue": "17.0.8+7" 518 | } 519 | }, 520 | { 521 | "key": "service.instance.id", 522 | "value": { 523 | "stringValue": "7d722ae5-15ae-47c5-81c5-41842a9b98ac" 524 | } 525 | }, 526 | { 527 | "key": "service.namespace", 528 | "value": { 529 | "stringValue": "shop" 530 | } 531 | }, 532 | { 533 | "key": "service.version", 534 | "value": { 535 | "stringValue": "1.1" 536 | } 537 | }, 538 | { 539 | "key": "telemetry.auto.version", 540 | "value": { 541 | "stringValue": "1.29.0" 542 | } 543 | }, 544 | { 545 | "key": "telemetry.sdk.language", 546 | "value": { 547 | "stringValue": "java" 548 | } 549 | }, 550 | { 551 | "key": "telemetry.sdk.name", 552 | "value": { 553 | "stringValue": "grafana" 554 | } 555 | }, 556 | { 557 | "key": "telemetry.sdk.version", 558 | "value": { 559 | "stringValue": "0.1" 560 | } 561 | }, 562 | { 563 | "key": "service.name", 564 | "value": { 565 | "stringValue": "app" 566 | } 567 | } 568 | ] 569 | }, 570 | "scopeSpans": [ 571 | { 572 | "scope": { 573 | "name": "io.opentelemetry.kafka-clients-0.11", 574 | "version": "1.29.0-alpha" 575 | }, 576 | "spans": [ 577 | { 578 | "traceId": "B3vTDy+BVLWUjcBHJPif9Q==", 579 | "spanId": "ymu/6+mnY7A=", 580 | "parentSpanId": "2qIRtIJLOhY=", 581 | "name": "kafkaTopic publish", 582 | "kind": "SPAN_KIND_PRODUCER", 583 | "startTimeUnixNano": "1693573635765969551", 584 | "endTimeUnixNano": "1693573635768210634", 585 | "attributes": [ 586 | { 587 | "key": "messaging.system", 588 | "value": { 589 | "stringValue": "kafka" 590 | } 591 | }, 592 | { 593 | "key": "messaging.kafka.destination.partition", 594 | "value": { 595 | "intValue": "2" 596 | } 597 | }, 598 | { 599 | "key": "messaging.kafka.client_id", 600 | "value": { 601 | "stringValue": "producer-1" 602 | } 603 | }, 604 | { 605 | "key": "thread.id", 606 | "value": { 607 | "intValue": "42" 608 | } 609 | }, 610 | { 611 | "key": "messaging.kafka.message.offset", 612 | "value": { 613 | "intValue": "34" 614 | } 615 | }, 616 | { 617 | "key": "messaging.destination.name", 618 | "value": { 619 | "stringValue": "kafkaTopic" 620 | } 621 | }, 622 | { 623 | "key": "thread.name", 624 | "value": { 625 | "stringValue": "http-nio-8080-exec-4" 626 | } 627 | } 628 | ], 629 | "status": {} 630 | } 631 | ] 632 | } 633 | ] 634 | } 635 | ] 636 | } 637 | -------------------------------------------------------------------------------- /yaml/README.md: -------------------------------------------------------------------------------- 1 | # Declarative YAML tests 2 | 3 | See [OpenTelemetry Acceptance Tests (OATs)](../README.md) for more information. 4 | -------------------------------------------------------------------------------- /yaml/docker-compose-docker-lgtm-template.yml: -------------------------------------------------------------------------------- 1 | services: 2 | lgtm: 3 | image: grafana/otel-lgtm:{{ .LgtmVersion }} 4 | environment: 5 | {{ range $key, $value := .LgtmLogSettings }} 6 | {{ $key }}: "{{ $value }}" 7 | {{ end }} 8 | ports: 9 | - "{{ .GrafanaHTTPPort }}:3000" 10 | - "{{ .PrometheusHTTPPort }}:9090" 11 | - "{{ .TempoHTTPPort }}:3200" 12 | - "{{ .LokiHTTPPort }}:3100" 13 | - "{{ .PyroscopeHttpPort }}:4040" 14 | -------------------------------------------------------------------------------- /yaml/docker-compose-include-base.yml: -------------------------------------------------------------------------------- 1 | include: 2 | {{ range .files }}- {{ . }} 3 | {{ end }} 4 | -------------------------------------------------------------------------------- /yaml/generator.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | import ( 4 | "bytes" 5 | _ "embed" 6 | "fmt" 7 | "log/slog" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | "strings" 12 | "text/template" 13 | 14 | "github.com/onsi/gomega" 15 | "gopkg.in/yaml.v3" 16 | ) 17 | 18 | //go:embed docker-compose-docker-lgtm-template.yml 19 | var lgtmTemplate []byte 20 | 21 | //go:embed docker-compose-include-base.yml 22 | var lgtmTemplateIncludeBase []byte 23 | 24 | func (c *TestCase) CreateDockerComposeFile() string { 25 | p := filepath.Join(c.OutputDir, "docker-compose.yml") 26 | content := c.getContent() 27 | err := os.WriteFile(p, content, 0644) 28 | gomega.Expect(err).ToNot(gomega.HaveOccurred()) 29 | return p 30 | } 31 | 32 | func (c *TestCase) getContent() []byte { 33 | return c.generateDockerComposeFile() 34 | } 35 | 36 | func (c *TestCase) generateDockerComposeFile() []byte { 37 | compose := c.Definition.DockerCompose 38 | slog.Info("using docker-compose", "lgtm-version", c.LgtmVersion) 39 | 40 | vars := map[string]any{} 41 | vars["ApplicationPort"] = c.PortConfig.ApplicationPort 42 | vars["GrafanaHTTPPort"] = c.PortConfig.GrafanaHTTPPort 43 | vars["PrometheusHTTPPort"] = c.PortConfig.PrometheusHTTPPort 44 | vars["LokiHTTPPort"] = c.PortConfig.LokiHTTPPort 45 | vars["TempoHTTPPort"] = c.PortConfig.TempoHTTPPort 46 | vars["PyroscopeHttpPort"] = c.PortConfig.PyroscopeHttpPort 47 | vars["LgtmVersion"] = c.LgtmVersion 48 | vars["LgtmLogSettings"] = c.LgtmLogSettings 49 | 50 | env := os.Environ() 51 | 52 | for k, v := range vars { 53 | env = append(env, fmt.Sprintf("%s=%s", k, v)) 54 | } 55 | 56 | env = append(env, compose.Environment...) 57 | 58 | t := template.Must(template.New("docker-compose").Parse(string(lgtmTemplate))) 59 | 60 | buf := bytes.NewBufferString("") 61 | err := t.Execute(buf, vars) 62 | gomega.Expect(err).ToNot(gomega.HaveOccurred()) 63 | name := filepath.FromSlash("./docker-compose-docker-lgtm-template.yml") 64 | generated, err := filepath.Abs(strings.TrimSuffix(name, filepath.Ext(name)) + "-generated.yml") 65 | gomega.Expect(err).ToNot(gomega.HaveOccurred()) 66 | _ = os.WriteFile(generated, buf.Bytes(), 0644) 67 | defer func(name string) { 68 | _ = os.Remove(name) 69 | }(generated) 70 | files := []string{generated} 71 | for _, filename := range compose.Files { 72 | t = template.Must(template.ParseFiles(filename)) 73 | addbuf := bytes.NewBufferString("") 74 | err = t.Execute(addbuf, vars) 75 | gomega.Expect(err).ToNot(gomega.HaveOccurred()) 76 | name := strings.TrimSuffix(filename, filepath.Ext(filename)) + "-generated.yml" 77 | err = os.WriteFile(name, addbuf.Bytes(), 0644) 78 | gomega.Expect(err).ToNot(gomega.HaveOccurred()) 79 | defer func(name string) { 80 | _ = os.Remove(name) 81 | }(name) 82 | files = append(files, name) 83 | } 84 | 85 | t = template.Must(template.New("docker-compose-base").Parse(string(lgtmTemplateIncludeBase))) 86 | buf = bytes.NewBufferString("") 87 | vars = map[string]any{} 88 | vars["files"] = files 89 | err = t.Execute(buf, vars) 90 | gomega.Expect(err).ToNot(gomega.HaveOccurred()) 91 | f, err := os.CreateTemp("", "docker-compose-base.yml") 92 | gomega.Expect(err).ToNot(gomega.HaveOccurred()) 93 | _, err = f.Write(buf.Bytes()) 94 | gomega.Expect(err).ToNot(gomega.HaveOccurred()) 95 | defer func(name string) { 96 | _ = os.Remove(name) 97 | }(f.Name()) 98 | 99 | // uses docker compose to merge templates 100 | args := []string{"compose", "-f", f.Name(), "config"} 101 | cmd := exec.Command("docker", args...) 102 | cmd.Env = env 103 | cmd.Stderr = os.Stderr 104 | content, err := cmd.Output() 105 | if err != nil { 106 | slog.Error("failed to run docker compose", "error", err) 107 | } 108 | gomega.Expect(err).ToNot(gomega.HaveOccurred()) 109 | return content 110 | } 111 | 112 | func joinComposeFiles(template []byte, addition []byte) ([]byte, error) { 113 | base := map[string]any{} 114 | add := map[string]any{} 115 | 116 | err := yaml.Unmarshal(template, &base) 117 | if err != nil { 118 | return nil, err 119 | } 120 | err = yaml.Unmarshal(addition, &add) 121 | if err != nil { 122 | return nil, err 123 | } 124 | 125 | //not a generic solution, but works for our use case 126 | addFromBase(base, add, "services") 127 | addFromBase(base, add, "networks") 128 | 129 | return yaml.Marshal(add) 130 | } 131 | 132 | func addFromBase(base map[string]any, add map[string]any, key string) { 133 | addMap, ok := add[key].(map[string]any) 134 | if !ok { 135 | addMap = map[string]any{} 136 | add[key] = addMap 137 | } 138 | 139 | baseMap, ok := base[key].(map[string]any) 140 | if ok { 141 | for k, v := range baseMap { 142 | addMap[k] = v 143 | } 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /yaml/generator_test.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | import ( 4 | "github.com/stretchr/testify/require" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func TestJoinDockerComposeFiles(t *testing.T) { 10 | template, err := os.ReadFile("testdata/docker-compose-template.yaml") 11 | require.NoError(t, err) 12 | add, err := os.ReadFile("testdata/docker-compose-addition.yaml") 13 | require.NoError(t, err) 14 | want, err := os.ReadFile("testdata/docker-compose-expected.yaml") 15 | require.NoError(t, err) 16 | c, err := joinComposeFiles(template, add) 17 | require.NoError(t, err) 18 | require.YAMLEq(t, string(want), string(c)) 19 | } 20 | -------------------------------------------------------------------------------- /yaml/logs.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/onsi/gomega" 6 | "log/slog" 7 | ) 8 | 9 | type LokiQueryResponse struct { 10 | Status string `json:"status"` 11 | Data struct { 12 | Result []struct { 13 | Stream map[string]string `json:"stream"` 14 | Values [][]string `json:"values"` 15 | } `json:"result"` 16 | } `json:"data"` 17 | } 18 | 19 | func AssertLoki(r *runner, l ExpectedLogs) { 20 | b, err := r.endpoint.SearchLoki(l.LogQL) 21 | r.LogQueryResult("logQL query %v response %v err=%v\n", l.LogQL, string(b), err) 22 | g := r.gomegaInst 23 | g.Expect(err).ToNot(gomega.HaveOccurred()) 24 | AssertLokiResponse(b, l, r) 25 | } 26 | 27 | func AssertLokiResponse(b []byte, l ExpectedLogs, r *runner) { 28 | g := r.gomegaInst 29 | g.Expect(len(b)).Should(gomega.BeNumerically(">", 0), "expected loki response to be non-empty") 30 | 31 | response := LokiQueryResponse{} 32 | err := json.Unmarshal(b, &response) 33 | if err != nil { 34 | slog.Info("error unmarshalling loki", "response", string(b)) 35 | } 36 | g.Expect(err).ToNot(gomega.HaveOccurred()) 37 | 38 | g.Expect(response.Status).To(gomega.Equal("success")) 39 | streams := response.Data.Result 40 | g.Expect(len(streams)).Should(gomega.BeNumerically(">", 0), "expected loki streams to be non-empty") 41 | 42 | stream := streams[0] 43 | line := stream.Values[0][1] 44 | if len(l.Equals) > 0 { 45 | // check for exact match in additional asserts 46 | g.Expect(line).To(gomega.ContainSubstring(l.Equals)) 47 | } 48 | if len(l.Regexp) > 0 { 49 | g.Expect(line).To(gomega.MatchRegexp(l.Regexp)) 50 | } 51 | for _, s := range l.Contains { 52 | g.Expect(line).To(gomega.ContainSubstring(s)) 53 | } 54 | 55 | // don't retry we've found the log 56 | r.additionalAsserts = append(r.additionalAsserts, func() { 57 | if len(l.Equals) > 0 { 58 | g.Expect(line).To(gomega.Equal(l.Equals)) 59 | } 60 | assertLabels(l, stream.Stream) 61 | }) 62 | } 63 | 64 | func assertLabels(l ExpectedLogs, labels map[string]string) { 65 | for k, v := range l.Attributes { 66 | gomega.Expect(labels).To(gomega.HaveKeyWithValue(k, v)) 67 | } 68 | for k, v := range l.AttributeRegexp { 69 | gomega.Expect(labels).To(gomega.HaveKey(k)) 70 | gomega.Expect(labels[k]).To(gomega.MatchRegexp(v)) 71 | } 72 | if l.NoExtraAttributes { 73 | var allowedKeys []string 74 | for k := range l.Attributes { 75 | allowedKeys = append(allowedKeys, k) 76 | } 77 | for k := range l.AttributeRegexp { 78 | allowedKeys = append(allowedKeys, k) 79 | } 80 | var keys []string 81 | for k := range labels { 82 | keys = append(keys, k) 83 | } 84 | gomega.Expect(keys).To(gomega.ConsistOf(allowedKeys)) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /yaml/logs_test.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | import ( 4 | "github.com/onsi/gomega" 5 | "github.com/stretchr/testify/require" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | func TestAssertLokiResponse(t *testing.T) { 11 | gomega.RegisterTestingT(t) 12 | 13 | file, err := os.ReadFile("testdata/loki_response.json") 14 | require.NoError(t, err) 15 | logs := ExpectedLogs{ 16 | Contains: []string{"simulating an error"}, 17 | Attributes: map[string]string{ 18 | "deployment_environment": "staging", 19 | "exception_message": "simulating an error", 20 | "exception_type": "java.lang.RuntimeException", 21 | "scope_name": "org.apache.catalina.core.ContainerBase.[Tomcat].[localhost].[/].[dispatcherServlet]", 22 | "service_name": "dice", 23 | "service_namespace": "shop", 24 | "severity_number": "17", 25 | "severity_text": "ERROR", 26 | "k8s_container_name": "dice", 27 | "k8s_namespace_name": "default", 28 | "message": "Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed: java.lang.RuntimeException: simulating an error] with root cause", 29 | }, 30 | AttributeRegexp: map[string]string{ 31 | "thread_name": ".*", 32 | "span_id": ".*", 33 | "trace_id": ".*", 34 | "k8s_pod_name": "dice-.*-.*", 35 | "k8s_pod_uid": ".*", 36 | "k8s_container_restart_count": ".*", 37 | "service_instance_id": ".*", 38 | "exception_stacktrace": ".*", 39 | }, 40 | NoExtraAttributes: true, 41 | } 42 | r := &runner{ 43 | gomegaInst: gomega.NewGomega(func(message string, callerSkip ...int) { 44 | t.Error(message) 45 | }), 46 | } 47 | AssertLokiResponse(file, logs, r) 48 | 49 | require.Len(t, r.additionalAsserts, 1) 50 | r.additionalAsserts[0]() 51 | } 52 | -------------------------------------------------------------------------------- /yaml/metrics.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | import ( 4 | "strconv" 5 | "strings" 6 | 7 | "github.com/grafana/oats/testhelpers/prometheus/responses" 8 | "github.com/onsi/gomega" 9 | ) 10 | 11 | var promQlVariables = []string{"$job", "$instance", "$pod", "$namespace", "$container"} 12 | 13 | func replaceVariables(promQL string) string { 14 | for _, variable := range promQlVariables { 15 | promQL = strings.ReplaceAll(promQL, variable, ".*") 16 | } 17 | return promQL 18 | } 19 | 20 | func AssertProm(r *runner, promQL string, value string) { 21 | promQL = replaceVariables(promQL) 22 | b, err := r.endpoint.RunPromQL(promQL) 23 | r.LogQueryResult("promQL query %v response %v err=%v\n", promQL, string(b), err) 24 | g := r.gomegaInst 25 | g.Expect(err).ToNot(gomega.HaveOccurred()) 26 | g.Expect(len(b)).Should(gomega.BeNumerically(">", 0), "expected prometheus response to be non-empty") 27 | 28 | pr, err := responses.ParseQueryOutput(b) 29 | g.Expect(err).ToNot(gomega.HaveOccurred()) 30 | g.Expect(len(pr)).Should(gomega.BeNumerically(">", 0), "expected prometheus results to be non-empty") 31 | 32 | s := strings.Split(value, " ") 33 | comp := s[0] 34 | val, err := strconv.ParseFloat(s[1], 64) 35 | if err != nil { 36 | g.Expect(err).ToNot(gomega.HaveOccurred()) 37 | } 38 | got, err := strconv.ParseFloat(pr[0].Value[1].(string), 64) 39 | if err != nil { 40 | g.Expect(err).ToNot(gomega.HaveOccurred()) 41 | } 42 | 43 | g.Expect(got).Should(gomega.BeNumerically(comp, val), "expected %s %f, got %f", comp, val, got) 44 | } 45 | -------------------------------------------------------------------------------- /yaml/model.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | import ( 4 | "fmt" 5 | "github.com/grafana/oats/testhelpers/kubernetes" 6 | "log/slog" 7 | "path/filepath" 8 | "strconv" 9 | "time" 10 | 11 | "github.com/onsi/gomega" 12 | "gopkg.in/yaml.v3" 13 | ) 14 | 15 | type ExpectedMetrics struct { 16 | PromQL string `yaml:"promql"` 17 | Value string `yaml:"value"` 18 | MatrixCondition string `yaml:"matrix-condition"` 19 | } 20 | 21 | type ExpectedSpan struct { 22 | Name string `yaml:"name"` 23 | Attributes map[string]string `yaml:"attributes"` 24 | AllowDups bool `yaml:"allow-duplicates"` 25 | } 26 | 27 | type ExpectedLogs struct { 28 | LogQL string `yaml:"logql"` 29 | Equals string `yaml:"equals"` 30 | Contains []string `yaml:"contains"` 31 | Regexp string `yaml:"regexp"` 32 | Attributes map[string]string `yaml:"attributes"` 33 | AttributeRegexp map[string]string `yaml:"attribute-regexp"` 34 | NoExtraAttributes bool `yaml:"no-extra-attributes"` 35 | MatrixCondition string `yaml:"matrix-condition"` 36 | } 37 | 38 | type Flamebearers struct { 39 | Contains string `yaml:"contains"` 40 | } 41 | 42 | type ExpectedProfiles struct { 43 | Query string `yaml:"query"` 44 | Flamebearers Flamebearers `yaml:"flamebearers"` 45 | MatrixCondition string `yaml:"matrix-condition"` 46 | } 47 | 48 | type ExpectedTraces struct { 49 | TraceQL string `yaml:"traceql"` 50 | Spans []ExpectedSpan `yaml:"spans"` 51 | MatrixCondition string `yaml:"matrix-condition"` 52 | } 53 | 54 | type CustomCheck struct { 55 | Script string `yaml:"script"` 56 | MatrixCondition string `yaml:"matrix-condition"` 57 | } 58 | 59 | type Expected struct { 60 | ComposeLogs []string `yaml:"compose-logs"` 61 | Logs []ExpectedLogs `yaml:"logs"` 62 | Traces []ExpectedTraces `yaml:"traces"` 63 | Metrics []ExpectedMetrics `yaml:"metrics"` 64 | Profiles []ExpectedProfiles `yaml:"profiles"` 65 | CustomChecks []CustomCheck `yaml:"custom-checks"` 66 | } 67 | 68 | type Matrix struct { 69 | Name string `yaml:"name"` 70 | DockerCompose *DockerCompose `yaml:"docker-compose"` 71 | Kubernetes *kubernetes.Kubernetes `yaml:"kubernetes"` 72 | } 73 | 74 | type DockerCompose struct { 75 | Files []string `yaml:"files"` 76 | Environment []string `yaml:"env"` 77 | } 78 | 79 | type Input struct { 80 | Path string `yaml:"path"` 81 | Status string `yaml:"status"` 82 | } 83 | 84 | type TestCaseDefinition struct { 85 | Include []string `yaml:"include"` 86 | DockerCompose *DockerCompose `yaml:"docker-compose"` 87 | Kubernetes *kubernetes.Kubernetes `yaml:"kubernetes"` 88 | Matrix []Matrix `yaml:"matrix"` 89 | Input []Input `yaml:"input"` 90 | Interval time.Duration `yaml:"interval"` 91 | Expected Expected `yaml:"expected"` 92 | } 93 | 94 | const DefaultTestCaseInterval = 100 * time.Millisecond 95 | 96 | func (d *TestCaseDefinition) Merge(other TestCaseDefinition) { 97 | d.Expected.Logs = append(d.Expected.Logs, other.Expected.Logs...) 98 | d.Expected.Traces = append(d.Expected.Traces, other.Expected.Traces...) 99 | d.Expected.Metrics = append(d.Expected.Metrics, other.Expected.Metrics...) 100 | d.Expected.Profiles = append(d.Expected.Profiles, other.Expected.Profiles...) 101 | d.Expected.CustomChecks = append(d.Expected.CustomChecks, other.Expected.CustomChecks...) 102 | d.Matrix = append(d.Matrix, other.Matrix...) 103 | if d.DockerCompose == nil { 104 | d.DockerCompose = other.DockerCompose 105 | } 106 | d.Input = append(d.Input, other.Input...) 107 | } 108 | 109 | type PortConfig struct { 110 | ApplicationPort int 111 | GrafanaHTTPPort int 112 | PrometheusHTTPPort int 113 | LokiHTTPPort int 114 | TempoHTTPPort int 115 | PyroscopeHttpPort int 116 | } 117 | 118 | type TestCase struct { 119 | Name string 120 | MatrixTestCaseName string 121 | Dir string 122 | OutputDir string 123 | Definition TestCaseDefinition 124 | PortConfig *PortConfig 125 | Timeout time.Duration 126 | LgtmVersion string 127 | LgtmLogSettings map[string]bool 128 | ManualDebug bool 129 | } 130 | 131 | func (r *runner) LogQueryResult(format string, a ...any) { 132 | if r.Verbose { 133 | result := fmt.Sprintf(format, a...) 134 | if len(result) > 1000 { 135 | result = result[:1000] + ".." 136 | } 137 | slog.Info(result) 138 | } 139 | } 140 | 141 | func (c *TestCase) validateAndSetVariables() { 142 | if c.Definition.Kubernetes != nil { 143 | validateK8s(c.Definition.Kubernetes) 144 | gomega.Expect(c.Definition.DockerCompose).To(gomega.BeNil(), "kubernetes and docker-compose are mutually exclusive") 145 | } else { 146 | validateDockerCompose(c.Definition.DockerCompose, c.Dir) 147 | } 148 | validateInput(c.Definition.Input) 149 | expected := c.Definition.Expected 150 | gomega.Expect(len(expected.Metrics) == 0 && len(expected.Traces) == 0 && len(expected.Logs) == 0 && len(expected.Profiles) == 0).To(gomega.BeFalse()) 151 | 152 | for _, c := range expected.CustomChecks { 153 | gomega.Expect(c.Script).ToNot(gomega.BeEmpty(), "script is empty in "+string(c.Script)) 154 | } 155 | for _, l := range expected.Logs { 156 | out, _ := yaml.Marshal(l) 157 | gomega.Expect(l.LogQL).ToNot(gomega.BeEmpty(), "logQL is empty in "+string(out)) 158 | gomega.Expect(l.Equals == "" && l.Contains == nil && l.Regexp == "").To(gomega.BeFalse()) 159 | for _, s := range l.Contains { 160 | gomega.Expect(s).ToNot(gomega.BeEmpty(), "contains string is empty in "+string(out)) 161 | } 162 | } 163 | for _, d := range expected.Metrics { 164 | out, _ := yaml.Marshal(d) 165 | gomega.Expect(d.PromQL).ToNot(gomega.BeEmpty(), "promQL is empty in "+string(out)) 166 | gomega.Expect(d.Value).ToNot(gomega.BeEmpty(), "value is empty in "+string(out)) 167 | } 168 | for _, d := range expected.Traces { 169 | out, _ := yaml.Marshal(d) 170 | gomega.Expect(d.TraceQL).ToNot(gomega.BeEmpty(), "traceQL is empty in "+string(out)) 171 | gomega.Expect(d.Spans).ToNot(gomega.BeEmpty(), "spans are empty in "+string(out)) 172 | for _, span := range d.Spans { 173 | gomega.Expect(span.Name).ToNot(gomega.BeEmpty(), "span name is empty in "+string(out)) 174 | for k, v := range span.Attributes { 175 | gomega.Expect(k).ToNot(gomega.BeEmpty(), "attribute key is empty in "+string(out)) 176 | gomega.Expect(v).ToNot(gomega.BeEmpty(), "attribute value is empty in "+string(out)) 177 | } 178 | } 179 | } 180 | for _, p := range expected.Profiles { 181 | out, _ := yaml.Marshal(p) 182 | gomega.Expect(p.Query).ToNot(gomega.BeEmpty(), "query is empty in "+string(out)) 183 | gomega.Expect(p.Flamebearers.Contains).ToNot(gomega.BeEmpty(), "Flamebearers.contains is empty in "+string(out)) 184 | } 185 | 186 | if c.PortConfig == nil { 187 | // We're in non-parallel mode, so we can static ports here. 188 | c.PortConfig = &PortConfig{ 189 | ApplicationPort: 8080, 190 | GrafanaHTTPPort: 3000, 191 | PrometheusHTTPPort: 9090, 192 | LokiHTTPPort: 3100, 193 | TempoHTTPPort: 3200, 194 | PyroscopeHttpPort: 4040, 195 | } 196 | } 197 | 198 | slog.Info("ports", 199 | "grafana", c.PortConfig.GrafanaHTTPPort, 200 | "prometheus", c.PortConfig.PrometheusHTTPPort, 201 | "loki", c.PortConfig.LokiHTTPPort, 202 | "tempo", c.PortConfig.TempoHTTPPort, 203 | "pyroscope", c.PortConfig.PyroscopeHttpPort, 204 | "application", c.PortConfig.ApplicationPort) 205 | } 206 | 207 | func validateK8s(kubernetes *kubernetes.Kubernetes) { 208 | gomega.Expect(kubernetes.Dir).ToNot(gomega.BeEmpty(), "k8s-dir is empty") 209 | gomega.Expect(kubernetes.AppService).ToNot(gomega.BeEmpty(), "k8s-app-service is empty") 210 | gomega.Expect(kubernetes.AppDockerFile).ToNot(gomega.BeEmpty(), "app-docker-file is empty") 211 | gomega.Expect(kubernetes.AppDockerTag).ToNot(gomega.BeEmpty(), "app-docker-tag is empty") 212 | gomega.Expect(kubernetes.AppDockerPort).ToNot(gomega.BeZero(), "app-docker-port is zero") 213 | } 214 | 215 | func validateInput(input []Input) { 216 | for _, i := range input { 217 | gomega.Expect(i.Path).ToNot(gomega.BeEmpty(), "input path is empty") 218 | if i.Status != "" { 219 | _, err := strconv.ParseInt(i.Status, 10, 32) 220 | gomega.Expect(err).To(gomega.BeNil(), "status must parse as integer or be empty") 221 | } 222 | } 223 | } 224 | 225 | func validateDockerCompose(d *DockerCompose, dir string) { 226 | if len(d.Files) > 0 { 227 | for i, filename := range d.Files { 228 | d.Files[i] = filepath.Join(dir, filename) 229 | gomega.Expect(d.Files[i]).To(gomega.BeARegularFile()) 230 | } 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /yaml/profiles.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | import ( 4 | "encoding/json" 5 | "log/slog" 6 | 7 | "github.com/onsi/gomega" 8 | ) 9 | 10 | type PyroscopeQueryResponse struct { 11 | Flamebearer struct { 12 | Names []string `json:"names"` 13 | } `json:"flamebearer"` 14 | } 15 | 16 | func AssertPyroscope(r *runner, p ExpectedProfiles) { 17 | b, err := r.endpoint.SearchPyroscope(p.Query) 18 | r.LogQueryResult("query %v response %v err=%v\n", p.Query, string(b), err) 19 | g := r.gomegaInst 20 | g.Expect(err).ToNot(gomega.HaveOccurred()) 21 | assertPyroscopeResponse(b, p, r) 22 | } 23 | 24 | func assertPyroscopeResponse(b []byte, p ExpectedProfiles, r *runner) { 25 | g := r.gomegaInst 26 | g.Expect(len(b)).Should(gomega.BeNumerically(">", 0), "expected pyroscope response to be non-empty") 27 | 28 | response := PyroscopeQueryResponse{} 29 | err := json.Unmarshal(b, &response) 30 | if err != nil { 31 | slog.Info("error unmarshalling pyroscope", "response", string(b)) 32 | } 33 | 34 | g.Expect(err).ToNot(gomega.HaveOccurred()) 35 | g.Expect(response.Flamebearer.Names).To(gomega.ContainElement(gomega.ContainSubstring(p.Flamebearers.Contains))) 36 | } 37 | -------------------------------------------------------------------------------- /yaml/runner.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/grafana/oats/testhelpers/kubernetes" 7 | "github.com/grafana/oats/testhelpers/remote" 8 | "github.com/onsi/gomega/format" 9 | "log/slog" 10 | "os" 11 | "os/exec" 12 | "path/filepath" 13 | "regexp" 14 | "strconv" 15 | "time" 16 | 17 | "github.com/grafana/oats/testhelpers/compose" 18 | "github.com/grafana/oats/testhelpers/requests" 19 | "github.com/onsi/gomega" 20 | ) 21 | 22 | type runner struct { 23 | testCase *TestCase 24 | endpoint *remote.Endpoint 25 | deadline time.Time 26 | Verbose bool 27 | gomegaInst gomega.Gomega 28 | additionalAsserts []func() 29 | } 30 | 31 | var VerboseLogging bool 32 | 33 | func RunTestCase(c *TestCase) { 34 | format.MaxLength = 100000 35 | r := &runner{ 36 | testCase: c, 37 | } 38 | 39 | c.OutputDir = prepareBuildDir(c.Name) 40 | c.validateAndSetVariables() 41 | endpoint, err := startEndpoint(c) 42 | gomega.Expect(err).ToNot(gomega.HaveOccurred(), "expected no error starting an observability endpoint") 43 | 44 | r.deadline = time.Now().Add(c.Timeout) 45 | r.endpoint = endpoint 46 | if c.ManualDebug { 47 | slog.Info(fmt.Sprintf("topping to let you manually debug on http://localhost:%d\n", r.testCase.PortConfig.GrafanaHTTPPort)) 48 | 49 | for { 50 | r.eventually(func() { 51 | // do nothing - just feed input into the application 52 | }) 53 | time.Sleep(1 * time.Second) 54 | } 55 | } 56 | 57 | slog.Info("deadline", "time", r.deadline) 58 | 59 | defer func() { 60 | slog.Info("stopping observability endpoint") 61 | 62 | var ctx = context.Background() 63 | 64 | stopErr := r.endpoint.Stop(ctx) 65 | gomega.Expect(stopErr).ToNot(gomega.HaveOccurred(), "expected no error stopping the local observability endpoint") 66 | slog.Info("stopped observability endpoint") 67 | }() 68 | 69 | expected := c.Definition.Expected 70 | for _, composeLog := range expected.ComposeLogs { 71 | slog.Info("searching for compose log", "log", composeLog) 72 | r.eventually(func() { 73 | found, err := r.endpoint.SearchComposeLogs(composeLog) 74 | r.gomegaInst.Expect(err).ToNot(gomega.HaveOccurred()) 75 | r.gomegaInst.Expect(found).To(gomega.BeTrue()) 76 | }) 77 | } 78 | 79 | // Assert logs traces first, because metrics can take longer to appear 80 | // (depending on OTEL_METRIC_EXPORT_INTERVAL). 81 | for _, log := range expected.Logs { 82 | if r.MatchesMatrixCondition(log.MatrixCondition, log.LogQL) { 83 | slog.Info("searching loki", "logql", log.LogQL) 84 | r.eventually(func() { 85 | AssertLoki(r, log) 86 | }) 87 | } 88 | } 89 | for _, trace := range expected.Traces { 90 | if r.MatchesMatrixCondition(trace.MatrixCondition, trace.TraceQL) { 91 | slog.Info("searching tempo", "traceql", trace.TraceQL) 92 | r.eventually(func() { 93 | AssertTempo(r, trace) 94 | }) 95 | } 96 | } 97 | for _, metric := range expected.Metrics { 98 | if r.MatchesMatrixCondition(metric.MatrixCondition, metric.PromQL) { 99 | slog.Info("searching prometheus", "promql", metric.PromQL) 100 | r.eventually(func() { 101 | AssertProm(r, metric.PromQL, metric.Value) 102 | }) 103 | } 104 | } 105 | for _, profile := range expected.Profiles { 106 | if r.MatchesMatrixCondition(profile.MatrixCondition, profile.Query) { 107 | slog.Info("searching pyroscope", "query", profile.Query) 108 | r.eventually(func() { 109 | AssertPyroscope(r, profile) 110 | }) 111 | } 112 | } 113 | for _, customCheck := range expected.CustomChecks { 114 | if r.MatchesMatrixCondition(customCheck.MatrixCondition, customCheck.Script) { 115 | slog.Info("executing custom check", "check", customCheck.Script) 116 | r.eventually(func() { 117 | assertCustomCheck(r, customCheck) 118 | }) 119 | } 120 | } 121 | } 122 | 123 | func assertCustomCheck(r *runner, c CustomCheck) { 124 | r.LogQueryResult("running custom check %v\n", c.Script) 125 | cmd := exec.Command(c.Script) 126 | cmd.Dir = r.testCase.Dir 127 | cmd.Stdout = os.Stdout 128 | cmd.Stderr = os.Stderr 129 | 130 | err := cmd.Run() 131 | r.LogQueryResult("custom check %v response %v err=%v\n", c.Script, "", err) 132 | r.gomegaInst.Expect(err).ToNot(gomega.HaveOccurred()) 133 | } 134 | 135 | func startEndpoint(c *TestCase) (*remote.Endpoint, error) { 136 | ports := remote.PortsConfig{ 137 | PrometheusHTTPPort: c.PortConfig.PrometheusHTTPPort, 138 | TempoHTTPPort: c.PortConfig.TempoHTTPPort, 139 | LokiHttpPort: c.PortConfig.LokiHTTPPort, 140 | PyroscopeHttpPort: c.PortConfig.PyroscopeHttpPort, 141 | } 142 | 143 | slog.Info("start test", "name", c.Name) 144 | var endpoint *remote.Endpoint 145 | if c.Definition.Kubernetes != nil { 146 | endpoint = kubernetes.NewEndpoint(c.Definition.Kubernetes, ports, c.Name, c.Dir) 147 | } else { 148 | endpoint = compose.NewEndpoint(c.CreateDockerComposeFile(), ports) 149 | } 150 | 151 | var ctx = context.Background() 152 | startErr := endpoint.Start(ctx) 153 | return endpoint, startErr 154 | } 155 | 156 | func prepareBuildDir(name string) string { 157 | dir := filepath.Join(".", "build", name) 158 | 159 | fileinfo, err := os.Stat(dir) 160 | if err == nil { 161 | if fileinfo.IsDir() { 162 | err := os.RemoveAll(dir) 163 | gomega.Expect(err).ToNot(gomega.HaveOccurred(), "expected no error removing output directory") 164 | } 165 | } 166 | err = os.MkdirAll(dir, 0755) 167 | gomega.Expect(err).ToNot(gomega.HaveOccurred(), "expected no error creating output directory") 168 | return dir 169 | } 170 | 171 | func (r *runner) eventually(asserter func()) { 172 | gomega.Expect(time.Now()).Should(gomega.BeTemporally("<", r.deadline)) 173 | start := time.Now() 174 | printTime := start 175 | ctx := context.Background() 176 | interval := r.testCase.Definition.Interval 177 | if interval == 0 { 178 | interval = DefaultTestCaseInterval 179 | } 180 | iterations := 0 181 | r.additionalAsserts = nil 182 | gomega.Eventually(ctx, func(g gomega.Gomega) { 183 | verbose := VerboseLogging 184 | if iterations == 0 || time.Since(printTime) > 10*time.Second { 185 | verbose = true 186 | printTime = time.Now() 187 | } 188 | iterations++ 189 | r.Verbose = verbose 190 | r.LogQueryResult("waiting for telemetry data\n") 191 | 192 | for _, i := range r.testCase.Definition.Input { 193 | url := fmt.Sprintf("http://localhost:%d%s", r.testCase.PortConfig.ApplicationPort, i.Path) 194 | status := 200 195 | if i.Status != "" { 196 | parsedStatus, err := strconv.ParseInt(i.Status, 10, 64) 197 | if err == nil { 198 | status = int(parsedStatus) 199 | } 200 | } 201 | err := requests.DoHTTPGet(url, status) 202 | g.Expect(err).ToNot(gomega.HaveOccurred(), "expected no error calling application endpoint %s", url) 203 | } 204 | 205 | r.gomegaInst = g 206 | asserter() 207 | }).WithTimeout(time.Until(r.deadline)).WithPolling(interval).Should(gomega.Succeed(), "calling application for %v should cause telemetry to appear", r.testCase.Timeout) 208 | slog.Info(fmt.Sprintf("time to get telemetry data: %v", time.Since(start))) 209 | for _, a := range r.additionalAsserts { 210 | a() 211 | } 212 | } 213 | 214 | func (r *runner) MatchesMatrixCondition(matrixCondition string, subject string) bool { 215 | if matrixCondition == "" { 216 | return true 217 | } 218 | name := r.testCase.MatrixTestCaseName 219 | if name == "" { 220 | slog.Info("matrix condition ignored we're not in a matrix test", "condition", matrixCondition) 221 | return true 222 | } 223 | if regexp.MustCompile(matrixCondition).MatchString(name) { 224 | return true 225 | } 226 | slog.Info("matrix condition not matched - ignoring assertion", 227 | "test case", r.testCase.Name, 228 | "name", name, 229 | "subject", subject) 230 | return false 231 | } 232 | -------------------------------------------------------------------------------- /yaml/testcase.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "gopkg.in/yaml.v3" 7 | "log/slog" 8 | "os" 9 | "path/filepath" 10 | "regexp" 11 | "strings" 12 | ) 13 | 14 | var oatsFileRegex = regexp.MustCompile(`oats.*\.yaml`) 15 | 16 | func ReadTestCases(base string) ([]*TestCase, string) { 17 | if base == "" { 18 | return []*TestCase{}, "" 19 | } 20 | 21 | base = absolutePath(base) 22 | 23 | cases, err := collectTestCases(base, true) 24 | if err != nil { 25 | panic(err) 26 | } 27 | 28 | return cases, base 29 | } 30 | 31 | func collectTestCases(base string, evaluateIgnoreFile bool) ([]*TestCase, error) { 32 | var cases []*TestCase 33 | var ignored []string 34 | err := filepath.WalkDir(base, func(p string, d os.DirEntry, err error) error { 35 | if err != nil { 36 | return err 37 | } 38 | 39 | if evaluateIgnoreFile { 40 | if d.IsDir() { 41 | if _, err := os.Stat(filepath.Join(p, ".oatsignore")); errors.Is(err, os.ErrNotExist) { 42 | // ignore file does not exist 43 | } else { 44 | // ignore file exists 45 | slog.Info("ignoring", "path", p) 46 | ignored = append(ignored, p) 47 | return nil 48 | } 49 | } 50 | } 51 | 52 | if !oatsFileRegex.MatchString(d.Name()) || strings.Contains(d.Name(), "-template.yaml") { 53 | return nil 54 | } 55 | 56 | for _, i := range ignored { 57 | if strings.HasPrefix(p, i) { 58 | return nil 59 | } 60 | } 61 | 62 | testCase, err := readTestCase(base, p) 63 | if err != nil { 64 | return err 65 | } 66 | if testCase.Definition.Matrix != nil { 67 | for _, matrix := range testCase.Definition.Matrix { 68 | newCase := testCase 69 | newCase.Definition = testCase.Definition 70 | newCase.Definition.DockerCompose = matrix.DockerCompose 71 | newCase.Definition.Kubernetes = matrix.Kubernetes 72 | newCase.Name = fmt.Sprintf("%s-%s", testCase.Name, matrix.Name) 73 | newCase.MatrixTestCaseName = matrix.Name 74 | cases = append(cases, &newCase) 75 | } 76 | return nil 77 | } 78 | cases = append(cases, &testCase) 79 | return nil 80 | }) 81 | return cases, err 82 | } 83 | 84 | func absolutePath(dir string) string { 85 | abs, err := filepath.Abs(dir) 86 | if err != nil { 87 | panic(err) 88 | } 89 | return abs 90 | } 91 | 92 | func readTestCase(testBase, filePath string) (TestCase, error) { 93 | def, err := readTestCaseDefinition(filePath) 94 | if err != nil { 95 | return TestCase{}, err 96 | } 97 | 98 | dir := filepath.Dir(absolutePath(filePath)) 99 | name := strings.TrimPrefix(dir, absolutePath(testBase)) + "-" + strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath)) 100 | sep := string(filepath.Separator) 101 | name = strings.TrimPrefix(name, sep) 102 | name = strings.ReplaceAll(name, sep, "-") 103 | name = "run" + name 104 | testCase := TestCase{ 105 | Name: name, 106 | Dir: dir, 107 | Definition: def, 108 | } 109 | return testCase, nil 110 | } 111 | 112 | func readTestCaseDefinition(filePath string) (TestCaseDefinition, error) { 113 | filePath = absolutePath(filePath) 114 | def := TestCaseDefinition{} 115 | content, err := os.ReadFile(filePath) 116 | if err != nil { 117 | return TestCaseDefinition{}, err 118 | } 119 | 120 | err = yaml.Unmarshal(content, &def) 121 | if err != nil { 122 | return TestCaseDefinition{}, err 123 | } 124 | 125 | for _, s := range def.Include { 126 | p := includePath(filePath, s) 127 | other, err := readTestCaseDefinition(p) 128 | if err != nil { 129 | return TestCaseDefinition{}, err 130 | } 131 | def.Merge(other) 132 | } 133 | def.Include = []string{} 134 | 135 | return def, nil 136 | } 137 | 138 | func includePath(filePath string, include string) string { 139 | dir := filepath.Dir(filePath) 140 | fromSlash := filepath.FromSlash(include) 141 | return filepath.Join(dir, fromSlash) 142 | } 143 | -------------------------------------------------------------------------------- /yaml/testcase_test.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | import ( 4 | "github.com/stretchr/testify/require" 5 | "path/filepath" 6 | "testing" 7 | ) 8 | 9 | func TestReadTestCaseDefinition(t *testing.T) { 10 | def, err := readTestCaseDefinition("testdata/foo/oats.yaml") 11 | require.NoError(t, err) 12 | merged, err := readTestCaseDefinition("testdata/oats-merged.yaml") 13 | require.NoError(t, err) 14 | require.Equal(t, merged, def) 15 | } 16 | 17 | func TestReadTestCase(t *testing.T) { 18 | tc, err := readTestCase("testdata", "testdata/foo/oats.yaml") 19 | require.NoError(t, err) 20 | require.Equal(t, "runfoo-oats", tc.Name) 21 | require.Equal(t, absolutePath("testdata/foo"), tc.Dir) 22 | } 23 | 24 | func TestIncludePath(t *testing.T) { 25 | require.Equal(t, 26 | filepath.FromSlash("/home/gregor/source/grafana-opentelemetry-java/examples/jdbc/oats-non-reactive.yaml"), 27 | includePath("/home/gregor/source/grafana-opentelemetry-java/examples/jdbc/spring-boot-non-reactive-2.7/oats.yaml", "../oats-non-reactive.yaml")) 28 | } 29 | 30 | func TestCollectTestCases(t *testing.T) { 31 | cases, err := collectTestCases("testdata", false) 32 | require.NoError(t, err) 33 | require.Len(t, cases, 2) 34 | require.Equal(t, "runfoo-oats", cases[0].Name) 35 | require.Equal(t, "run-oats-merged", cases[1].Name) 36 | } 37 | -------------------------------------------------------------------------------- /yaml/testdata/.oatsignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grafana/oats/27efab9d582aee0b75d70914c3098abd0db3c075/yaml/testdata/.oatsignore -------------------------------------------------------------------------------- /yaml/testdata/docker-compose-addition.yaml: -------------------------------------------------------------------------------- 1 | version: '3.9' 2 | 3 | x-default-logging: &logging 4 | driver: "json-file" 5 | options: 6 | max-size: "5m" 7 | max-file: "2" 8 | 9 | services: 10 | mongodb: 11 | image: mongo:6-jammy 12 | ports: 13 | - '27017:27017' 14 | volumes: 15 | - dbdata6:/data/db 16 | logging: *logging 17 | volumes: 18 | dbdata6: 19 | -------------------------------------------------------------------------------- /yaml/testdata/docker-compose-expected.yaml: -------------------------------------------------------------------------------- 1 | networks: 2 | default: 3 | driver: bridge 4 | ipam: 5 | config: 6 | - subnet: 172.16.57.0/24 7 | services: 8 | grafana: 9 | image: grafana/grafana:10.0.0 10 | network_mode: host 11 | volumes: 12 | - ./configs/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/grafana-datasources.yaml 13 | mongodb: 14 | image: mongo:6-jammy 15 | logging: 16 | driver: json-file 17 | options: 18 | max-file: "2" 19 | max-size: 5m 20 | ports: 21 | - 27017:27017 22 | volumes: 23 | - dbdata6:/data/db 24 | version: "3.9" 25 | volumes: 26 | dbdata6: null 27 | x-default-logging: 28 | driver: json-file 29 | options: 30 | max-file: "2" 31 | max-size: 5m 32 | -------------------------------------------------------------------------------- /yaml/testdata/docker-compose-template.yaml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | networks: # see https://stackoverflow.com/questions/43720339/docker-error-could-not-find-an-available-non-overlapping-ipv4-address-pool-am 3 | default: 4 | driver: bridge 5 | ipam: 6 | config: 7 | - subnet: 172.16.57.0/24 8 | services: 9 | grafana: 10 | image: grafana/grafana:10.0.0 11 | network_mode: host 12 | volumes: 13 | - ./configs/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/grafana-datasources.yaml 14 | -------------------------------------------------------------------------------- /yaml/testdata/foo/oats.yaml: -------------------------------------------------------------------------------- 1 | include: 2 | - ../oats-template.yaml 3 | expected: 4 | traces: 5 | - traceql: '{ name =~ "SELECT .*cart"}' 6 | spans: 7 | - name: 'regex:SELECT .*cart' 8 | attributes: 9 | db.system: h2 10 | metrics: 11 | - promql: foo 12 | value: '>= 0' 13 | 14 | -------------------------------------------------------------------------------- /yaml/testdata/oats-merged.yaml: -------------------------------------------------------------------------------- 1 | docker-compose: 2 | input: 3 | - path: /stock 4 | status: 200 5 | expected: 6 | traces: 7 | - traceql: '{ name =~ "SELECT .*cart"}' 8 | spans: 9 | - name: 'regex:SELECT .*cart' 10 | attributes: 11 | db.system: h2 12 | - traceql: '{ name =~ "SELECT .*product"}' 13 | spans: 14 | - name: 'regex:SELECT .*' 15 | attributes: 16 | db.system: h2 17 | metrics: 18 | - promql: foo 19 | value: '>= 0' 20 | - promql: bar 21 | value: '>= 0' 22 | -------------------------------------------------------------------------------- /yaml/testdata/oats-template.yaml: -------------------------------------------------------------------------------- 1 | docker-compose: 2 | input: 3 | - path: /stock 4 | status: 200 5 | expected: 6 | traces: 7 | - traceql: '{ name =~ "SELECT .*product"}' 8 | spans: 9 | - name: 'regex:SELECT .*' 10 | attributes: 11 | db.system: h2 12 | metrics: 13 | - promql: bar 14 | value: '>= 0' 15 | -------------------------------------------------------------------------------- /yaml/traces.go: -------------------------------------------------------------------------------- 1 | package yaml 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/grafana/oats/testhelpers/tempo/responses" 7 | "github.com/onsi/gomega" 8 | "go.opentelemetry.io/collector/pdata/pcommon" 9 | ) 10 | 11 | func AssertTempo(r *runner, t ExpectedTraces) { 12 | ctx := context.Background() 13 | 14 | b, err := r.endpoint.SearchTempo(ctx, t.TraceQL) 15 | r.LogQueryResult("traceQL query %v response %v err=%v\n", t.TraceQL, string(b), err) 16 | g := r.gomegaInst 17 | g.Expect(err).ToNot(gomega.HaveOccurred()) 18 | g.Expect(len(b)).Should(gomega.BeNumerically(">", 0)) 19 | 20 | res, err := responses.ParseTempoSearchResult(b) 21 | g.Expect(err).ToNot(gomega.HaveOccurred()) 22 | g.Expect(res.Traces).ToNot(gomega.BeEmpty()) 23 | 24 | assertTrace(r, res.Traces[0], t.Spans) 25 | } 26 | 27 | func assertTrace(r *runner, tr responses.Trace, wantSpans []ExpectedSpan) { 28 | ctx := context.Background() 29 | 30 | b, err := r.endpoint.GetTraceByID(ctx, tr.TraceID) 31 | r.LogQueryResult("traceQL traceID %v response %v err=%v\n", tr.TraceID, string(b), err) 32 | 33 | g := r.gomegaInst 34 | g.Expect(err).ToNot(gomega.HaveOccurred(), "we should find the trace by traceID") 35 | g.Expect(len(b)).Should(gomega.BeNumerically(">", 0)) 36 | 37 | td, err := responses.ParseTraceDetails(b) 38 | g.Expect(err).ToNot(gomega.HaveOccurred(), "we should be able to parse the GET trace by traceID API output") 39 | 40 | for _, wantSpan := range wantSpans { 41 | spans, atts := responses.FindSpansWithAttributes(td, wantSpan.Name) 42 | if wantSpan.AllowDups { 43 | g.Expect(len(spans)).Should(gomega.BeNumerically(">", 0), "we should find at least one span with the name %s", wantSpan.Name) 44 | } else { 45 | g.Expect(spans).To(gomega.HaveLen(1), "we should find a single span with the name %s", wantSpan.Name) 46 | } 47 | 48 | for k, v := range wantSpan.Attributes { 49 | for k, v := range spans[0].Attributes().AsRaw() { 50 | atts[k] = v 51 | } 52 | m := pcommon.NewMap() 53 | err = m.FromRaw(atts) 54 | g.Expect(err).ToNot(gomega.HaveOccurred(), "we should be able to convert the map to a pdata.Map") 55 | err := responses.MatchTraceAttribute(m, pcommon.ValueTypeStr, k, v) 56 | g.Expect(err).ToNot(gomega.HaveOccurred(), "span attribute should match") 57 | } 58 | } 59 | } 60 | --------------------------------------------------------------------------------