├── .errcheck-exclude ├── .github └── workflows │ └── ci.yaml ├── .golangci.yml ├── LICENSE ├── Makefile ├── README.md ├── cache └── cache.go ├── composite_service.go ├── db ├── db.go └── kafka.go ├── go.mod ├── go.sum ├── images └── images.go ├── logger.go ├── metrics.go ├── metrics_options.go ├── scenario.go ├── scenario_test.go ├── service.go ├── service_test.go ├── util.go └── util_test.go /.errcheck-exclude: -------------------------------------------------------------------------------- 1 | (github.com/go-kit/log.Logger).Log 2 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: ci 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | branches: 8 | - main 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | test: 15 | runs-on: ubuntu-24.04 16 | steps: 17 | - uses: actions/checkout@v2 18 | with: 19 | persist-credentials: false 20 | - uses: actions/setup-go@v2 21 | with: 22 | go-version: "1.19" 23 | - name: Go test 24 | run: make test 25 | lint: 26 | runs-on: ubuntu-24.04 27 | steps: 28 | - uses: actions/checkout@v2 29 | with: 30 | persist-credentials: false 31 | - uses: actions/setup-go@v2 32 | with: 33 | go-version: "1.19" 34 | - name: Lint 35 | run: make lint 36 | 37 | # integration is run as a GitHub Action, as to run those tests we require a 38 | # docker socket to be available so containers can be run. 39 | integration: 40 | runs-on: ubuntu-24.04 41 | steps: 42 | - uses: actions/checkout@v2 43 | with: 44 | persist-credentials: false 45 | - uses: actions/setup-go@v2 46 | with: 47 | go-version: "1.19" 48 | - name: Install Docker Client 49 | run: | 50 | sudo bash <<"EOF" 51 | set -xeuo pipefail 52 | VER="18.06.3-ce" 53 | curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz 54 | echo "346f9394393ee8db5f8bd1e229ee9d90e5b36931bdd754308b2ae68884dd6822 /tmp/docker-$VER.tgz" | sha256sum -c 55 | tar -xz -C /tmp -f /tmp/docker-$VER.tgz 56 | mv /tmp/docker/* /usr/bin 57 | EOF 58 | - name: Integration Tests 59 | run: | 60 | make integration 61 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | output: 2 | format: line-number 3 | 4 | linters: 5 | enable: 6 | - goimports 7 | - gofmt 8 | - misspell 9 | - revive 10 | 11 | linters-settings: 12 | errcheck: 13 | # path to a file containing a list of functions to exclude from checking 14 | # see https://github.com/kisielk/errcheck#excluding-functions for details 15 | exclude: ./.errcheck-exclude 16 | goimports: 17 | local-prefixes: "github.com/grafana/e2e" 18 | 19 | run: 20 | timeout: 5m 21 | 22 | # List of build tags, all linters use it. 23 | build-tags: 24 | - netgo 25 | - requires_docker 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2021 Grafana Labs 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test 2 | test: 3 | go test -tags netgo -timeout 30m -race -count 1 ./... 4 | 5 | .PHONY: lint 6 | lint: 7 | go run github.com/client9/misspell/cmd/misspell@v0.3.4 -error README.md LICENSE 8 | 9 | # Configured via .golangci.yml. 10 | go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.43.0 run 11 | 12 | .PHONY: integration 13 | integration: 14 | go test -tags netgo,requires_docker -timeout 30m -v -count=1 ./ 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Grafana E2E 2 | 3 | This library contains utilities that are useful for running E2E tests using docker containers. 4 | 5 | ## License 6 | 7 | [Apache 2.0 License](https://github.com/grafana/dskit/blob/main/LICENSE) 8 | -------------------------------------------------------------------------------- /cache/cache.go: -------------------------------------------------------------------------------- 1 | package e2ecache 2 | 3 | import ( 4 | "github.com/grafana/e2e" 5 | "github.com/grafana/e2e/images" 6 | ) 7 | 8 | const ( 9 | MemcachedPort = 11211 10 | RedisPort = 6379 11 | ) 12 | 13 | func NewMemcached() *e2e.ConcreteService { 14 | return e2e.NewConcreteService( 15 | "memcached", 16 | images.Memcached, 17 | nil, 18 | e2e.NewTCPReadinessProbe(MemcachedPort), 19 | MemcachedPort, 20 | ) 21 | } 22 | 23 | func NewRedis() *e2e.ConcreteService { 24 | return e2e.NewConcreteService( 25 | "redis", 26 | images.Redis, 27 | // Set a memory limit, eviction policy, and disable persistence since Redis 28 | // is used as a cache not a database. 64mb is picked for parity with the Memcached 29 | // default memory limit. 30 | e2e.NewCommand( 31 | "redis-server", 32 | "--maxmemory", "64mb", 33 | "--maxmemory-policy", "allkeys-lru", 34 | "--save", "''", 35 | "--appendonly", "no", 36 | ), 37 | e2e.NewTCPReadinessProbe(RedisPort), 38 | RedisPort, 39 | ) 40 | } 41 | -------------------------------------------------------------------------------- /composite_service.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/grafana/dskit/backoff" 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | // CompositeHTTPService abstract an higher-level service composed, under the hood, 13 | // by 2+ HTTPService. 14 | type CompositeHTTPService struct { 15 | services []*HTTPService 16 | 17 | // Generic retry backoff. 18 | retryBackoff *backoff.Backoff 19 | } 20 | 21 | func NewCompositeHTTPService(services ...*HTTPService) *CompositeHTTPService { 22 | return &CompositeHTTPService{ 23 | services: services, 24 | retryBackoff: backoff.New(context.Background(), backoff.Config{ 25 | MinBackoff: 300 * time.Millisecond, 26 | MaxBackoff: 600 * time.Millisecond, 27 | MaxRetries: 50, // Sometimes the CI is slow ¯\_(ツ)_/¯ 28 | }), 29 | } 30 | } 31 | 32 | func (s *CompositeHTTPService) NumInstances() int { 33 | return len(s.services) 34 | } 35 | 36 | func (s *CompositeHTTPService) Instances() []*HTTPService { 37 | return s.services 38 | } 39 | 40 | // WaitSumMetrics waits for at least one instance of each given metric names to be present and their sums, returning true 41 | // when passed to given isExpected(...). 42 | func (s *CompositeHTTPService) WaitSumMetrics(isExpected func(sums ...float64) bool, metricNames ...string) error { 43 | return s.WaitSumMetricsWithOptions(isExpected, metricNames) 44 | } 45 | 46 | func (s *CompositeHTTPService) WaitSumMetricsWithOptions(isExpected func(sums ...float64) bool, metricNames []string, opts ...MetricsOption) error { 47 | var ( 48 | sums []float64 49 | err error 50 | options = buildMetricsOptions(opts) 51 | ) 52 | 53 | for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); { 54 | sums, err = s.SumMetrics(metricNames, opts...) 55 | if options.WaitMissingMetrics && errors.Is(err, errMissingMetric) { 56 | continue 57 | } 58 | if err != nil { 59 | return err 60 | } 61 | 62 | if isExpected(sums...) { 63 | return nil 64 | } 65 | 66 | s.retryBackoff.Wait() 67 | } 68 | 69 | return fmt.Errorf("unable to find metrics %s with expected values. Last error: %v. Last values: %v", metricNames, err, sums) 70 | } 71 | 72 | // SumMetrics returns the sum of the values of each given metric names. 73 | func (s *CompositeHTTPService) SumMetrics(metricNames []string, opts ...MetricsOption) ([]float64, error) { 74 | sums := make([]float64, len(metricNames)) 75 | 76 | for _, service := range s.services { 77 | partials, err := service.SumMetrics(metricNames, opts...) 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | if len(partials) != len(sums) { 83 | return nil, fmt.Errorf("unexpected mismatching sum metrics results (got %d, expected %d)", len(partials), len(sums)) 84 | } 85 | 86 | for i := 0; i < len(sums); i++ { 87 | sums[i] += partials[i] 88 | } 89 | } 90 | 91 | return sums, nil 92 | } 93 | -------------------------------------------------------------------------------- /db/db.go: -------------------------------------------------------------------------------- 1 | package e2edb 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "strings" 7 | 8 | "github.com/grafana/e2e" 9 | "github.com/grafana/e2e/images" 10 | ) 11 | 12 | const ( 13 | MinioAccessKey = "Cheescake" 14 | MinioSecretKey = "supersecret" 15 | ) 16 | 17 | // NewMinio returns minio server, used as a local replacement for S3. 18 | func NewMinio(port int, bktNames ...string) *e2e.HTTPService { 19 | return newMinio(port, map[string]string{}, bktNames...) 20 | } 21 | 22 | // NewMinioWithKES returns minio server, configured to talk to a KES service. 23 | func NewMinioWithKES(port int, kesEndpoint, rootKeyFile, rootCertFile, caCertFile string, bktNames ...string) *e2e.HTTPService { 24 | kesEnvVars := map[string]string{ 25 | "MINIO_KMS_KES_ENDPOINT": kesEndpoint, 26 | "MINIO_KMS_KES_KEY_FILE": filepath.Join(e2e.ContainerSharedDir, rootKeyFile), 27 | "MINIO_KMS_KES_CERT_FILE": filepath.Join(e2e.ContainerSharedDir, rootCertFile), 28 | "MINIO_KMS_KES_CAPATH": filepath.Join(e2e.ContainerSharedDir, caCertFile), 29 | "MINIO_KMS_KES_KEY_NAME": "my-minio-key", 30 | } 31 | return newMinio(port, kesEnvVars, bktNames...) 32 | } 33 | 34 | func newMinio(port int, envVars map[string]string, bktNames ...string) *e2e.HTTPService { 35 | commands := []string{} 36 | for _, bkt := range bktNames { 37 | commands = append(commands, fmt.Sprintf("mkdir -p /data/%s", bkt)) 38 | } 39 | commands = append(commands, fmt.Sprintf("minio server --address :%v --quiet /data", port)) 40 | 41 | m := e2e.NewHTTPService( 42 | fmt.Sprintf("minio-%v", port), 43 | images.Minio, 44 | // Create the buckets before starting minio 45 | e2e.NewCommandWithoutEntrypoint("sh", "-c", strings.Join(commands, " && ")), 46 | e2e.NewHTTPReadinessProbe(port, "/minio/health/ready", 200, 200), 47 | port, 48 | ) 49 | envVars["MINIO_ACCESS_KEY"] = MinioAccessKey 50 | envVars["MINIO_SECRET_KEY"] = MinioSecretKey 51 | envVars["MINIO_BROWSER"] = "off" 52 | envVars["ENABLE_HTTPS"] = "0" 53 | m.SetEnvVars(envVars) 54 | return m 55 | } 56 | 57 | // NewKES returns KES server, used as a local key management store 58 | func NewKES(port int, serverName, serverKeyFile, serverCertFile, clientKeyFile, clientCertFile, rootCertFile, hostSharedDir string) (*e2e.HTTPService, error) { 59 | // Run this as a shell command, so sub-shell can evaluate 'identity' of root user. 60 | command := fmt.Sprintf("/kes server --addr 0.0.0.0:%d --key=%s --cert=%s --root=$(/kes tool identity of %s) --auth=off", 61 | port, filepath.Join(e2e.ContainerSharedDir, serverKeyFile), filepath.Join(e2e.ContainerSharedDir, serverCertFile), filepath.Join(e2e.ContainerSharedDir, clientCertFile)) 62 | 63 | readinessProbe, err := e2e.NewHTTPSReadinessProbe(port, "/v1/status", serverName, filepath.Join(hostSharedDir, clientKeyFile), filepath.Join(hostSharedDir, clientCertFile), filepath.Join(hostSharedDir, rootCertFile), 200, 200) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | return e2e.NewHTTPService( 69 | "kes", 70 | images.KES, 71 | e2e.NewCommandWithoutEntrypoint("sh", "-c", command), 72 | readinessProbe, 73 | port, 74 | ), nil 75 | } 76 | 77 | func NewConsul() *e2e.HTTPService { 78 | return e2e.NewHTTPService( 79 | "consul", 80 | images.Consul, 81 | // Run consul in "dev" mode so that the initial leader election is immediate 82 | e2e.NewCommand("agent", "-server", "-client=0.0.0.0", "-dev", "-log-level=err"), 83 | e2e.NewHTTPReadinessProbe(8500, "/v1/operator/autopilot/health", 200, 200, `"Healthy": true`), 84 | 8500, 85 | ) 86 | } 87 | 88 | func NewETCD() *e2e.HTTPService { 89 | return e2e.NewHTTPService( 90 | "etcd", 91 | images.ETCD, 92 | e2e.NewCommand("/usr/local/bin/etcd", "--listen-client-urls=http://0.0.0.0:2379", "--advertise-client-urls=http://0.0.0.0:2379", "--listen-metrics-urls=http://0.0.0.0:9000", "--log-level=error"), 93 | e2e.NewHTTPReadinessProbe(9000, "/health", 200, 204), 94 | 2379, 95 | 9000, // Metrics 96 | ) 97 | } 98 | 99 | func NewDynamoDB() *e2e.HTTPService { 100 | return e2e.NewHTTPService( 101 | "dynamodb", 102 | images.DynamoDB, 103 | e2e.NewCommand("-jar", "DynamoDBLocal.jar", "-inMemory", "-sharedDb"), 104 | // DynamoDB doesn't have a readiness probe, so we check if the / works even if returns 400 105 | e2e.NewHTTPReadinessProbe(8000, "/", 400, 400), 106 | 8000, 107 | ) 108 | } 109 | -------------------------------------------------------------------------------- /db/kafka.go: -------------------------------------------------------------------------------- 1 | package e2edb 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/twmb/franz-go/pkg/kadm" 10 | "github.com/twmb/franz-go/pkg/kgo" 11 | 12 | "github.com/grafana/e2e" 13 | "github.com/grafana/e2e/images" 14 | ) 15 | 16 | type KafkaService struct { 17 | *e2e.HTTPService 18 | } 19 | 20 | func NewKafka() *KafkaService { 21 | return &KafkaService{ 22 | HTTPService: e2e.NewHTTPService( 23 | "kafka", 24 | images.Kafka, 25 | nil, // No custom command. 26 | NewKafkaReadinessProbe(9092), 27 | 9092, 28 | ), 29 | } 30 | } 31 | 32 | func (s *KafkaService) Start(networkName, sharedDir string) (err error) { 33 | // Configures Kafka right before starting it so that we have the networkName to correctly compute 34 | // the advertised host. 35 | s.HTTPService.SetEnvVars(map[string]string{ 36 | // Configure Kafka to run in KRaft mode (without Zookeeper). 37 | "CLUSTER_ID": "NqnEdODVKkiLTfJvqd1uqQ==", // A random ID (16 bytes of a base64-encoded UUID). 38 | "KAFKA_BROKER_ID": "1", 39 | "KAFKA_NODE_ID": "1", 40 | "KAFKA_LISTENERS": "PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093,PLAINTEXT_HOST://localhost:29092", // Host and port to which Kafka binds to for listening. 41 | "KAFKA_PROCESS_ROLES": "broker,controller", 42 | "KAFKA_CONTROLLER_QUORUM_VOTERS": "1@kafka:29093", 43 | "KAFKA_CONTROLLER_LISTENER_NAMES": "CONTROLLER", 44 | 45 | // Configure the advertised host and post. 46 | "KAFKA_ADVERTISED_LISTENERS": fmt.Sprintf("PLAINTEXT://%s-kafka:9092,PLAINTEXT_HOST://localhost:29092", networkName), 47 | 48 | // RF=1. 49 | "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR": "1", 50 | "KAFKA_TRANSACTION_STATE_LOG_MIN_ISR": "1", 51 | "KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR": "1", 52 | "KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS": "0", 53 | 54 | // No TLS. 55 | "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT", 56 | "KAFKA_INTER_BROKER_LISTENER_NAME": "PLAINTEXT", 57 | 58 | // Enough partitions for integration tests. 59 | "KAFKA_NUM_PARTITIONS": "3", 60 | 61 | "LOG4J_ROOT_LOGLEVEL": "WARN", 62 | }) 63 | 64 | return s.HTTPService.Start(networkName, sharedDir) 65 | } 66 | 67 | // KafkaReadinessProbe checks readiness by ensure a Kafka broker is up and running. 68 | type KafkaReadinessProbe struct { 69 | port int 70 | } 71 | 72 | func NewKafkaReadinessProbe(port int) *KafkaReadinessProbe { 73 | return &KafkaReadinessProbe{ 74 | port: port, 75 | } 76 | } 77 | 78 | func (p *KafkaReadinessProbe) Ready(service *e2e.ConcreteService) (err error) { 79 | const timeout = time.Second 80 | 81 | endpoint := service.Endpoint(p.port) 82 | if endpoint == "" { 83 | return fmt.Errorf("cannot get service endpoint for port %d", p.port) 84 | } else if endpoint == "stopped" { 85 | return errors.New("service has stopped") 86 | } 87 | 88 | client, err := kgo.NewClient(kgo.SeedBrokers(endpoint), kgo.DialTimeout(timeout)) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | // Ensure we close the client once done. 94 | defer client.Close() 95 | 96 | admin := kadm.NewClient(client) 97 | 98 | ctxWithTimeout, cancel := context.WithTimeout(context.Background(), timeout) 99 | defer cancel() 100 | 101 | _, err = admin.ApiVersions(ctxWithTimeout) 102 | return err 103 | } 104 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/grafana/e2e 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/go-kit/log v0.2.1 7 | github.com/grafana/dskit v0.0.0-20230914143233-4b32fbf08128 8 | github.com/pkg/errors v0.9.1 9 | github.com/prometheus/client_model v0.4.0 10 | github.com/prometheus/common v0.44.0 11 | github.com/prometheus/prometheus v0.44.0 12 | github.com/stretchr/testify v1.8.2 13 | github.com/thanos-io/objstore v0.0.0-20220809103346-8ef1f215e2bf 14 | github.com/twmb/franz-go v1.15.4 15 | github.com/twmb/franz-go/pkg/kadm v1.10.0 16 | gopkg.in/yaml.v3 v3.0.1 17 | ) 18 | 19 | require ( 20 | cloud.google.com/go v0.110.2 // indirect 21 | cloud.google.com/go/iam v1.1.2 // indirect 22 | cloud.google.com/go/storage v1.29.0 // indirect 23 | github.com/aws/aws-sdk-go-v2 v1.16.0 // indirect 24 | github.com/aws/aws-sdk-go-v2/config v1.15.1 // indirect 25 | github.com/aws/aws-sdk-go-v2/credentials v1.11.0 // indirect 26 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.1 // indirect 27 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.7 // indirect 28 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.1 // indirect 29 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.8 // indirect 30 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.1 // indirect 31 | github.com/aws/aws-sdk-go-v2/service/sso v1.11.1 // indirect 32 | github.com/aws/aws-sdk-go-v2/service/sts v1.16.1 // indirect 33 | github.com/aws/smithy-go v1.11.1 // indirect 34 | github.com/beorn7/perks v1.0.1 // indirect 35 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 36 | github.com/davecgh/go-spew v1.1.1 // indirect 37 | github.com/dustin/go-humanize v1.0.0 // indirect 38 | github.com/efficientgo/tools/core v0.0.0-20220225185207-fe763185946b // indirect 39 | github.com/go-logfmt/logfmt v0.6.0 // indirect 40 | github.com/gogo/protobuf v1.3.2 // indirect 41 | github.com/golang/protobuf v1.5.3 // indirect 42 | github.com/google/uuid v1.3.0 // indirect 43 | github.com/googleapis/gax-go/v2 v2.11.0 // indirect 44 | github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect 45 | github.com/json-iterator/go v1.1.12 // indirect 46 | github.com/klauspost/compress v1.17.0 // indirect 47 | github.com/klauspost/cpuid v1.3.1 // indirect 48 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 49 | github.com/minio/md5-simd v1.1.0 // indirect 50 | github.com/minio/minio-go/v7 v7.0.23 // indirect 51 | github.com/minio/sha256-simd v0.1.1 // indirect 52 | github.com/mitchellh/go-homedir v1.1.0 // indirect 53 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 54 | github.com/modern-go/reflect2 v1.0.2 // indirect 55 | github.com/opentracing/opentracing-go v1.2.0 // indirect 56 | github.com/pierrec/lz4/v4 v4.1.19 // indirect 57 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 58 | github.com/prometheus/client_golang v1.15.1 // indirect 59 | github.com/prometheus/procfs v0.9.0 // indirect 60 | github.com/rs/xid v1.2.1 // indirect 61 | github.com/sirupsen/logrus v1.8.1 // indirect 62 | github.com/twmb/franz-go/pkg/kmsg v1.7.0 // indirect 63 | go.uber.org/goleak v1.2.1 // indirect 64 | golang.org/x/crypto v0.17.0 // indirect 65 | golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect 66 | golang.org/x/net v0.10.0 // indirect 67 | golang.org/x/sync v0.3.0 // indirect 68 | golang.org/x/sys v0.15.0 // indirect 69 | golang.org/x/text v0.14.0 // indirect 70 | google.golang.org/api v0.126.0 // indirect 71 | google.golang.org/protobuf v1.31.0 // indirect 72 | gopkg.in/ini.v1 v1.67.0 // indirect 73 | gopkg.in/yaml.v2 v2.4.0 // indirect 74 | ) 75 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= 2 | cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= 3 | cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds= 4 | cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= 5 | cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= 6 | cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= 7 | cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= 8 | cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= 9 | github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= 10 | github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.0 h1:Ut0ZGdOwJDw0npYEg+TLlPls3Pq6JiZaP2/aGKir7Zw= 11 | github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= 12 | github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5xuRL7NZgHameVYF6BurY= 13 | github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 h1:QSdcrd/UFJv6Bp/CfoVf2SrENpFn9P6Yh8yb+xNhYMM= 14 | github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= 15 | github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible h1:9gWa46nstkJ9miBReJcN8Gq34cBFbzSpQZVVT9N09TM= 16 | github.com/aws/aws-sdk-go-v2 v1.16.0 h1:cBAYjiiexRAg9v2z9vb6IdxAa7ef4KCtjW7w7e3GxGo= 17 | github.com/aws/aws-sdk-go-v2 v1.16.0/go.mod h1:lJYcuZZEHWNIb6ugJjbQY1fykdoobWbOS7kJYb4APoI= 18 | github.com/aws/aws-sdk-go-v2/config v1.15.1 h1:hTIZFepYESYyowQUBo47lu69WSxsYqGUILY9Nu8+7pY= 19 | github.com/aws/aws-sdk-go-v2/config v1.15.1/go.mod h1:MZHGbuW2WnqIOQQBKu2ZkhTjuutZSTnn56TDq4QyydE= 20 | github.com/aws/aws-sdk-go-v2/credentials v1.11.0 h1:gc4Uhs80s60nmLon5Z4JXWinX2BkAGT0YROoUT8h8U4= 21 | github.com/aws/aws-sdk-go-v2/credentials v1.11.0/go.mod h1:EdV1ZFgtZ4XM5RDHWcRWK8H+xW5duNVBqWj2oLu7tRo= 22 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.1 h1:F9Je1nq5YXfMOv6451NHvMf6U0iTWeMnsG0MMIQoUmk= 23 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.1/go.mod h1:Yph0XsTbQ5GGZ2+mO1a03P/SO9fdX3t1nejIp2tq79g= 24 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.7 h1:KUErSJgdqmqAPBWAp6Zx9CjL0YXfytXJeXcsWnuCM1c= 25 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.7/go.mod h1:oB9nZcxH1cGq7NPGurVJwxrO2vmJ9mmEBayCwcAlmT8= 26 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.1 h1:feVfa9eJonhJiss7g51ikjNB2DrUzbNZNvPL8pw/54k= 27 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.1/go.mod h1:K4vz7lRYCyLYpYAMCLObODahFgARdD3YVa0MvQte9Co= 28 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.8 h1:adr3PfiggFtqgFofAMUFCtdvwzpf3QxPES4ezK4M3iI= 29 | github.com/aws/aws-sdk-go-v2/internal/ini v1.3.8/go.mod h1:wLbQYt36AJqaRZUQiCNXzbtkNigyPfKHrotHuIDiCy8= 30 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.1 h1:B/SPX7J+Y0Yrcjv60Nhbh1gC2uBN47SfN8JYre6Mp4M= 31 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.1/go.mod h1:2Hhr9Eh1gJzDatwACX/ozAZ/ljq5vzvPRu5cdu25tzc= 32 | github.com/aws/aws-sdk-go-v2/service/sso v1.11.1 h1:DyHctRsJIAWIvom1Itb4T84D2jwpIu+KIi3d0SFaswg= 33 | github.com/aws/aws-sdk-go-v2/service/sso v1.11.1/go.mod h1:CvFTucADIx7U/M44vjLs/ZttpQHdpxwK+62+dUGhDeY= 34 | github.com/aws/aws-sdk-go-v2/service/sts v1.16.1 h1:xsOtPAvHqhvQvBza5ohaUcfq1LceH2lZKMUGZJKiZiM= 35 | github.com/aws/aws-sdk-go-v2/service/sts v1.16.1/go.mod h1:Aq2/Qggh2oemSfyHH+EO4UBbgWG6zFCXLHYI4ILTY7w= 36 | github.com/aws/smithy-go v1.11.1 h1:IQ+lPZVkSM3FRtyaDox41R8YS6iwPMYIreejOgPW49g= 37 | github.com/aws/smithy-go v1.11.1/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= 38 | github.com/baidubce/bce-sdk-go v0.9.111 h1:yGgtPpZYUZW4uoVorQ4xnuEgVeddACydlcJKW87MDV4= 39 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 40 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 41 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 42 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 43 | github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= 44 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 45 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 46 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 47 | github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= 48 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 49 | github.com/efficientgo/e2e v0.12.1 h1:ZYNTf09ptlba0I3ZStYaF7gCbevWdalriiX7usOSiFM= 50 | github.com/efficientgo/tools/core v0.0.0-20220225185207-fe763185946b h1:ZHiD4/yE4idlbqvAO6iYCOYRzOMRpxkW+FKasRA3tsQ= 51 | github.com/efficientgo/tools/core v0.0.0-20220225185207-fe763185946b/go.mod h1:OmVcnJopJL8d3X3sSXTiypGoUSgFq1aDGmlrdi9dn/M= 52 | github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= 53 | github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= 54 | github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= 55 | github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= 56 | github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= 57 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 58 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 59 | github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= 60 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= 61 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 62 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 63 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 64 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 65 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 66 | github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= 67 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 68 | github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= 69 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 70 | github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= 71 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 72 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 73 | github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= 74 | github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= 75 | github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= 76 | github.com/grafana/dskit v0.0.0-20230914143233-4b32fbf08128 h1:LO1AYS3ZR85Mb2o5VEsYgM+/AE34IH68Okv7jurWkn0= 77 | github.com/grafana/dskit v0.0.0-20230914143233-4b32fbf08128/go.mod h1:mZkAJTODiHxuvqvmwwFvp0ww9XA3mwlGA8Iv0SHP+5c= 78 | github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww= 79 | github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= 80 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 81 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 82 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 83 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 84 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 85 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 86 | github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= 87 | github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= 88 | github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 89 | github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= 90 | github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= 91 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 92 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 93 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 94 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 95 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 96 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 97 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 98 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 99 | github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= 100 | github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= 101 | github.com/minio/minio-go/v7 v7.0.23 h1:NleyGQvAn9VQMU+YHVrgV4CX+EPtxPt/78lHOOTncy4= 102 | github.com/minio/minio-go/v7 v7.0.23/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do= 103 | github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= 104 | github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= 105 | github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= 106 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 107 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= 108 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 109 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 110 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 111 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 112 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 113 | github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= 114 | github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= 115 | github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= 116 | github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= 117 | github.com/oracle/oci-go-sdk/v65 v65.13.0 h1:0+9ea5goYfhI3/MPfbIQU6yzHYWE6sCk6VuUepxk5Nk= 118 | github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4= 119 | github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 120 | github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= 121 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 122 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 123 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 124 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 125 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 126 | github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= 127 | github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= 128 | github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= 129 | github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= 130 | github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= 131 | github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= 132 | github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= 133 | github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= 134 | github.com/prometheus/prometheus v0.44.0 h1:sgn8Fdx+uE5tHQn0/622swlk2XnIj6udoZCnbVjHIgc= 135 | github.com/prometheus/prometheus v0.44.0/go.mod h1:aPsmIK3py5XammeTguyqTmuqzX/jeCdyOWWobLHNKQg= 136 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 137 | github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= 138 | github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= 139 | github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= 140 | github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= 141 | github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= 142 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 143 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 144 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 145 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 146 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 147 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 148 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 149 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 150 | github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= 151 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 152 | github.com/tencentyun/cos-go-sdk-v5 v0.7.34 h1:xm+Pg+6m486y4eugRI7/E4WasbVmpY1hp9QBSRErgp8= 153 | github.com/thanos-io/objstore v0.0.0-20220809103346-8ef1f215e2bf h1:onQsPyHlq2yIWU+Nfl6yStuqnZuVQQN8FZ8sBb2wqtw= 154 | github.com/thanos-io/objstore v0.0.0-20220809103346-8ef1f215e2bf/go.mod h1:v0NhuxxxUFUPatQcVNSCUkBEVezXzl7LSdaBOZygq98= 155 | github.com/twmb/franz-go v1.15.4 h1:qBCkHaiutetnrXjAUWA99D9FEcZVMt2AYwkH3vWEQTw= 156 | github.com/twmb/franz-go v1.15.4/go.mod h1:rC18hqNmfo8TMc1kz7CQmHL74PLNF8KVvhflxiiJZCU= 157 | github.com/twmb/franz-go/pkg/kadm v1.10.0 h1:3oYKNP+e3HGo4GYadrDeRxOaAIsOXmX6LBVMz9PxpCU= 158 | github.com/twmb/franz-go/pkg/kadm v1.10.0/go.mod h1:hUMoV4SRho+2ij/S9cL39JaLsr+XINjn0ZkCdBY2DXc= 159 | github.com/twmb/franz-go/pkg/kmsg v1.7.0 h1:a457IbvezYfA5UkiBvyV3zj0Is3y1i8EJgqjJYoij2E= 160 | github.com/twmb/franz-go/pkg/kmsg v1.7.0/go.mod h1:se9Mjdt0Nwzc9lnjJ0HyDtLyBnaBDAd7pCje47OhSyw= 161 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 162 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 163 | go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= 164 | go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= 165 | go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= 166 | go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= 167 | go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= 168 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 169 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 170 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 171 | golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= 172 | golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= 173 | golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= 174 | golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= 175 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 176 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 177 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 178 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 179 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 180 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 181 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 182 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 183 | golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= 184 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 185 | golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= 186 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 187 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 188 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 189 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 190 | golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= 191 | golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= 192 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 193 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 194 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 195 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 196 | golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= 197 | golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 198 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 199 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 200 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 201 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 202 | golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= 203 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 204 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 205 | golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 206 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 207 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 208 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 209 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 210 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 211 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 212 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 213 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= 214 | google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= 215 | google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= 216 | google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= 217 | google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= 218 | google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= 219 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= 220 | google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= 221 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 222 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 223 | google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= 224 | google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 225 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 226 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 227 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 228 | gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= 229 | gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 230 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 231 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 232 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 233 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 234 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 235 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 236 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 237 | -------------------------------------------------------------------------------- /images/images.go: -------------------------------------------------------------------------------- 1 | package images 2 | 3 | var ( 4 | // These are variables so that they can be modified. 5 | Memcached = "memcached:1.6.12" 6 | Redis = "redis:7.0.7" 7 | Minio = "minio/minio:RELEASE.2021-10-13T00-23-17Z" 8 | KES = "minio/kes:v0.17.1" 9 | Consul = "consul:1.8.15" 10 | ETCD = "gcr.io/etcd-development/etcd:v3.4.13" 11 | DynamoDB = "amazon/dynamodb-local:1.17.0" 12 | BigtableEmulator = "shopify/bigtable-emulator:0.1.0" 13 | Cassandra = "rinscy/cassandra:3.11.0" 14 | SwiftEmulator = "bouncestorage/swift-aio:55ba4331" 15 | Kafka = "apache/kafka:4.0.0" 16 | ) 17 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "strings" 8 | "time" 9 | 10 | "github.com/go-kit/log" 11 | ) 12 | 13 | // Global logger to use in integration tests. We use a global logger to simplify 14 | // writing integration tests and avoiding having to pass the logger instance 15 | // every time. 16 | var logger log.Logger 17 | 18 | func init() { 19 | logger = NewLogger(os.Stdout) 20 | } 21 | 22 | type Logger struct { 23 | w io.Writer 24 | } 25 | 26 | func NewLogger(w io.Writer) *Logger { 27 | return &Logger{ 28 | w: w, 29 | } 30 | } 31 | 32 | func (l *Logger) Log(keyvals ...interface{}) error { 33 | log := strings.Builder{} 34 | log.WriteString(time.Now().Format("15:04:05")) 35 | 36 | for _, v := range keyvals { 37 | log.WriteString(" " + fmt.Sprint(v)) 38 | } 39 | 40 | log.WriteString("\n") 41 | 42 | _, err := l.w.Write([]byte(log.String())) 43 | return err 44 | } 45 | -------------------------------------------------------------------------------- /metrics.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "math" 5 | 6 | io_prometheus_client "github.com/prometheus/client_model/go" 7 | ) 8 | 9 | func getMetricValue(m *io_prometheus_client.Metric) float64 { 10 | if m.GetGauge() != nil { 11 | return m.GetGauge().GetValue() 12 | } else if m.GetCounter() != nil { 13 | return m.GetCounter().GetValue() 14 | } else if m.GetHistogram() != nil { 15 | return m.GetHistogram().GetSampleSum() 16 | } else if m.GetSummary() != nil { 17 | return m.GetSummary().GetSampleSum() 18 | } else { 19 | return 0 20 | } 21 | } 22 | 23 | func getMetricCount(m *io_prometheus_client.Metric) float64 { 24 | if m.GetHistogram() != nil { 25 | return float64(m.GetHistogram().GetSampleCount()) 26 | } else if m.GetSummary() != nil { 27 | return float64(m.GetSummary().GetSampleCount()) 28 | } else { 29 | return 0 30 | } 31 | } 32 | 33 | func getValues(metrics []*io_prometheus_client.Metric, opts MetricsOptions) []float64 { 34 | values := make([]float64, 0, len(metrics)) 35 | for _, m := range metrics { 36 | values = append(values, opts.GetValue(m)) 37 | } 38 | return values 39 | } 40 | 41 | func filterMetrics(metrics []*io_prometheus_client.Metric, opts MetricsOptions) []*io_prometheus_client.Metric { 42 | // If no label matcher is configured, then no filtering should be done. 43 | if len(opts.LabelMatchers) == 0 { 44 | return metrics 45 | } 46 | if len(metrics) == 0 { 47 | return metrics 48 | } 49 | 50 | filtered := make([]*io_prometheus_client.Metric, 0, len(metrics)) 51 | 52 | for _, m := range metrics { 53 | metricLabels := map[string]string{} 54 | for _, lp := range m.GetLabel() { 55 | metricLabels[lp.GetName()] = lp.GetValue() 56 | } 57 | 58 | matches := true 59 | for _, matcher := range opts.LabelMatchers { 60 | if !matcher.Matches(metricLabels[matcher.Name]) { 61 | matches = false 62 | break 63 | } 64 | } 65 | 66 | if !matches { 67 | continue 68 | } 69 | 70 | filtered = append(filtered, m) 71 | } 72 | 73 | return filtered 74 | } 75 | 76 | func SumValues(values []float64) float64 { 77 | sum := 0.0 78 | for _, v := range values { 79 | sum += v 80 | } 81 | return sum 82 | } 83 | 84 | func EqualsSingle(expected float64) func(float64) bool { 85 | return func(v float64) bool { 86 | return v == expected || (math.IsNaN(v) && math.IsNaN(expected)) 87 | } 88 | } 89 | 90 | // Equals is an isExpected function for WaitSumMetrics that returns true if given single sum is equals to given value. 91 | func Equals(value float64) func(sums ...float64) bool { 92 | return func(sums ...float64) bool { 93 | if len(sums) != 1 { 94 | panic("equals: expected one value") 95 | } 96 | return sums[0] == value || math.IsNaN(sums[0]) && math.IsNaN(value) 97 | } 98 | } 99 | 100 | // Greater is an isExpected function for WaitSumMetrics that returns true if given single sum is greater than given value. 101 | func Greater(value float64) func(sums ...float64) bool { 102 | return func(sums ...float64) bool { 103 | if len(sums) != 1 { 104 | panic("greater: expected one value") 105 | } 106 | return sums[0] > value 107 | } 108 | } 109 | 110 | // GreaterOrEqual is an isExpected function for WaitSumMetrics that returns true if given single sum is greater or equal than given value. 111 | func GreaterOrEqual(value float64) func(sums ...float64) bool { 112 | return func(sums ...float64) bool { 113 | if len(sums) != 1 { 114 | panic("greater: expected one value") 115 | } 116 | return sums[0] >= value 117 | } 118 | } 119 | 120 | // Less is an isExpected function for WaitSumMetrics that returns true if given single sum is less than given value. 121 | func Less(value float64) func(sums ...float64) bool { 122 | return func(sums ...float64) bool { 123 | if len(sums) != 1 { 124 | panic("less: expected one value") 125 | } 126 | return sums[0] < value 127 | } 128 | } 129 | 130 | // Between is an isExpected function for WaitSumMetrics that returns true if given single sum is greater than or equal to lower 131 | // and less than or equal to upper. 132 | func Between(lower, upper float64) func(sums ...float64) bool { 133 | return func(sums ...float64) bool { 134 | if len(sums) != 1 { 135 | panic("equals: expected one value") 136 | } 137 | return sums[0] >= lower && sums[0] <= upper 138 | } 139 | } 140 | 141 | // EqualsAmongTwo is an isExpected function for WaitSumMetrics that returns true if first sum is equal to the second. 142 | // NOTE: Be careful on scrapes in between of process that changes two metrics. Those are 143 | // usually not atomic. 144 | func EqualsAmongTwo(sums ...float64) bool { 145 | if len(sums) != 2 { 146 | panic("equalsAmongTwo: expected two values") 147 | } 148 | return sums[0] == sums[1] 149 | } 150 | 151 | // GreaterAmongTwo is an isExpected function for WaitSumMetrics that returns true if first sum is greater than second. 152 | // NOTE: Be careful on scrapes in between of process that changes two metrics. Those are 153 | // usually not atomic. 154 | func GreaterAmongTwo(sums ...float64) bool { 155 | if len(sums) != 2 { 156 | panic("greaterAmongTwo: expected two values") 157 | } 158 | return sums[0] > sums[1] 159 | } 160 | 161 | // LessAmongTwo is an isExpected function for WaitSumMetrics that returns true if first sum is smaller than second. 162 | // NOTE: Be careful on scrapes in between of process that changes two metrics. Those are 163 | // usually not atomic. 164 | func LessAmongTwo(sums ...float64) bool { 165 | if len(sums) != 2 { 166 | panic("lessAmongTwo: expected two values") 167 | } 168 | return sums[0] < sums[1] 169 | } 170 | -------------------------------------------------------------------------------- /metrics_options.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | io_prometheus_client "github.com/prometheus/client_model/go" 5 | "github.com/prometheus/prometheus/model/labels" 6 | ) 7 | 8 | var ( 9 | DefaultMetricsOptions = MetricsOptions{ 10 | GetValue: getMetricValue, 11 | WaitMissingMetrics: false, 12 | } 13 | ) 14 | 15 | // GetMetricValueFunc defined the signature of a function used to get the metric value. 16 | type GetMetricValueFunc func(m *io_prometheus_client.Metric) float64 17 | 18 | // MetricsOption defined the signature of a function used to manipulate options. 19 | type MetricsOption func(*MetricsOptions) 20 | 21 | // MetricsOptions is the structure holding all options. 22 | type MetricsOptions struct { 23 | GetValue GetMetricValueFunc 24 | LabelMatchers []*labels.Matcher 25 | WaitMissingMetrics bool 26 | SkipMissingMetrics bool 27 | } 28 | 29 | // WithMetricCount is an option to get the histogram/summary count as metric value. 30 | func WithMetricCount(opts *MetricsOptions) { 31 | opts.GetValue = getMetricCount 32 | } 33 | 34 | // WithLabelMatchers is an option to filter only matching series. 35 | func WithLabelMatchers(matchers ...*labels.Matcher) MetricsOption { 36 | return func(opts *MetricsOptions) { 37 | opts.LabelMatchers = matchers 38 | } 39 | } 40 | 41 | // WithWaitMissingMetrics is an option to wait whenever an expected metric is missing. If this 42 | // option is not enabled, will return error on missing metrics. 43 | func WaitMissingMetrics(opts *MetricsOptions) { 44 | opts.WaitMissingMetrics = true 45 | } 46 | 47 | // SkipWaitMissingMetrics is an option to skip/ignore whenever an expected metric is missing. 48 | func SkipMissingMetrics(opts *MetricsOptions) { 49 | opts.SkipMissingMetrics = true 50 | } 51 | 52 | func buildMetricsOptions(opts []MetricsOption) MetricsOptions { 53 | result := DefaultMetricsOptions 54 | for _, opt := range opts { 55 | opt(&result) 56 | } 57 | return result 58 | } 59 | -------------------------------------------------------------------------------- /scenario.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "sync" 8 | 9 | "github.com/pkg/errors" 10 | tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" 11 | ) 12 | 13 | const ( 14 | ContainerSharedDir = "/shared" 15 | ) 16 | 17 | type Service interface { 18 | Name() string 19 | Start(networkName, dir string) error 20 | WaitReady() error 21 | 22 | // It should be ok to Stop and Kill more than once, with next invokes being noop. 23 | Kill() error 24 | Stop() error 25 | } 26 | 27 | type Scenario struct { 28 | services []Service 29 | 30 | networkName string 31 | sharedDir string 32 | } 33 | 34 | func NewScenario(networkName string) (*Scenario, error) { 35 | s := &Scenario{networkName: networkName} 36 | 37 | var err error 38 | s.sharedDir, err = GetTempDirectory() 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | // Force a shutdown in order to cleanup from a spurious situation in case 44 | // the previous tests run didn't cleanup correctly. 45 | s.shutdown() 46 | 47 | args := []string{ 48 | "network", "create", networkName, 49 | } 50 | 51 | if extraArgs := os.Getenv("DOCKER_NETWORK_CREATE_EXTRA_ARGS"); extraArgs != "" { 52 | args = append( 53 | args, 54 | strings.Split(extraArgs, ",")..., 55 | ) 56 | } 57 | 58 | // Setup the docker network. 59 | if out, err := RunCommandAndGetOutput("docker", args...); err != nil { 60 | logger.Log(string(out)) 61 | s.clean() 62 | return nil, errors.Wrapf(err, "create docker network '%s'", networkName) 63 | } 64 | 65 | return s, nil 66 | } 67 | 68 | // SharedDir returns the absolute path of the directory on the host that is shared with all services in docker. 69 | func (s *Scenario) SharedDir() string { 70 | return s.sharedDir 71 | } 72 | 73 | // NetworkName returns the network name that scenario is responsible for. 74 | func (s *Scenario) NetworkName() string { 75 | return s.networkName 76 | } 77 | 78 | func (s *Scenario) isRegistered(name string) bool { 79 | for _, service := range s.services { 80 | if service.Name() == name { 81 | return true 82 | } 83 | } 84 | return false 85 | } 86 | 87 | func (s *Scenario) StartAndWaitReady(services ...Service) error { 88 | if err := s.Start(services...); err != nil { 89 | return err 90 | } 91 | return s.WaitReady(services...) 92 | } 93 | 94 | func (s *Scenario) Start(services ...Service) error { 95 | var ( 96 | wg = sync.WaitGroup{} 97 | startedMx = sync.Mutex{} 98 | started = make([]Service, 0, len(services)) 99 | errsMx = sync.Mutex{} 100 | errs = tsdb_errors.NewMulti() 101 | ) 102 | 103 | // Ensure provided services don't conflict with existing ones. 104 | if err := s.assertNoConflicts(services...); err != nil { 105 | return err 106 | } 107 | 108 | // Start the services concurrently. 109 | wg.Add(len(services)) 110 | 111 | for _, service := range services { 112 | go func(service Service) { 113 | defer wg.Done() 114 | 115 | logger.Log("Starting", service.Name()) 116 | 117 | // Start the service. 118 | if err := service.Start(s.networkName, s.SharedDir()); err != nil { 119 | errsMx.Lock() 120 | errs.Add(err) 121 | errsMx.Unlock() 122 | return 123 | } 124 | 125 | logger.Log("Started", service.Name()) 126 | 127 | startedMx.Lock() 128 | started = append(started, service) 129 | startedMx.Unlock() 130 | }(service) 131 | } 132 | 133 | // Wait until all services have been started. 134 | wg.Wait() 135 | 136 | // Add the successfully started services to the scenario. 137 | s.services = append(s.services, started...) 138 | 139 | return errs.Err() 140 | } 141 | 142 | func (s *Scenario) Stop(services ...Service) error { 143 | for _, service := range services { 144 | if !s.isRegistered(service.Name()) { 145 | return fmt.Errorf("unable to stop service %s because it does not exist", service.Name()) 146 | } 147 | if err := service.Stop(); err != nil { 148 | return err 149 | } 150 | 151 | // Remove the service from the list of services. 152 | for i, entry := range s.services { 153 | if entry.Name() == service.Name() { 154 | s.services = append(s.services[:i], s.services[i+1:]...) 155 | break 156 | } 157 | } 158 | } 159 | return nil 160 | } 161 | 162 | func (s *Scenario) WaitReady(services ...Service) error { 163 | for _, service := range services { 164 | if !s.isRegistered(service.Name()) { 165 | return fmt.Errorf("unable to wait for service %s because it does not exist", service.Name()) 166 | } 167 | if err := service.WaitReady(); err != nil { 168 | return err 169 | } 170 | } 171 | return nil 172 | } 173 | 174 | func (s *Scenario) Close() { 175 | if s == nil { 176 | return 177 | } 178 | s.shutdown() 179 | s.clean() 180 | } 181 | 182 | func (s *Scenario) assertNoConflicts(services ...Service) error { 183 | // Build a map of services already registered. 184 | names := map[string]struct{}{} 185 | for _, service := range s.services { 186 | names[service.Name()] = struct{}{} 187 | } 188 | 189 | // Check if input services conflict with already existing ones or between them. 190 | for _, service := range services { 191 | if _, exists := names[service.Name()]; exists { 192 | return fmt.Errorf("another service with the same name '%s' exists", service.Name()) 193 | } 194 | 195 | names[service.Name()] = struct{}{} 196 | } 197 | 198 | return nil 199 | } 200 | 201 | // TODO(bwplotka): Add comments. 202 | func (s *Scenario) clean() { 203 | if err := os.RemoveAll(s.sharedDir); err != nil { 204 | logger.Log("error while removing sharedDir", s.sharedDir, "err:", err) 205 | } 206 | } 207 | 208 | func (s *Scenario) shutdown() { 209 | // Kill the services concurrently. 210 | wg := sync.WaitGroup{} 211 | wg.Add(len(s.services)) 212 | 213 | for _, srv := range s.services { 214 | go func(service Service) { 215 | defer wg.Done() 216 | 217 | if err := service.Kill(); err != nil { 218 | logger.Log("Unable to kill service", service.Name(), ":", err.Error()) 219 | } 220 | }(srv) 221 | } 222 | 223 | // Wait until all services have been killed. 224 | wg.Wait() 225 | 226 | // Ensure there are no leftover containers. 227 | if out, err := RunCommandAndGetOutput( 228 | "docker", 229 | "ps", 230 | "-a", 231 | "--quiet", 232 | "--filter", 233 | fmt.Sprintf("network=%s", s.networkName), 234 | ); err == nil { 235 | for _, containerID := range strings.Split(string(out), "\n") { 236 | containerID = strings.TrimSpace(containerID) 237 | if containerID == "" { 238 | continue 239 | } 240 | 241 | if out, err = RunCommandAndGetOutput("docker", "rm", "--force", containerID); err != nil { 242 | logger.Log(string(out)) 243 | logger.Log("Unable to cleanup leftover container", containerID, ":", err.Error()) 244 | } 245 | } 246 | } else { 247 | logger.Log(string(out)) 248 | logger.Log("Unable to cleanup leftover containers:", err.Error()) 249 | } 250 | 251 | // Teardown the docker network. In case the network does not exists (ie. this function 252 | // is called during the setup of the scenario) we skip the removal in order to not log 253 | // an error which may be misleading. 254 | if ok, err := existDockerNetwork(s.networkName); ok || err != nil { 255 | if out, err := RunCommandAndGetOutput("docker", "network", "rm", s.networkName); err != nil { 256 | logger.Log(string(out)) 257 | logger.Log("Unable to remove docker network", s.networkName, ":", err.Error()) 258 | } 259 | } 260 | } 261 | 262 | func existDockerNetwork(networkName string) (bool, error) { 263 | out, err := RunCommandAndGetOutput("docker", "network", "ls", "--quiet", "--filter", fmt.Sprintf("name=%s", networkName)) 264 | if err != nil { 265 | logger.Log(string(out)) 266 | logger.Log("Unable to check if docker network", networkName, "exists:", err.Error()) 267 | } 268 | 269 | return strings.TrimSpace(string(out)) != "", nil 270 | } 271 | -------------------------------------------------------------------------------- /scenario_test.go: -------------------------------------------------------------------------------- 1 | //go:build requires_docker 2 | // +build requires_docker 3 | 4 | package e2e_test 5 | 6 | import ( 7 | "bytes" 8 | "context" 9 | "io/ioutil" 10 | "testing" 11 | "time" 12 | 13 | "github.com/go-kit/log" 14 | "github.com/stretchr/testify/assert" 15 | "github.com/stretchr/testify/require" 16 | "github.com/thanos-io/objstore/providers/s3" 17 | 18 | // nolint: typecheck 19 | "gopkg.in/yaml.v3" 20 | 21 | "github.com/grafana/e2e" 22 | e2edb "github.com/grafana/e2e/db" 23 | ) 24 | 25 | const bktName = "cheesecake" 26 | 27 | func spinup(t *testing.T, networkName string) (*e2e.Scenario, *e2e.HTTPService, *e2e.HTTPService) { 28 | s, err := e2e.NewScenario(networkName) 29 | assert.NoError(t, err) 30 | 31 | m1 := e2edb.NewMinio(9000, bktName) 32 | m2 := e2edb.NewMinio(9001, bktName) 33 | 34 | closePlease := true 35 | defer func() { 36 | if closePlease { 37 | // You're welcome. 38 | s.Close() 39 | } 40 | }() 41 | require.NoError(t, s.StartAndWaitReady(m1, m2)) 42 | require.Error(t, s.Start(m1)) 43 | require.Error(t, s.Start(e2edb.NewMinio(9000, "cheescake"))) 44 | 45 | closePlease = false 46 | return s, m1, m2 47 | } 48 | 49 | func testMinioWorking(t *testing.T, m *e2e.HTTPService) { 50 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) 51 | defer cancel() 52 | 53 | // nolint: typecheck 54 | b, err := yaml.Marshal(s3.Config{ 55 | Endpoint: m.HTTPEndpoint(), 56 | Bucket: bktName, 57 | AccessKey: e2edb.MinioAccessKey, 58 | SecretKey: e2edb.MinioSecretKey, 59 | Insecure: true, // WARNING: Our secret cheesecake recipes might leak. 60 | }) 61 | require.NoError(t, err) 62 | 63 | bkt, err := s3.NewBucket(log.NewNopLogger(), b, "test") 64 | require.NoError(t, err) 65 | 66 | require.NoError(t, bkt.Upload(ctx, "recipe", bytes.NewReader([]byte("Just go to Pastry Shop and buy.")))) 67 | require.NoError(t, bkt.Upload(ctx, "mom/recipe", bytes.NewReader([]byte("https://www.bbcgoodfood.com/recipes/strawberry-cheesecake-4-easy-steps")))) 68 | 69 | r, err := bkt.Get(ctx, "recipe") 70 | require.NoError(t, err) 71 | b, err = ioutil.ReadAll(r) 72 | require.NoError(t, err) 73 | require.Equal(t, "Just go to Pastry Shop and buy.", string(b)) 74 | 75 | r, err = bkt.Get(ctx, "mom/recipe") 76 | require.NoError(t, err) 77 | b, err = ioutil.ReadAll(r) 78 | require.NoError(t, err) 79 | require.Equal(t, "https://www.bbcgoodfood.com/recipes/strawberry-cheesecake-4-easy-steps", string(b)) 80 | } 81 | 82 | func TestScenario(t *testing.T) { 83 | t.Parallel() 84 | 85 | s, m1, m2 := spinup(t, "e2e-scenario-test") 86 | defer s.Close() 87 | 88 | t.Run("minio is working", func(t *testing.T) { 89 | testMinioWorking(t, m1) 90 | testMinioWorking(t, m2) 91 | }) 92 | 93 | t.Run("concurrent nested scenario 1 is working just fine as well", func(t *testing.T) { 94 | t.Parallel() 95 | 96 | s, m1, m2 := spinup(t, "e2e-scenario-test1") 97 | defer s.Close() 98 | 99 | testMinioWorking(t, m1) 100 | testMinioWorking(t, m2) 101 | }) 102 | t.Run("concurrent nested scenario 2 is working just fine as well", func(t *testing.T) { 103 | t.Parallel() 104 | 105 | s, m1, m2 := spinup(t, "e2e-scenario-test2") 106 | defer s.Close() 107 | 108 | testMinioWorking(t, m1) 109 | testMinioWorking(t, m2) 110 | }) 111 | 112 | require.NoError(t, s.Stop(m1)) 113 | 114 | // Expect m1 not working. 115 | // nolint: typecheck 116 | b, err := yaml.Marshal(s3.Config{ 117 | Endpoint: m1.Name(), 118 | Bucket: "cheescake", 119 | AccessKey: e2edb.MinioAccessKey, 120 | SecretKey: e2edb.MinioSecretKey, 121 | }) 122 | require.NoError(t, err) 123 | bkt, err := s3.NewBucket(log.NewNopLogger(), b, "test") 124 | require.NoError(t, err) 125 | 126 | _, err = bkt.Get(context.Background(), "recipe") 127 | require.Error(t, err) 128 | 129 | testMinioWorking(t, m2) 130 | 131 | require.Error(t, s.Stop(m1)) 132 | // Should be noop. 133 | require.NoError(t, m1.Stop()) 134 | // I can run closes as many times I want. 135 | s.Close() 136 | s.Close() 137 | s.Close() 138 | 139 | // Expect m2 not working. 140 | // nolint: typecheck 141 | b, err = yaml.Marshal(s3.Config{ 142 | Endpoint: m2.Name(), 143 | Bucket: "cheescake", 144 | AccessKey: e2edb.MinioAccessKey, 145 | SecretKey: e2edb.MinioSecretKey, 146 | }) 147 | require.NoError(t, err) 148 | bkt, err = s3.NewBucket(log.NewNopLogger(), b, "test") 149 | require.NoError(t, err) 150 | 151 | _, err = bkt.Get(context.Background(), "recipe") 152 | require.Error(t, err) 153 | } 154 | -------------------------------------------------------------------------------- /service.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "crypto/tls" 7 | "crypto/x509" 8 | "fmt" 9 | "io/ioutil" 10 | "net" 11 | "os/exec" 12 | "regexp" 13 | "strconv" 14 | "strings" 15 | "time" 16 | 17 | "github.com/go-kit/log" 18 | "github.com/grafana/dskit/backoff" 19 | "github.com/grafana/dskit/runutil" 20 | "github.com/pkg/errors" 21 | "github.com/prometheus/common/expfmt" 22 | ) 23 | 24 | var ( 25 | dockerIPv4PortPattern = regexp.MustCompile(`^\d+\.\d+\.\d+\.\d+:(\d+)$`) 26 | errMissingMetric = errors.New("metric not found") 27 | ) 28 | 29 | // ConcreteService represents microservice with optional ports which will be discoverable from docker 30 | // with :. For connecting from test, use `Endpoint` method. 31 | // 32 | // ConcreteService can be reused (started and stopped many time), but it can represent only one running container 33 | // at the time. 34 | type ConcreteService struct { 35 | name string 36 | image string 37 | networkPorts []int 38 | env map[string]string 39 | user string 40 | command *Command 41 | cmd *exec.Cmd 42 | readiness ReadinessProbe 43 | privileged bool 44 | 45 | // Maps container ports to dynamically binded local ports. 46 | networkPortsContainerToLocal map[int]int 47 | 48 | // Generic retry backoff. 49 | retryBackoff *backoff.Backoff 50 | 51 | // docker NetworkName used to start this container. 52 | // If empty it means service is stopped. 53 | usedNetworkName string 54 | } 55 | 56 | func NewConcreteService( 57 | name string, 58 | image string, 59 | command *Command, 60 | readiness ReadinessProbe, 61 | networkPorts ...int, 62 | ) *ConcreteService { 63 | return &ConcreteService{ 64 | name: name, 65 | image: image, 66 | networkPorts: networkPorts, 67 | command: command, 68 | networkPortsContainerToLocal: map[int]int{}, 69 | readiness: readiness, 70 | retryBackoff: backoff.New(context.Background(), backoff.Config{ 71 | MinBackoff: 300 * time.Millisecond, 72 | MaxBackoff: 600 * time.Millisecond, 73 | MaxRetries: 100, // Sometimes the CI is slow ¯\_(ツ)_/¯ 74 | }), 75 | } 76 | } 77 | 78 | func (s *ConcreteService) isExpectedRunning() bool { 79 | return s.usedNetworkName != "" 80 | } 81 | 82 | func (s *ConcreteService) Name() string { return s.name } 83 | 84 | // Less often used options. 85 | 86 | func (s *ConcreteService) SetBackoff(cfg backoff.Config) { 87 | s.retryBackoff = backoff.New(context.Background(), cfg) 88 | } 89 | 90 | func (s *ConcreteService) SetEnvVars(env map[string]string) { 91 | s.env = env 92 | } 93 | 94 | func (s *ConcreteService) SetUser(user string) { 95 | s.user = user 96 | } 97 | 98 | func (s *ConcreteService) SetPrivileged(privileged bool) { 99 | s.privileged = privileged 100 | } 101 | 102 | func (s *ConcreteService) Start(networkName, sharedDir string) (err error) { 103 | // In case of any error, if the container was already created, we 104 | // have to cleanup removing it. We ignore the error of the "docker rm" 105 | // because we don't know if the container was created or not. 106 | defer func() { 107 | if err != nil { 108 | _, _ = RunCommandAndGetOutput("docker", "rm", "--force", s.name) 109 | } 110 | }() 111 | 112 | s.cmd = exec.Command("docker", s.buildDockerRunArgs(networkName, sharedDir)...) 113 | s.cmd.Stdout = &LinePrefixLogger{prefix: s.name + ": ", logger: logger} 114 | s.cmd.Stderr = &LinePrefixLogger{prefix: s.name + ": ", logger: logger} 115 | if err = s.cmd.Start(); err != nil { 116 | return err 117 | } 118 | s.usedNetworkName = networkName 119 | 120 | // Wait until the container has been started. 121 | if err = s.WaitForRunning(); err != nil { 122 | return err 123 | } 124 | 125 | // Get the dynamic local ports mapped to the container. 126 | for _, containerPort := range s.networkPorts { 127 | var out []byte 128 | 129 | out, err = RunCommandAndGetOutput("docker", "port", s.containerName(), strconv.Itoa(containerPort)) 130 | if err != nil { 131 | // Catch init errors. 132 | if werr := s.WaitForRunning(); werr != nil { 133 | return errors.Wrapf(werr, "failed to get mapping for port as container %s exited: %v", s.containerName(), err) 134 | } 135 | return errors.Wrapf(err, "unable to get mapping for port %d; service: %s; output: %q", containerPort, s.name, out) 136 | } 137 | 138 | localPort, err := parseDockerIPv4Port(string(out)) 139 | if err != nil { 140 | return errors.Wrapf(err, "unable to get mapping for port %d (output: %s); service: %s", containerPort, string(out), s.name) 141 | } 142 | 143 | s.networkPortsContainerToLocal[containerPort] = localPort 144 | } 145 | 146 | logger.Log("Ports for container:", s.containerName(), "Mapping:", s.networkPortsContainerToLocal) 147 | return nil 148 | } 149 | 150 | func (s *ConcreteService) Stop() error { 151 | if !s.isExpectedRunning() { 152 | return nil 153 | } 154 | 155 | logger.Log("Stopping", s.name) 156 | 157 | if out, err := RunCommandAndGetOutput("docker", "stop", "--time=30", s.containerName()); err != nil { 158 | logger.Log(string(out)) 159 | return err 160 | } 161 | s.usedNetworkName = "" 162 | 163 | return s.cmd.Wait() 164 | } 165 | 166 | func (s *ConcreteService) Kill() error { 167 | if !s.isExpectedRunning() { 168 | return nil 169 | } 170 | 171 | logger.Log("Killing", s.name) 172 | 173 | if out, err := RunCommandAndGetOutput("docker", "kill", s.containerName()); err != nil { 174 | logger.Log(string(out)) 175 | return err 176 | } 177 | 178 | // Wait until the container actually stopped. However, this could fail if 179 | // the container already exited, so we just ignore the error. 180 | _, _ = RunCommandAndGetOutput("docker", "wait", s.containerName()) 181 | 182 | s.usedNetworkName = "" 183 | 184 | logger.Log("Killed", s.name) 185 | return nil 186 | } 187 | 188 | // Endpoint returns external (from host perspective) service endpoint (host:port) for given internal port. 189 | // External means that it will be accessible only from host, but not from docker containers. 190 | // 191 | // If your service is not running, this method returns incorrect `stopped` endpoint. 192 | func (s *ConcreteService) Endpoint(port int) string { 193 | if !s.isExpectedRunning() { 194 | return "stopped" 195 | } 196 | 197 | // Map the container port to the local port. 198 | localPort, ok := s.networkPortsContainerToLocal[port] 199 | if !ok { 200 | return "" 201 | } 202 | 203 | // Use an IPv4 address instead of "localhost" hostname because our port mapping assumes IPv4 204 | // (a port published by a Docker container could be different between IPv4 and IPv6). 205 | return fmt.Sprintf("127.0.0.1:%d", localPort) 206 | } 207 | 208 | // NetworkEndpoint returns internal service endpoint (host:port) for given internal port. 209 | // Internal means that it will be accessible only from docker containers within the network that this 210 | // service is running in. If you configure your local resolver with docker DNS namespace you can access it from host 211 | // as well. Use `Endpoint` for host access. 212 | // 213 | // If your service is not running, use `NetworkEndpointFor` instead. 214 | func (s *ConcreteService) NetworkEndpoint(port int) string { 215 | if s.usedNetworkName == "" { 216 | return "stopped" 217 | } 218 | return s.NetworkEndpointFor(s.usedNetworkName, port) 219 | } 220 | 221 | // NetworkEndpointFor returns internal service endpoint (host:port) for given internal port and network. 222 | // Internal means that it will be accessible only from docker containers within the given network. If you configure 223 | // your local resolver with docker DNS namespace you can access it from host as well. 224 | // 225 | // This method return correct endpoint for the service in any state. 226 | func (s *ConcreteService) NetworkEndpointFor(networkName string, port int) string { 227 | return fmt.Sprintf("%s:%d", NetworkContainerHost(networkName, s.name), port) 228 | } 229 | 230 | func (s *ConcreteService) SetReadinessProbe(probe ReadinessProbe) { 231 | s.readiness = probe 232 | } 233 | 234 | func (s *ConcreteService) Ready() error { 235 | if !s.isExpectedRunning() { 236 | return fmt.Errorf("service %s is stopped", s.Name()) 237 | } 238 | 239 | // Ensure the service has a readiness probe configure. 240 | if s.readiness == nil { 241 | return nil 242 | } 243 | 244 | return s.readiness.Ready(s) 245 | } 246 | 247 | func (s *ConcreteService) containerName() string { 248 | return NetworkContainerHost(s.usedNetworkName, s.name) 249 | } 250 | 251 | func (s *ConcreteService) WaitForRunning() (err error) { 252 | if !s.isExpectedRunning() { 253 | return fmt.Errorf("service %s is stopped", s.Name()) 254 | } 255 | 256 | for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); { 257 | // Enforce a timeout on the command execution because we've seen some flaky tests 258 | // stuck here. 259 | 260 | var out []byte 261 | out, err = RunCommandWithTimeoutAndGetOutput(5*time.Second, "docker", "inspect", "--format={{json .State.Running}}", s.containerName()) 262 | if err != nil { 263 | s.retryBackoff.Wait() 264 | continue 265 | } 266 | 267 | if out == nil { 268 | err = fmt.Errorf("nil output") 269 | s.retryBackoff.Wait() 270 | continue 271 | } 272 | 273 | str := strings.TrimSpace(string(out)) 274 | if str != "true" { 275 | err = fmt.Errorf("unexpected output: %q", str) 276 | s.retryBackoff.Wait() 277 | continue 278 | } 279 | 280 | return nil 281 | } 282 | 283 | return fmt.Errorf("docker container %s failed to start: %v", s.name, err) 284 | } 285 | 286 | func (s *ConcreteService) WaitReady() (err error) { 287 | if !s.isExpectedRunning() { 288 | return fmt.Errorf("service %s is stopped", s.Name()) 289 | } 290 | 291 | for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); { 292 | err = s.Ready() 293 | if err == nil { 294 | return nil 295 | } 296 | 297 | s.retryBackoff.Wait() 298 | } 299 | 300 | return fmt.Errorf("the service %s is not ready; err: %v", s.name, err) 301 | } 302 | 303 | func (s *ConcreteService) buildDockerRunArgs(networkName, sharedDir string) []string { 304 | args := []string{"run", "--rm", "--net=" + networkName, "--name=" + networkName + "-" + s.name, "--hostname=" + s.name} 305 | 306 | // If running a dind container, this needs to be privileged. 307 | if s.privileged { 308 | args = append(args, "--privileged") 309 | } 310 | 311 | // For Drone CI users, expire the container after 6 hours using drone-gc 312 | args = append(args, "--label", fmt.Sprintf("io.drone.expires=%d", time.Now().Add(6*time.Hour).Unix())) 313 | 314 | // Mount the shared/ directory into the container 315 | args = append(args, "-v", fmt.Sprintf("%s:%s:z", sharedDir, ContainerSharedDir)) 316 | 317 | // Environment variables 318 | for name, value := range s.env { 319 | args = append(args, "-e", name+"="+value) 320 | } 321 | 322 | if s.user != "" { 323 | args = append(args, "--user", s.user) 324 | } 325 | 326 | // Published ports 327 | for _, port := range s.networkPorts { 328 | args = append(args, "-p", strconv.Itoa(port)) 329 | } 330 | 331 | // Disable entrypoint if required 332 | if s.command != nil && s.command.entrypointDisabled { 333 | args = append(args, "--entrypoint", "") 334 | } 335 | 336 | args = append(args, s.image) 337 | 338 | if s.command != nil { 339 | args = append(args, s.command.cmd) 340 | args = append(args, s.command.args...) 341 | } 342 | 343 | return args 344 | } 345 | 346 | // Exec runs the provided against a the docker container specified by this 347 | // service. It returns the stdout, stderr, and error response from attempting 348 | // to run the command. 349 | func (s *ConcreteService) Exec(command *Command) (string, string, error) { 350 | args := []string{"exec", s.containerName()} 351 | args = append(args, command.cmd) 352 | args = append(args, command.args...) 353 | 354 | cmd := exec.Command("docker", args...) 355 | var stdout bytes.Buffer 356 | cmd.Stdout = &stdout 357 | 358 | var stderr bytes.Buffer 359 | cmd.Stderr = &stderr 360 | 361 | err := cmd.Run() 362 | 363 | return stdout.String(), stderr.String(), err 364 | } 365 | 366 | // NetworkContainerHost return the hostname of the container within the network. This is 367 | // the address a container should use to connect to other containers. 368 | func NetworkContainerHost(networkName, containerName string) string { 369 | return fmt.Sprintf("%s-%s", networkName, containerName) 370 | } 371 | 372 | // NetworkContainerHostPort return the host:port address of a container within the network. 373 | func NetworkContainerHostPort(networkName, containerName string, port int) string { 374 | return fmt.Sprintf("%s-%s:%d", networkName, containerName, port) 375 | } 376 | 377 | type Command struct { 378 | cmd string 379 | args []string 380 | entrypointDisabled bool 381 | } 382 | 383 | func NewCommand(cmd string, args ...string) *Command { 384 | return &Command{ 385 | cmd: cmd, 386 | args: args, 387 | } 388 | } 389 | 390 | func NewCommandWithoutEntrypoint(cmd string, args ...string) *Command { 391 | return &Command{ 392 | cmd: cmd, 393 | args: args, 394 | entrypointDisabled: true, 395 | } 396 | } 397 | 398 | type ReadinessProbe interface { 399 | Ready(service *ConcreteService) (err error) 400 | } 401 | 402 | // HTTPReadinessProbe checks readiness by making HTTP(S) call and checking for expected response status code. 403 | type HTTPReadinessProbe struct { 404 | schema string 405 | port int 406 | path string 407 | expectedStatusRangeStart int 408 | expectedStatusRangeEnd int 409 | expectedContent []string 410 | 411 | // The TLS config to use when issuing the HTTPS request. 412 | clientTLSConfig *tls.Config 413 | } 414 | 415 | func NewHTTPReadinessProbe(port int, path string, expectedStatusRangeStart, expectedStatusRangeEnd int, expectedContent ...string) *HTTPReadinessProbe { 416 | return &HTTPReadinessProbe{ 417 | schema: "http", 418 | port: port, 419 | path: path, 420 | expectedStatusRangeStart: expectedStatusRangeStart, 421 | expectedStatusRangeEnd: expectedStatusRangeEnd, 422 | expectedContent: expectedContent, 423 | } 424 | } 425 | 426 | func NewHTTPSReadinessProbe(port int, path, serverName, clientKeyFile, clientCertFile, rootCertFile string, expectedStatusRangeStart, expectedStatusRangeEnd int, expectedContent ...string) (*HTTPReadinessProbe, error) { 427 | // Load client certificate and private key. 428 | cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile) 429 | if err != nil { 430 | return nil, errors.Wrapf(err, "error creating x509 keypair from client cert file %s and client key file %s", clientCertFile, clientKeyFile) 431 | } 432 | 433 | caCert, err := ioutil.ReadFile(rootCertFile) 434 | if err != nil { 435 | return nil, errors.Wrapf(err, "error opening root CA cert file %s", rootCertFile) 436 | } 437 | 438 | caCertPool := x509.NewCertPool() 439 | caCertPool.AppendCertsFromPEM(caCert) 440 | 441 | return &HTTPReadinessProbe{ 442 | schema: "https", 443 | port: port, 444 | path: path, 445 | expectedStatusRangeStart: expectedStatusRangeStart, 446 | expectedStatusRangeEnd: expectedStatusRangeEnd, 447 | expectedContent: expectedContent, 448 | clientTLSConfig: &tls.Config{ 449 | Certificates: []tls.Certificate{cert}, 450 | RootCAs: caCertPool, 451 | ServerName: serverName, 452 | }, 453 | }, nil 454 | } 455 | 456 | func (p *HTTPReadinessProbe) Ready(service *ConcreteService) (err error) { 457 | endpoint := service.Endpoint(p.port) 458 | if endpoint == "" { 459 | return fmt.Errorf("cannot get service endpoint for port %d", p.port) 460 | } else if endpoint == "stopped" { 461 | return errors.New("service has stopped") 462 | } 463 | 464 | res, err := DoGetTLS(p.schema+"://"+endpoint+p.path, p.clientTLSConfig) 465 | if err != nil { 466 | return err 467 | } 468 | 469 | defer runutil.ExhaustCloseWithErrCapture(&err, res.Body, "response readiness") 470 | body, _ := ioutil.ReadAll(res.Body) 471 | 472 | if res.StatusCode < p.expectedStatusRangeStart || res.StatusCode > p.expectedStatusRangeEnd { 473 | return fmt.Errorf("expected code in range: [%v, %v], got status code: %v and body: %v", p.expectedStatusRangeStart, p.expectedStatusRangeEnd, res.StatusCode, string(body)) 474 | } 475 | 476 | for _, expected := range p.expectedContent { 477 | if !strings.Contains(string(body), expected) { 478 | return fmt.Errorf("expected body containing %s, got: %v", expected, string(body)) 479 | } 480 | } 481 | 482 | return nil 483 | } 484 | 485 | // TCPReadinessProbe checks readiness by ensure a TCP connection can be established. 486 | type TCPReadinessProbe struct { 487 | port int 488 | } 489 | 490 | func NewTCPReadinessProbe(port int) *TCPReadinessProbe { 491 | return &TCPReadinessProbe{ 492 | port: port, 493 | } 494 | } 495 | 496 | func (p *TCPReadinessProbe) Ready(service *ConcreteService) (err error) { 497 | endpoint := service.Endpoint(p.port) 498 | if endpoint == "" { 499 | return fmt.Errorf("cannot get service endpoint for port %d", p.port) 500 | } else if endpoint == "stopped" { 501 | return errors.New("service has stopped") 502 | } 503 | 504 | conn, err := net.DialTimeout("tcp", endpoint, time.Second) 505 | if err != nil { 506 | return err 507 | } 508 | 509 | return conn.Close() 510 | } 511 | 512 | // CmdReadinessProbe checks readiness by `Exec`ing a command (within container) which returns 0 to consider status being ready 513 | type CmdReadinessProbe struct { 514 | cmd *Command 515 | } 516 | 517 | func NewCmdReadinessProbe(cmd *Command) *CmdReadinessProbe { 518 | return &CmdReadinessProbe{cmd: cmd} 519 | } 520 | 521 | func (p *CmdReadinessProbe) Ready(service *ConcreteService) error { 522 | _, _, err := service.Exec(p.cmd) 523 | return err 524 | } 525 | 526 | type LinePrefixLogger struct { 527 | prefix string 528 | logger log.Logger 529 | } 530 | 531 | func (w *LinePrefixLogger) Write(p []byte) (n int, err error) { 532 | for _, line := range strings.Split(string(p), "\n") { 533 | // Skip empty lines 534 | line = strings.TrimSpace(line) 535 | if line == "" { 536 | continue 537 | } 538 | 539 | // Write the prefix + line to the wrapped writer 540 | if err := w.logger.Log(w.prefix + line); err != nil { 541 | return 0, err 542 | } 543 | } 544 | 545 | return len(p), nil 546 | } 547 | 548 | // HTTPService represents opinionated microservice with at least HTTP port that as mandatory requirement, 549 | // serves metrics. 550 | type HTTPService struct { 551 | *ConcreteService 552 | 553 | metricsTimeout time.Duration 554 | httpPort int 555 | } 556 | 557 | func NewHTTPService( 558 | name string, 559 | image string, 560 | command *Command, 561 | readiness ReadinessProbe, 562 | httpPort int, 563 | otherPorts ...int, 564 | ) *HTTPService { 565 | return &HTTPService{ 566 | ConcreteService: NewConcreteService(name, image, command, readiness, append(otherPorts, httpPort)...), 567 | metricsTimeout: time.Second, 568 | httpPort: httpPort, 569 | } 570 | } 571 | 572 | func (s *HTTPService) SetMetricsTimeout(timeout time.Duration) { 573 | s.metricsTimeout = timeout 574 | } 575 | 576 | func (s *HTTPService) Metrics() (_ string, err error) { 577 | // Map the container port to the local port 578 | localPort := s.networkPortsContainerToLocal[s.httpPort] 579 | 580 | // Fetch metrics. 581 | // Use an IPv4 address instead of "localhost" hostname because our port mapping assumes IPv4 582 | // (a port published by a Docker container could be different between IPv4 and IPv6). 583 | res, err := DoGetWithTimeout(fmt.Sprintf("http://127.0.0.1:%d/metrics", localPort), s.metricsTimeout) 584 | if err != nil { 585 | return "", err 586 | } 587 | 588 | // Check the status code. 589 | if res.StatusCode < 200 || res.StatusCode >= 300 { 590 | return "", fmt.Errorf("unexpected status code %d while fetching metrics", res.StatusCode) 591 | } 592 | 593 | defer runutil.ExhaustCloseWithErrCapture(&err, res.Body, "metrics response") 594 | body, err := ioutil.ReadAll(res.Body) 595 | 596 | return string(body), err 597 | } 598 | 599 | func (s *HTTPService) HTTPPort() int { 600 | return s.httpPort 601 | } 602 | 603 | func (s *HTTPService) HTTPEndpoint() string { 604 | return s.Endpoint(s.httpPort) 605 | } 606 | 607 | func (s *HTTPService) NetworkHTTPEndpoint() string { 608 | return s.NetworkEndpoint(s.httpPort) 609 | } 610 | 611 | func (s *HTTPService) NetworkHTTPEndpointFor(networkName string) string { 612 | return s.NetworkEndpointFor(networkName, s.httpPort) 613 | } 614 | 615 | // WaitSumMetrics waits for at least one instance of each given metric names to be present and their sums, returning true 616 | // when passed to given isExpected(...). 617 | func (s *HTTPService) WaitSumMetrics(isExpected func(sums ...float64) bool, metricNames ...string) error { 618 | return s.WaitSumMetricsWithOptions(isExpected, metricNames) 619 | } 620 | 621 | func (s *HTTPService) WaitSumMetricsWithOptions(isExpected func(sums ...float64) bool, metricNames []string, opts ...MetricsOption) error { 622 | var ( 623 | sums []float64 624 | err error 625 | options = buildMetricsOptions(opts) 626 | ) 627 | 628 | for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); { 629 | sums, err = s.SumMetrics(metricNames, opts...) 630 | if options.WaitMissingMetrics && errors.Is(err, errMissingMetric) { 631 | continue 632 | } 633 | if err != nil { 634 | return err 635 | } 636 | 637 | if isExpected(sums...) { 638 | return nil 639 | } 640 | 641 | s.retryBackoff.Wait() 642 | } 643 | 644 | return fmt.Errorf("unable to find metrics %s with expected values. Last error: %v. Last values: %v", metricNames, err, sums) 645 | } 646 | 647 | // SumMetrics returns the sum of the values of each given metric names. 648 | func (s *HTTPService) SumMetrics(metricNames []string, opts ...MetricsOption) ([]float64, error) { 649 | options := buildMetricsOptions(opts) 650 | sums := make([]float64, len(metricNames)) 651 | 652 | metrics, err := s.Metrics() 653 | if err != nil { 654 | return nil, err 655 | } 656 | 657 | var tp expfmt.TextParser 658 | families, err := tp.TextToMetricFamilies(strings.NewReader(metrics)) 659 | if err != nil { 660 | return nil, err 661 | } 662 | 663 | for i, m := range metricNames { 664 | sums[i] = 0.0 665 | 666 | // Get the metric family. 667 | mf, ok := families[m] 668 | if !ok { 669 | if options.SkipMissingMetrics { 670 | continue 671 | } 672 | 673 | return nil, errors.Wrapf(errMissingMetric, "metric=%s service=%s", m, s.name) 674 | } 675 | 676 | // Filter metrics. 677 | metrics := filterMetrics(mf.GetMetric(), options) 678 | if len(metrics) == 0 { 679 | if options.SkipMissingMetrics { 680 | continue 681 | } 682 | 683 | return nil, errors.Wrapf(errMissingMetric, "metric=%s service=%s", m, s.name) 684 | } 685 | 686 | sums[i] = SumValues(getValues(metrics, options)) 687 | } 688 | 689 | return sums, nil 690 | } 691 | 692 | // WaitRemovedMetric waits until a metric disappear from the list of metrics exported by the service. 693 | func (s *HTTPService) WaitRemovedMetric(metricName string, opts ...MetricsOption) error { 694 | options := buildMetricsOptions(opts) 695 | 696 | for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); { 697 | // Fetch metrics. 698 | metrics, err := s.Metrics() 699 | if err != nil { 700 | return err 701 | } 702 | 703 | // Parse metrics. 704 | var tp expfmt.TextParser 705 | families, err := tp.TextToMetricFamilies(strings.NewReader(metrics)) 706 | if err != nil { 707 | return err 708 | } 709 | 710 | // Get the metric family. 711 | mf, ok := families[metricName] 712 | if !ok { 713 | return nil 714 | } 715 | 716 | // Filter metrics. 717 | if len(filterMetrics(mf.GetMetric(), options)) == 0 { 718 | return nil 719 | } 720 | 721 | s.retryBackoff.Wait() 722 | } 723 | 724 | return fmt.Errorf("the metric %s is still exported by %s", metricName, s.name) 725 | } 726 | 727 | // parseDockerIPv4Port parses the input string which is expected to be the output of "docker port" 728 | // command and returns the first IPv4 port found. 729 | func parseDockerIPv4Port(out string) (int, error) { 730 | // The "docker port" output may be multiple lines if both IPv4 and IPv6 are supported, 731 | // so we need to parse each line. 732 | for _, line := range strings.Split(out, "\n") { 733 | matches := dockerIPv4PortPattern.FindStringSubmatch(strings.TrimSpace(line)) 734 | if len(matches) != 2 { 735 | continue 736 | } 737 | 738 | port, err := strconv.Atoi(matches[1]) 739 | if err != nil { 740 | continue 741 | } 742 | 743 | return port, nil 744 | } 745 | 746 | // We've not been able to parse the output format. 747 | return 0, errors.New("unknown output format") 748 | } 749 | -------------------------------------------------------------------------------- /service_test.go: -------------------------------------------------------------------------------- 1 | //go:build requires_docker 2 | // +build requires_docker 3 | 4 | package e2e 5 | 6 | import ( 7 | "math" 8 | "net" 9 | "net/http" 10 | "strconv" 11 | "testing" 12 | "time" 13 | 14 | "github.com/grafana/dskit/backoff" 15 | "github.com/stretchr/testify/assert" 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | func TestWaitSumMetric(t *testing.T) { 20 | // Listen on a random port before starting the HTTP server, to 21 | // make sure the port is already open when we'll call WaitSumMetric() 22 | // the first time (this avoid flaky tests). 23 | ln, err := net.Listen("tcp", "localhost:0") 24 | require.NoError(t, err) 25 | defer ln.Close() 26 | 27 | // Get the port. 28 | _, addrPort, err := net.SplitHostPort(ln.Addr().String()) 29 | require.NoError(t, err) 30 | 31 | port, err := strconv.Atoi(addrPort) 32 | require.NoError(t, err) 33 | 34 | // Start an HTTP server exposing the metrics. 35 | srv := &http.Server{ 36 | Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 37 | _, _ = w.Write([]byte(` 38 | # HELP metric_c cheescake 39 | # TYPE metric_c gauge 40 | metric_c 20 41 | # HELP metric_a cheescake 42 | # TYPE metric_a gauge 43 | metric_a 1 44 | metric_a{first="value1"} 10 45 | metric_a{first="value1", something="x"} 4 46 | metric_a{first="value1", something2="a"} 203 47 | metric_a{first="value2"} 2 48 | metric_a{second="value1"} 1 49 | # HELP metric_b cheescake 50 | # TYPE metric_b gauge 51 | metric_b 1000 52 | # HELP metric_b_counter cheescake 53 | # TYPE metric_b_counter counter 54 | metric_b_counter 1020 55 | # HELP metric_b_hist cheescake 56 | # TYPE metric_b_hist histogram 57 | metric_b_hist_count 5 58 | metric_b_hist_sum 124 59 | metric_b_hist_bucket{le="5.36870912e+08"} 1 60 | metric_b_hist_bucket{le="+Inf"} 5 61 | # HELP metric_b_summary cheescake 62 | # TYPE metric_b_summary summary 63 | metric_b_summary_sum 22 64 | metric_b_summary_count 1 65 | `)) 66 | }), 67 | } 68 | defer srv.Close() 69 | 70 | go func() { 71 | _ = srv.Serve(ln) 72 | }() 73 | 74 | s := &HTTPService{ 75 | httpPort: 0, 76 | ConcreteService: &ConcreteService{ 77 | networkPortsContainerToLocal: map[int]int{ 78 | 0: port, 79 | }, 80 | }, 81 | } 82 | 83 | s.SetBackoff(backoff.Config{ 84 | MinBackoff: 300 * time.Millisecond, 85 | MaxBackoff: 600 * time.Millisecond, 86 | MaxRetries: 50, 87 | }) 88 | require.NoError(t, s.WaitSumMetrics(Equals(221), "metric_a")) 89 | 90 | // No retry. 91 | s.SetBackoff(backoff.Config{ 92 | MinBackoff: 0, 93 | MaxBackoff: 0, 94 | MaxRetries: 1, 95 | }) 96 | require.Error(t, s.WaitSumMetrics(Equals(16), "metric_a")) 97 | 98 | require.NoError(t, s.WaitSumMetrics(Equals(1000), "metric_b")) 99 | require.NoError(t, s.WaitSumMetrics(Equals(1020), "metric_b_counter")) 100 | require.NoError(t, s.WaitSumMetrics(Equals(124), "metric_b_hist")) 101 | require.NoError(t, s.WaitSumMetrics(Equals(22), "metric_b_summary")) 102 | 103 | require.NoError(t, s.WaitSumMetrics(EqualsAmongTwo, "metric_a", "metric_a")) 104 | require.Error(t, s.WaitSumMetrics(EqualsAmongTwo, "metric_a", "metric_b")) 105 | 106 | require.NoError(t, s.WaitSumMetrics(GreaterAmongTwo, "metric_b", "metric_a")) 107 | require.Error(t, s.WaitSumMetrics(GreaterAmongTwo, "metric_a", "metric_b")) 108 | 109 | require.NoError(t, s.WaitSumMetrics(LessAmongTwo, "metric_a", "metric_b")) 110 | require.Error(t, s.WaitSumMetrics(LessAmongTwo, "metric_b", "metric_a")) 111 | 112 | require.Error(t, s.WaitSumMetrics(Equals(0), "non_existing_metric")) 113 | } 114 | 115 | func TestWaitSumMetric_Nan(t *testing.T) { 116 | // Listen on a random port before starting the HTTP server, to 117 | // make sure the port is already open when we'll call WaitSumMetric() 118 | // the first time (this avoid flaky tests). 119 | ln, err := net.Listen("tcp", "localhost:0") 120 | require.NoError(t, err) 121 | defer ln.Close() 122 | 123 | // Get the port. 124 | _, addrPort, err := net.SplitHostPort(ln.Addr().String()) 125 | require.NoError(t, err) 126 | 127 | port, err := strconv.Atoi(addrPort) 128 | require.NoError(t, err) 129 | 130 | // Start an HTTP server exposing the metrics. 131 | srv := &http.Server{ 132 | Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 133 | _, _ = w.Write([]byte(` 134 | # HELP metric_c cheescake 135 | # TYPE metric_c GAUGE 136 | metric_c 20 137 | # HELP metric_a cheescake 138 | # TYPE metric_a GAUGE 139 | metric_a 1 140 | metric_a{first="value1"} 10 141 | metric_a{first="value1", something="x"} 4 142 | metric_a{first="value1", something2="a"} 203 143 | metric_a{first="value1", something3="b"} Nan 144 | metric_a{first="value2"} 2 145 | metric_a{second="value1"} 1 146 | # HELP metric_b cheescake 147 | # TYPE metric_b GAUGE 148 | metric_b 1000 149 | `)) 150 | }), 151 | } 152 | defer srv.Close() 153 | 154 | go func() { 155 | _ = srv.Serve(ln) 156 | }() 157 | 158 | s := &HTTPService{ 159 | httpPort: 0, 160 | ConcreteService: &ConcreteService{ 161 | networkPortsContainerToLocal: map[int]int{ 162 | 0: port, 163 | }, 164 | }, 165 | } 166 | 167 | s.SetBackoff(backoff.Config{ 168 | MinBackoff: 300 * time.Millisecond, 169 | MaxBackoff: 600 * time.Millisecond, 170 | MaxRetries: 50, 171 | }) 172 | require.NoError(t, s.WaitSumMetrics(Equals(math.NaN()), "metric_a")) 173 | } 174 | 175 | func TestParseDockerPort(t *testing.T) { 176 | _, err := parseDockerIPv4Port("") 177 | assert.Error(t, err) 178 | 179 | actual, err := parseDockerIPv4Port("0.0.0.0:36999") 180 | assert.NoError(t, err) 181 | assert.Equal(t, 36999, actual) 182 | 183 | actual, err = parseDockerIPv4Port("0.0.0.0:49155\n:::49156") 184 | assert.NoError(t, err) 185 | assert.Equal(t, 49155, actual) 186 | 187 | actual, err = parseDockerIPv4Port(":::49156\n0.0.0.0:49155") 188 | assert.NoError(t, err) 189 | assert.Equal(t, 49155, actual) 190 | } 191 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "io" 7 | "io/ioutil" 8 | "math" 9 | "math/rand" 10 | "net/http" 11 | "os" 12 | "os/exec" 13 | "path/filepath" 14 | "strings" 15 | "time" 16 | 17 | "github.com/prometheus/common/model" 18 | "github.com/prometheus/prometheus/model/labels" 19 | "github.com/prometheus/prometheus/prompb" 20 | ) 21 | 22 | func RunCommandAndGetOutput(name string, args ...string) ([]byte, error) { 23 | cmd := exec.Command(name, args...) 24 | return cmd.CombinedOutput() 25 | } 26 | 27 | func RunCommandWithTimeoutAndGetOutput(timeout time.Duration, name string, args ...string) ([]byte, error) { 28 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 29 | defer cancel() 30 | 31 | cmd := exec.CommandContext(ctx, name, args...) 32 | return cmd.CombinedOutput() 33 | } 34 | 35 | func EmptyFlags() map[string]string { 36 | return map[string]string{} 37 | } 38 | 39 | func MergeFlags(inputs ...map[string]string) map[string]string { 40 | output := MergeFlagsWithoutRemovingEmpty(inputs...) 41 | 42 | for k, v := range output { 43 | if v == "" { 44 | delete(output, k) 45 | } 46 | } 47 | 48 | return output 49 | } 50 | 51 | func MergeFlagsWithoutRemovingEmpty(inputs ...map[string]string) map[string]string { 52 | output := map[string]string{} 53 | 54 | for _, input := range inputs { 55 | for name, value := range input { 56 | output[name] = value 57 | } 58 | } 59 | 60 | return output 61 | } 62 | 63 | func BuildArgs(flags map[string]string) []string { 64 | args := make([]string, 0, len(flags)) 65 | 66 | for name, value := range flags { 67 | if value != "" { 68 | args = append(args, name+"="+value) 69 | } else { 70 | args = append(args, name) 71 | } 72 | } 73 | 74 | return args 75 | } 76 | 77 | // DoGet performs an HTTP GET request towards the supplied URL and using a 78 | // timeout of 1 second. 79 | func DoGet(url string) (*http.Response, error) { 80 | return doRequestWithTimeout("GET", url, nil, nil, time.Second) 81 | } 82 | 83 | // DoGetWithTimeout performs an HTTP GET request towards the supplied URL and using a 84 | // specified timeout. 85 | func DoGetWithTimeout(url string, timeout time.Duration) (*http.Response, error) { 86 | return doRequestWithTimeout("GET", url, nil, nil, timeout) 87 | } 88 | 89 | // DoGetTLS is like DoGet but allows to configure a TLS config. 90 | func DoGetTLS(url string, tlsConfig *tls.Config) (*http.Response, error) { 91 | return doRequestWithTimeout("GET", url, nil, tlsConfig, time.Second) 92 | } 93 | 94 | // DoPost performs a HTTP POST request towards the supplied URL with an empty 95 | // body and using a timeout of 1 second. 96 | func DoPost(url string) (*http.Response, error) { 97 | return doRequestWithTimeout("POST", url, strings.NewReader(""), nil, time.Second) 98 | } 99 | 100 | func doRequestWithTimeout(method, url string, body io.Reader, tlsConfig *tls.Config, timeout time.Duration) (*http.Response, error) { 101 | req, err := http.NewRequest(method, url, body) 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | client := &http.Client{ 107 | Timeout: timeout, 108 | Transport: &http.Transport{ 109 | TLSClientConfig: tlsConfig, 110 | }, 111 | } 112 | 113 | return client.Do(req) 114 | } 115 | 116 | // TimeToMilliseconds returns the input time as milliseconds, using the same 117 | // formula used by Prometheus in order to get the same timestamp when asserting 118 | // on query results. The formula we're mimicking here is Prometheus parseTime(). 119 | // See: https://github.com/prometheus/prometheus/blob/df80dc4d3970121f2f76cba79050983ffb3cdbb0/web/api/v1/api.go#L1690-L1694 120 | func TimeToMilliseconds(t time.Time) int64 { 121 | // Convert to seconds. 122 | sec := float64(t.Unix()) + float64(t.Nanosecond())/1e9 123 | 124 | // Parse seconds. 125 | s, ns := math.Modf(sec) 126 | 127 | // Round nanoseconds part. 128 | ns = math.Round(ns*1000) / 1000 129 | 130 | // Convert to millis. 131 | return (int64(s) * 1e3) + (int64(ns * 1e3)) 132 | } 133 | 134 | func GenerateSeries(name string, ts time.Time, additionalLabels ...prompb.Label) (series []prompb.TimeSeries, vector model.Vector, matrix model.Matrix) { 135 | tsMillis := TimeToMilliseconds(ts) 136 | value := rand.Float64() 137 | 138 | lbls := append( 139 | []prompb.Label{ 140 | {Name: labels.MetricName, Value: name}, 141 | }, 142 | additionalLabels..., 143 | ) 144 | 145 | // Generate the series 146 | series = append(series, prompb.TimeSeries{ 147 | Labels: lbls, 148 | Exemplars: []prompb.Exemplar{ 149 | {Value: value, Timestamp: tsMillis, Labels: []prompb.Label{ 150 | {Name: "trace_id", Value: "1234"}, 151 | }}, 152 | }, 153 | Samples: []prompb.Sample{ 154 | {Value: value, Timestamp: tsMillis}, 155 | }, 156 | }) 157 | 158 | // Generate the expected vector and matrix when querying it 159 | metric := model.Metric{} 160 | metric[labels.MetricName] = model.LabelValue(name) 161 | for _, lbl := range additionalLabels { 162 | metric[model.LabelName(lbl.Name)] = model.LabelValue(lbl.Value) 163 | } 164 | 165 | vector = append(vector, &model.Sample{ 166 | Metric: metric, 167 | Value: model.SampleValue(value), 168 | Timestamp: model.Time(tsMillis), 169 | }) 170 | 171 | matrix = append(matrix, &model.SampleStream{ 172 | Metric: metric, 173 | Values: []model.SamplePair{ 174 | { 175 | Timestamp: model.Time(tsMillis), 176 | Value: model.SampleValue(value), 177 | }, 178 | }, 179 | }) 180 | 181 | return 182 | } 183 | 184 | func GenerateNSeries(nSeries, nExemplars int, name func() string, ts time.Time, additionalLabels func() []prompb.Label) (series []prompb.TimeSeries, vector model.Vector) { 185 | tsMillis := TimeToMilliseconds(ts) 186 | 187 | // Generate the series 188 | for i := 0; i < nSeries; i++ { 189 | lbls := []prompb.Label{ 190 | {Name: labels.MetricName, Value: name()}, 191 | } 192 | if additionalLabels != nil { 193 | lbls = append(lbls, additionalLabels()...) 194 | } 195 | 196 | value := rand.Float64() 197 | 198 | exemplars := []prompb.Exemplar{} 199 | if i < nExemplars { 200 | exemplars = []prompb.Exemplar{ 201 | {Value: value, Timestamp: tsMillis, Labels: []prompb.Label{{Name: "trace_id", Value: "1234"}}}, 202 | } 203 | } 204 | 205 | series = append(series, prompb.TimeSeries{ 206 | Labels: lbls, 207 | Samples: []prompb.Sample{ 208 | {Value: value, Timestamp: tsMillis}, 209 | }, 210 | Exemplars: exemplars, 211 | }) 212 | } 213 | 214 | // Generate the expected vector when querying it 215 | for i := 0; i < nSeries; i++ { 216 | metric := model.Metric{} 217 | for _, lbl := range series[i].Labels { 218 | metric[model.LabelName(lbl.Name)] = model.LabelValue(lbl.Value) 219 | } 220 | 221 | vector = append(vector, &model.Sample{ 222 | Metric: metric, 223 | Value: model.SampleValue(series[i].Samples[0].Value), 224 | Timestamp: model.Time(tsMillis), 225 | }) 226 | } 227 | return 228 | } 229 | 230 | // GetTempDirectory creates a temporary directory for shared integration 231 | // test files, either in the working directory or a directory referenced by 232 | // the E2E_TEMP_DIR environment variable 233 | func GetTempDirectory() (string, error) { 234 | var ( 235 | dir string 236 | err error 237 | ) 238 | // If a temp dir is referenced, return that 239 | if os.Getenv("E2E_TEMP_DIR") != "" { 240 | dir = os.Getenv("E2E_TEMP_DIR") 241 | } else { 242 | dir, err = os.Getwd() 243 | if err != nil { 244 | return "", err 245 | } 246 | } 247 | 248 | tmpDir, err := ioutil.TempDir(dir, "e2e_integration_test") 249 | if err != nil { 250 | return "", err 251 | } 252 | // Allow use of the temporary directory for testing with non-root 253 | // users. 254 | if err := os.Chmod(tmpDir, 0777); err != nil { 255 | return "", err 256 | } 257 | absDir, err := filepath.Abs(tmpDir) 258 | if err != nil { 259 | _ = os.RemoveAll(tmpDir) 260 | return "", err 261 | } 262 | 263 | return absDir, nil 264 | } 265 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestTimeToMilliseconds(t *testing.T) { 11 | input := time.Unix(0, 1614091978491118000) 12 | assert.Equal(t, int64(1614091978491), TimeToMilliseconds(input)) 13 | 14 | input = time.Unix(0, 1614091978505374000) 15 | assert.Equal(t, int64(1614091978505), TimeToMilliseconds(input)) 16 | 17 | input = time.Unix(0, 1614091921796500000) 18 | assert.Equal(t, int64(1614091921796), TimeToMilliseconds(input)) 19 | 20 | input = time.Unix(0, 1614092667692610000) 21 | assert.Equal(t, int64(1614092667693), TimeToMilliseconds(input)) 22 | } 23 | --------------------------------------------------------------------------------