├── .dockerignore ├── .github └── workflows │ └── push.yaml ├── .gitignore ├── .ionide └── symbolCache.db ├── .vscode └── launch.json ├── Dockerfile ├── LICENSE ├── README.md ├── api ├── api.go ├── health │ └── health.go └── metrics │ └── metrics.go ├── cmd └── dqd.go ├── config └── config.go ├── docs ├── examples │ └── azure.yaml ├── index.md └── sources │ ├── azure.md │ ├── service-bus.md │ └── sqs.md ├── go.mod ├── go.sum ├── handlers ├── handlers.go ├── http.go └── none.go ├── health └── probe.go ├── integration ├── docker │ ├── docker-compose.base.yaml │ ├── docker-compose.producer.yaml │ ├── docker-compose.worker.yaml │ ├── pipes │ │ ├── default.yaml │ │ ├── multi.yaml │ │ └── with-error.yaml │ ├── producer │ │ ├── Dockerfile │ │ └── entry.sh │ ├── providers │ │ ├── azure-local │ │ │ ├── config.yaml │ │ │ ├── data │ │ │ │ ├── __azurite_db_blob__.json │ │ │ │ ├── __azurite_db_blob_extent__.json │ │ │ │ ├── __azurite_db_queue__.json │ │ │ │ └── __azurite_db_queue_extent__.json │ │ │ └── docker-compose.yaml │ │ └── sqs-local │ │ │ ├── config.yaml │ │ │ └── docker-compose.yaml │ └── worker │ │ ├── Dockerfile │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go ├── envs │ ├── Tiltfile │ ├── docker-compose.yaml │ └── dqd.yaml └── tests │ ├── clean-azure.sh │ ├── error-azure.sh │ ├── multi.sh │ ├── naive-azure.sh │ └── naive-sqs.sh ├── listeners ├── http.go └── listener.go ├── main.go ├── metrics └── metrics.go ├── pipe ├── type.go └── worker.go ├── providers ├── azure │ └── azure.go ├── servicebus │ └── servicebus.go └── sqs │ └── sqs.go ├── utils ├── context.go ├── env.go ├── sources.go └── viper.go └── v1 ├── common.go ├── health.go └── request-context.go /.dockerignore: -------------------------------------------------------------------------------- 1 | integration/**/* 2 | Dockerfile 3 | docs/**/* 4 | benchmark/**/* 5 | .vscode -------------------------------------------------------------------------------- /.github/workflows/push.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | paths-ignore: 4 | - 'docs/**' 5 | - README.md 6 | jobs: 7 | test: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - name: install latest buildx 12 | run: mkdir -p ~/.docker/cli-plugins && wget -O ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.4.1/buildx-v0.4.1.linux-amd64 && chmod a+x ~/.docker/cli-plugins/docker-buildx 13 | - name: build 14 | env: 15 | DOCKER_BUILDKIT: 1 16 | DOCKER_CLI_EXPERIMENTAL: enabled 17 | working-directory: ./integration/docker 18 | run: docker buildx bake -f ./docker-compose.base.yaml -f ./docker-compose.worker.yaml -f ./docker-compose.producer.yaml --load 19 | - name: test-multi 20 | working-directory: ./integration/tests 21 | run: ./multi.sh 22 | - name: Login to GHCR 23 | uses: docker/login-action@v1 24 | with: 25 | registry: ghcr.io 26 | username: ${{ secrets.ORG_CR_WRITER_USER }} 27 | password: ${{ secrets.ORG_CR_WRITER_PASSWORD }} 28 | - name: Set up Docker Buildx 29 | uses: docker/setup-buildx-action@v1 30 | - id: get_version 31 | uses: battila7/get-version-action@v2 32 | - name: Push to GHCR 33 | uses: docker/build-push-action@v2 34 | if: ${{ startsWith(github.ref, 'refs/tags/') }} 35 | with: 36 | push: true 37 | tags: ghcr.io/soluto/dqd:${{ steps.get_version.outputs.version }} 38 | labels: | 39 | org.opencontainers.image.source=${{ github.event.repository.html_url }} 40 | - name: Push to Dockerhub 41 | uses: docker/build-push-action@v1 42 | env: 43 | DOCKER_BUILDKIT: 1 44 | with: 45 | username: solutodqd 46 | password: ${{ secrets.DOCKERHUB_TOKEN }} 47 | repository: soluto/dqd 48 | tag_with_sha: ${{ github.ref == 'refs/heads/master'}} 49 | tag_with_ref: ${{ github.ref != 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | node_modules 15 | .vscode/launch.json 16 | cloud-envs -------------------------------------------------------------------------------- /.ionide/symbolCache.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OS-Guild/dqd/3909df48251ef997c9d13b78db235e473ad2d066/.ionide/symbolCache.db -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch", 9 | "type": "go", 10 | "request": "launch", 11 | "mode": "debug", 12 | "remotePath": "", 13 | "port": 2345, 14 | "host": "127.0.0.1", 15 | "program": "${workspaceRoot}/src/main.go", 16 | "env": {}, 17 | "args": [], 18 | "showLog": true 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax = docker/dockerfile:1.0-experimental 2 | FROM golang:1.14.0-alpine as builder 3 | RUN apk update && apk add --no-cache git ca-certificates && update-ca-certificates 4 | 5 | ENV UID=10001 6 | RUN adduser \ 7 | --disabled-password \ 8 | --gecos "" \ 9 | --home "/nonexistent" \ 10 | --shell "/sbin/nologin" \ 11 | --no-create-home \ 12 | --uid "${UID}" \ 13 | appuser 14 | 15 | WORKDIR /src 16 | COPY go.mod go.sum ./ 17 | RUN --mount=type=cache,target=/root/.cache/go-build go mod download 18 | RUN go mod verify 19 | COPY . . 20 | 21 | RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /go/bin/dqd 22 | 23 | FROM scratch 24 | 25 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 26 | COPY --from=builder /go/bin/dqd /dqd 27 | COPY --from=builder /etc/passwd /etc/passwd 28 | COPY --from=builder /etc/group /etc/group 29 | USER appuser:appuser 30 | ENTRYPOINT ["/dqd"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Soluto 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DQD - Queue daemon 2 | 3 | DQD is a daemon that enables flows of consuming data from queue systems or pushing data, making it optimal 4 | We designed it to run as a sidecar container adjacent to an existing rest-service affectively, enabling the service to behave as a worker. 5 | 6 | ## Features 7 | 8 | - Multiple Providers support 9 | - Multiple pipelines per instance 10 | - Error handling 11 | - Backpressure with dynamic concurrency 12 | - Rate metrics 13 | - Single binary or Docker image 14 | 15 | ## Supported Providers 16 | 17 | - AWS SQS 18 | - Azure Queue 19 | - Azure Service bus 20 | 21 | # Usage 22 | 23 | ## Installing/Running 24 | 25 | ### Using Go 26 | 27 | run `go install github.com/soluto/dqd` 28 | run `dqd` 29 | 30 | ### Using Docker 31 | 32 | run `` docker run -v `pwd`:/etc/dqd soluto/dqd `` in a folder that conatins dqd.yaml 33 | 34 | ## Examples 35 | 36 | ### Consume and Process using an HTTP route 37 | 38 | The dqd.yaml should look like this: 39 | 40 | ``` 41 | sources: 42 | my-queue: 43 | type: sqs 44 | url: http://aws-sqs:9324/queue/my-sqs 45 | region: us-east-1 46 | pipe: 47 | source: my-queue 48 | rate: 49 | fixed: 1 50 | handler: 51 | http: 52 | endpoint: http://localhost:3000/processSqsMessages 53 | 54 | ``` 55 | 56 | ### Send to Queue Using DQD 57 | 58 | The dqd.yaml should look like this: 59 | 60 | ``` 61 | sources: 62 | my-queue: 63 | type: sqs 64 | url: http://aws-sqs:9324/queue/my-sqs 65 | region: us-east-1 66 | visibilityTimeoutInSeconds: 60 67 | ``` 68 | 69 | In your code, send the data to `http://localhost:9999/my-sqs` 70 | 71 | ### Example for DQD configuration in docker-compose 72 | 73 | ``` 74 | image: soluto/dqd 75 | ports: 76 | - containerPort: 8888 77 | protocol: TCP 78 | env: 79 | - name: AWS_SDK_LOAD_CONFIG 80 | value: 'true' 81 | ``` 82 | -------------------------------------------------------------------------------- /api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | 8 | "github.com/julienschmidt/httprouter" 9 | "github.com/soluto/dqd/api/health" 10 | "github.com/soluto/dqd/api/metrics" 11 | v1 "github.com/soluto/dqd/v1" 12 | ) 13 | 14 | func Start(ctx context.Context, port int, healthChecker v1.HealthChecker) error { 15 | router := httprouter.New() 16 | router.GET("/metrics", metrics.CreateMetricsHandler()) 17 | router.GET("/health", health.CreateHealthHandler(healthChecker)) 18 | srv := &http.Server{Addr: fmt.Sprintf(":%v", port)} 19 | e := make(chan error, 1) 20 | go func() { 21 | srv.Handler = router 22 | e <- srv.ListenAndServe() 23 | }() 24 | 25 | select { 26 | case err := <-e: 27 | return err 28 | case <-ctx.Done(): 29 | return nil 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /api/health/health.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | 7 | "github.com/julienschmidt/httprouter" 8 | v1 "github.com/soluto/dqd/v1" 9 | ) 10 | 11 | func CreateHealthHandler(healthChecker v1.HealthChecker) httprouter.Handle { 12 | return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { 13 | s := healthChecker.HealthStatus() 14 | if !s.IsHealthy() { 15 | w.WriteHeader(500) 16 | } 17 | json.NewEncoder(w).Encode(s) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /api/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/julienschmidt/httprouter" 7 | "github.com/prometheus/client_golang/prometheus" 8 | "github.com/prometheus/client_golang/prometheus/promhttp" 9 | "github.com/soluto/dqd/metrics" 10 | ) 11 | 12 | func CreateMetricsHandler() httprouter.Handle { 13 | prometheus.MustRegister( 14 | metrics.HandlerProcessingHistogram, 15 | metrics.PipeProcessingMessagesHistogram, 16 | metrics.WorkerBatchSizeGauge, 17 | metrics.WorkerMaxConcurrencyGauge, 18 | ) 19 | handler := promhttp.Handler() 20 | return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { 21 | handler.ServeHTTP(w, r) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /cmd/dqd.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | gofigure "github.com/NCAR/go-figure" 9 | "github.com/spf13/pflag" 10 | "github.com/spf13/viper" 11 | ) 12 | 13 | func ConfigurationError(err error) { 14 | fmt.Println(err) 15 | pflag.Usage() 16 | os.Exit(1) 17 | } 18 | 19 | func Load() (*viper.Viper, error) { 20 | v := viper.New() 21 | configFiles := pflag.CommandLine.StringSlice("config", []string{"./dqd.yaml"}, "Load a DQD config file") 22 | configDirs := pflag.CommandLine.StringSlice("configDir", []string{"/etc/dqd", "/dqd/config"}, "Lookup for config files in these folders") 23 | configOverrides := pflag.CommandLine.StringSlice("set", []string{}, "Override specific configuration keys") 24 | pflag.Parse() 25 | v.SetConfigType("yaml") 26 | v.SetEnvPrefix("dqd") 27 | v.AutomaticEnv() 28 | err := gofigure.Parse(v, *configDirs) 29 | if err != nil { 30 | panic(fmt.Errorf("Fatal error config file: %s \n", err)) 31 | } 32 | for _, f := range *configFiles { 33 | r, err := os.Open(f) 34 | defer r.Close() 35 | if err == nil { 36 | v.MergeConfig(r) 37 | } else { 38 | if f != "./dqd.yaml" { 39 | return nil, err 40 | } 41 | } 42 | 43 | } 44 | 45 | for _, override := range *configOverrides { 46 | entry := strings.Split(override, "=") 47 | if len(entry) != 2 { 48 | return nil, fmt.Errorf("invalid set value '%v'", override) 49 | } 50 | v.Set(entry[0], entry[1]) 51 | } 52 | 53 | return v, nil 54 | } 55 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/soluto/dqd/handlers" 7 | "github.com/soluto/dqd/listeners" 8 | "github.com/soluto/dqd/pipe" 9 | "github.com/soluto/dqd/providers/azure" 10 | "github.com/soluto/dqd/providers/servicebus" 11 | "github.com/soluto/dqd/providers/sqs" 12 | "github.com/soluto/dqd/utils" 13 | v1 "github.com/soluto/dqd/v1" 14 | "github.com/spf13/viper" 15 | ) 16 | 17 | type App struct { 18 | Sources map[string]*v1.Source 19 | Listeners []listeners.Listener 20 | Workers []*pipe.Worker 21 | } 22 | 23 | var sourceProviders = map[string]struct { 24 | v1.ConsumerFactory 25 | v1.ProducerFactory 26 | }{ 27 | "azure-queue": { 28 | &azure.AzureQueueClientFactory{}, 29 | &azure.AzureQueueClientFactory{}, 30 | }, 31 | "sqs": { 32 | &sqs.SQSClientFactory{}, 33 | &sqs.SQSClientFactory{}, 34 | }, 35 | "service-bus": { 36 | &servicebus.ServiceBusClientFactory{}, 37 | &servicebus.ServiceBusClientFactory{}, 38 | }, 39 | "io": { 40 | &utils.IoSourceFactory{}, 41 | &utils.IoSourceFactory{}, 42 | }, 43 | } 44 | 45 | func createSources(v *viper.Viper) map[string]*v1.Source { 46 | sources := map[string]*v1.Source{} 47 | for sourceName, subSource := range utils.ViperSubMap(v, "sources") { 48 | sourceType := subSource.GetString("type") 49 | factory, exist := sourceProviders[sourceType] 50 | if !exist { 51 | panic(fmt.Errorf("FATAL - Unkown source provider:%v", sourceType)) 52 | } 53 | sources[sourceName] = v1.NewSource(factory, factory, subSource, sourceName) 54 | } 55 | return sources 56 | } 57 | 58 | func getSource(sources map[string]*v1.Source, sourceName string) *v1.Source { 59 | source, exists := sources[sourceName] 60 | if !exists { 61 | panic(fmt.Sprintf("Missing source definition: %v", sourceName)) 62 | } 63 | return source 64 | } 65 | 66 | func getPipeSources(sources map[string]*v1.Source, v *viper.Viper) (pipeSources []*v1.Source) { 67 | sourcesConfig := v.GetStringSlice("sources") 68 | for _, s := range sourcesConfig { 69 | pipeSources = append(pipeSources, getSource(sources, s)) 70 | } 71 | if len(pipeSources) == 0 { 72 | pipeSources = []*v1.Source{getSource(sources, v.GetString("source"))} 73 | } 74 | return 75 | } 76 | 77 | func createHandler(v *viper.Viper) handlers.Handler { 78 | if v == nil { 79 | panic("no handler define for pipe, use 'none' handler if it's the desired behavior") 80 | } 81 | if v.Get("none") != nil { 82 | return handlers.None 83 | } 84 | v.SetDefault("http.path", "/") 85 | v.SetDefault("http.host", "localhost") 86 | v.SetDefault("http.port", 80) 87 | v.SetDefault("http.method", "POST") 88 | v.SetDefault("http.host", "") 89 | v.SetDefault("http.headers", map[string]string{}) 90 | 91 | httpEndpoint := v.GetString("http.endpoint") 92 | if httpEndpoint == "" { 93 | httpEndpoint = fmt.Sprintf("http://%v:%v%v", v.GetString("http.host"), v.GetString("http.port"), v.GetString("http.path")) 94 | } 95 | 96 | options := &handlers.HttpHandlerOptions{ 97 | Endpoint: httpEndpoint, 98 | Method: v.GetString("http.method"), 99 | Host: v.GetString("http.host"), 100 | Headers: v.GetStringMapString("http.headers"), 101 | } 102 | 103 | return handlers.NewHttpHandler(options) 104 | } 105 | 106 | func createWorkers(v *viper.Viper, sources map[string]*v1.Source) []*pipe.Worker { 107 | var wList []*pipe.Worker 108 | pipesConfig := utils.ViperSubMap(v, "pipes") 109 | for name, pipeConfig := range pipesConfig { 110 | pipeConfig.SetDefault("rate.init", 10) 111 | pipeConfig.SetDefault("rate.min", 1) 112 | pipeConfig.SetDefault("rate.window", "30s") 113 | pipeConfig.SetDefault("source", "default") 114 | handler := createHandler(pipeConfig.Sub("handler")) 115 | 116 | pipeSources := getPipeSources(sources, pipeConfig) 117 | 118 | var opts = []pipe.WorkerOption{} 119 | writeToSource := pipeConfig.GetString("onError.writeTo.source") 120 | if writeToSource != "" { 121 | opts = append(opts, pipe.WithErrorSource(getSource(sources, writeToSource))) 122 | } 123 | 124 | if pipeConfig.IsSet("rate.fixed") { 125 | opts = append(opts, pipe.WithFixedRate(pipeConfig.GetInt("rate.fixed"))) 126 | } else { 127 | opts = append(opts, pipe.WithDynamicRate(pipeConfig.GetInt("rate.init"), 128 | pipeConfig.GetInt("rate.min"), 129 | pipeConfig.GetDuration("rate.window"))) 130 | } 131 | output := pipeConfig.GetString("output") 132 | if output != "" { 133 | opts = append(opts, pipe.WithOutput(getSource(sources, output))) 134 | } else { 135 | opts = append(opts, pipe.WithDynamicRate(pipeConfig.GetInt("rate.init"), 136 | pipeConfig.GetInt("rate.min"), 137 | pipeConfig.GetDuration("rate.window"))) 138 | } 139 | 140 | wList = append(wList, pipe.NewWorker( 141 | name, 142 | pipeSources, 143 | handler, 144 | opts..., 145 | )) 146 | } 147 | return wList 148 | } 149 | 150 | func createListeners(v *viper.Viper, sources map[string]*v1.Source) []listeners.Listener { 151 | v.SetDefault("listeners.http.host", "0.0.0.0:9999") 152 | host := v.GetString("listeners.http.host") 153 | listener := listeners.Http(host) 154 | for _, s := range sources { 155 | listener.Add(s, viper.New()) 156 | } 157 | return []listeners.Listener{listener} 158 | } 159 | 160 | func CreateApp(v *viper.Viper) (_ *App, err error) { 161 | defer func() { 162 | if r := recover(); r != nil { 163 | err = fmt.Errorf("%v", r) 164 | } 165 | }() 166 | 167 | err = utils.NormalizeEntityConfig(v, "pipe", "pipes") 168 | if err != nil { 169 | return 170 | } 171 | err = utils.NormalizeEntityConfig(v, "source", "sources") 172 | if err != nil { 173 | return 174 | } 175 | 176 | sources := createSources(v) 177 | sources["stdout"] = v1.NewSource(&utils.IoSourceFactory{}, &utils.IoSourceFactory{}, &viper.Viper{}, "stdout") 178 | listeners := createListeners(v, sources) 179 | workers := createWorkers(v, sources) 180 | return &App{ 181 | sources, 182 | listeners, 183 | workers, 184 | }, nil 185 | } 186 | -------------------------------------------------------------------------------- /docs/examples/azure.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OS-Guild/dqd/3909df48251ef997c9d13b78db235e473ad2d066/docs/examples/azure.yaml -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # DQD 2 | 3 | -------------------------------------------------------------------------------- /docs/sources/azure.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ```yaml 4 | source: 5 | type: azure-queue 6 | 7 | # Location 8 | storageAccount: test 9 | queue: dqd 10 | #connection: http://azure:10001/devstoreaccount1 useful for local testing 11 | 12 | # Credentials 13 | storageAccountKey: **** 14 | sasToken: **** 15 | 16 | # Options 17 | visibilityTimeoutInSeconds: 100 # defaults to 60 18 | maxDequeueCount: 1 # deaults to 5 19 | retryVisiblityTimeoutInSeconds: [10, 500, 600] 20 | ``` -------------------------------------------------------------------------------- /docs/sources/service-bus.md: -------------------------------------------------------------------------------- 1 | ```yaml 2 | source: 3 | type: service-bus 4 | 5 | connectionString: "" 6 | topic: "my-topic" 7 | subscription: "my sub" 8 | 9 | prefetchCount: 30 #defaults to 1 10 | ``` -------------------------------------------------------------------------------- /docs/sources/sqs.md: -------------------------------------------------------------------------------- 1 | 2 | # SQS Source 3 | 4 | Credentials are taken by the aws sdk defaukt provider chain: 5 | https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html 6 | 7 | ```yaml 8 | source: 9 | type: sqs 10 | 11 | 12 | # Location 13 | url: 14 | region: us-east-1 15 | #endpoint: http://sqs:9324 useful for local testing 16 | 17 | # Options 18 | visibilityTimeoutInSeconds: 100 # defaults to 30 19 | ``` -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/soluto/dqd 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/Azure/azure-pipeline-go v0.1.8 7 | github.com/Azure/azure-service-bus-go v0.10.1 8 | github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd 9 | github.com/NCAR/go-figure v0.0.0-20181011044936-3924b68896e8 10 | github.com/aws/aws-sdk-go v1.30.12 11 | github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee // indirect 12 | github.com/google/uuid v1.1.1 13 | github.com/gorilla/mux v1.7.4 14 | github.com/influxdata/tdigest v0.0.1 // indirect 15 | github.com/jpillora/backoff v1.0.0 16 | github.com/julienschmidt/httprouter v1.2.0 17 | github.com/levigross/grequests v0.0.0-20190908174114-253788527a1a // indirect 18 | github.com/mailru/easyjson v0.7.1 // indirect 19 | github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 // indirect 20 | github.com/prometheus/client_golang v1.5.1 21 | github.com/rs/zerolog v1.18.0 22 | github.com/spf13/cast v1.3.0 23 | github.com/spf13/pflag v1.0.3 24 | github.com/spf13/viper v1.6.2 25 | github.com/tsenart/go-tsz v0.0.0-20180814235614-0bd30b3df1c3 // indirect 26 | github.com/tsenart/vegeta v12.7.0+incompatible // indirect 27 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect 28 | golang.org/x/text v0.3.2 // indirect 29 | gopkg.in/eapache/go-resiliency.v1 v1.2.0 30 | gopkg.in/h2non/gentleman.v2 v2.0.4 31 | ) 32 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/Azure/azure-amqp-common-go/v3 v3.0.0 h1:j9tjcwhypb/jek3raNrwlCIl7iKQYOug7CLpSyBBodc= 3 | github.com/Azure/azure-amqp-common-go/v3 v3.0.0/go.mod h1:SY08giD/XbhTz07tJdpw1SoxQXHPN30+DI3Z04SYqyg= 4 | github.com/Azure/azure-pipeline-go v0.1.8 h1:KmVRa8oFMaargVesEuuEoiLCQ4zCCwQ8QX/xg++KS20= 5 | github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= 6 | github.com/Azure/azure-sdk-for-go v37.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= 7 | github.com/Azure/azure-service-bus-go v0.10.1 h1:w9foWsHoOt1n8R0O58Co/ddrazx5vfDY0g64/6UWyuo= 8 | github.com/Azure/azure-service-bus-go v0.10.1/go.mod h1:E/FOceuKAFUfpbIJDKWz/May6guE+eGibfGT6q+n1to= 9 | github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= 10 | github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= 11 | github.com/Azure/go-amqp v0.12.6 h1:34yItuwhA/nusvq2sPSNPQxZLCf/CtaogYH8n578mnY= 12 | github.com/Azure/go-amqp v0.12.6/go.mod h1:qApuH6OFTSKZFmCOxccvAv5rLizBQf4v8pRmG138DPo= 13 | github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= 14 | github.com/Azure/go-autorest/autorest v0.9.3 h1:OZEIaBbMdUE/Js+BQKlpO81XlISgipr6yDJ+PSwsgi4= 15 | github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= 16 | github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= 17 | github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= 18 | github.com/Azure/go-autorest/autorest/adal v0.8.1 h1:pZdL8o72rK+avFWl+p9nE8RWi1JInZrWJYlnpfXJwHk= 19 | github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= 20 | github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= 21 | github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= 22 | github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= 23 | github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= 24 | github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= 25 | github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= 26 | github.com/Azure/go-autorest/autorest/to v0.3.0 h1:zebkZaadz7+wIQYgC7GXaz3Wb28yKYfVkkBKwc38VF8= 27 | github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= 28 | github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= 29 | github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= 30 | github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= 31 | github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= 32 | github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= 33 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 34 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 35 | github.com/NCAR/go-figure v0.0.0-20181011044936-3924b68896e8 h1:tiIt/Xklteljlg14899t2bd2/HtPqedF+GCQHHmaDDc= 36 | github.com/NCAR/go-figure v0.0.0-20181011044936-3924b68896e8/go.mod h1:BDb7YKe7GQP+n1imobaA1RSPSR4wJfHlrxj9V1c3Cc8= 37 | github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= 38 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 39 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 40 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 41 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 42 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 43 | github.com/aws/aws-sdk-go v1.30.12 h1:KrjyosZvkpJjcwMk0RNxMZewQ47v7+ZkbQDXjWsJMs8= 44 | github.com/aws/aws-sdk-go v1.30.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= 45 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 46 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 47 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 48 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 49 | github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee h1:BnPxIde0gjtTnc9Er7cxvBk8DHLWhEux0SxayC8dP6I= 50 | github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= 51 | github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= 52 | github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= 53 | github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= 54 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 55 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 56 | github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= 57 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 58 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 59 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 60 | github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 61 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 62 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 63 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 64 | github.com/devigned/tab v0.1.1 h1:3mD6Kb1mUOYeLpJvTVSDwSg5ZsfSxfvxGRTxRsJsITA= 65 | github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= 66 | github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= 67 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 68 | github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= 69 | github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= 70 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 71 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 72 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 73 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 74 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 75 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 76 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 77 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 78 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 79 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 80 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 81 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 82 | github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 83 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 84 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 85 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 86 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 87 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 88 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 89 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 90 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 91 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 92 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 93 | github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= 94 | github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 95 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 96 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 97 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 98 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= 99 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 100 | github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= 101 | github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 102 | github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 103 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= 104 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 105 | github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= 106 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 107 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 108 | github.com/influxdata/tdigest v0.0.1 h1:XpFptwYmnEKUqmkcDjrzffswZ3nvNeevbUSLPP/ZzIY= 109 | github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y= 110 | github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= 111 | github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= 112 | github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= 113 | github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 114 | github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= 115 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 116 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 117 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 118 | github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= 119 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 120 | github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= 121 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 122 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 123 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 124 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 125 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 126 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 127 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 128 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 129 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 130 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 131 | github.com/levigross/grequests v0.0.0-20190908174114-253788527a1a h1:DGFy/362j92vQRE3ThU1yqg9TuJS8YJOSbQuB7BP9cA= 132 | github.com/levigross/grequests v0.0.0-20190908174114-253788527a1a/go.mod h1:jVntzcUU+2BtVohZBQmSHWUmh8B55LCNfPhcNCIvvIg= 133 | github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= 134 | github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 135 | github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= 136 | github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= 137 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 138 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 139 | github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= 140 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 141 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 142 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 143 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 144 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 145 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 146 | github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= 147 | github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= 148 | github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= 149 | github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= 150 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 151 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 152 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 153 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 154 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 155 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 156 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 157 | github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= 158 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 159 | github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= 160 | github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= 161 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 162 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 163 | github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= 164 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 165 | github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 166 | github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 167 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 168 | github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= 169 | github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= 170 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 171 | github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 172 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 173 | github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= 174 | github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= 175 | github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= 176 | github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= 177 | github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= 178 | github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8= 179 | github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I= 180 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 181 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 182 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= 183 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 184 | github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= 185 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 186 | github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= 187 | github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 188 | github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= 189 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 190 | github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= 191 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 192 | github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= 193 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 194 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 195 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 196 | github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= 197 | github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= 198 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 199 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 200 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 201 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 202 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 203 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 204 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 205 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 206 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 207 | github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= 208 | github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= 209 | github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 210 | github.com/tsenart/go-tsz v0.0.0-20180814235614-0bd30b3df1c3 h1:pcQGQzTwCg//7FgVywqge1sW9Yf8VMsMdG58MI5kd8s= 211 | github.com/tsenart/go-tsz v0.0.0-20180814235614-0bd30b3df1c3/go.mod h1:SWZznP1z5Ki7hDT2ioqiFKEse8K9tU2OUvaRI0NeGQo= 212 | github.com/tsenart/vegeta v1.2.0 h1:ootdZOMPo6TDg4SnP9OTbdqc1apYEjhtgQi3fPyB/RA= 213 | github.com/tsenart/vegeta v12.7.0+incompatible h1:sGlrv11EMxQoKOlDuMWR23UdL90LE5VlhKw/6PWkZmU= 214 | github.com/tsenart/vegeta v12.7.0+incompatible/go.mod h1:Smz/ZWfhKRcyDDChZkG3CyTHdj87lHzio/HOCkbndXM= 215 | github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= 216 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 217 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 218 | github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= 219 | go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= 220 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 221 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 222 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 223 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 224 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 225 | golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= 226 | golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 227 | golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 228 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 229 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 230 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 231 | golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 232 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 233 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 234 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 235 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 236 | golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 237 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 238 | golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 239 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= 240 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 241 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 242 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= 243 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 244 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 245 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 246 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 247 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 248 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 249 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 250 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 251 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 252 | golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 253 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 254 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 255 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 256 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 257 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= 258 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 259 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= 260 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 261 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 262 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 263 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 264 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 265 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 266 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 267 | golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 268 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 269 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 270 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 271 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 272 | golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 273 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 274 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 275 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 276 | gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= 277 | gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= 278 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 279 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 280 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 281 | google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 282 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 283 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 284 | gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 285 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 286 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 287 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 288 | gopkg.in/eapache/go-resiliency.v1 v1.2.0 h1:Ga62yQGVh5jQ/k6rDYhn2UsV9evgp2ZmMwXgGu2YcOQ= 289 | gopkg.in/eapache/go-resiliency.v1 v1.2.0/go.mod h1:ufQ2tre3XZoQT9X8nKYgTaqO8DrIudC5V1EOYUwIka0= 290 | gopkg.in/h2non/gentleman.v2 v2.0.4 h1:9R3K6CFYd/RdXDLi0pGXwaPnRx/pn5EZlrN3VkNygWc= 291 | gopkg.in/h2non/gentleman.v2 v2.0.4/go.mod h1:A1c7zwrTgAyyf6AbpvVksYtBayTB4STBUGmdkEtlHeA= 292 | gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= 293 | gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 294 | gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= 295 | gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 296 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 297 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 298 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 299 | gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= 300 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 301 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 302 | -------------------------------------------------------------------------------- /handlers/handlers.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "github.com/rs/zerolog/log" 5 | v1 "github.com/soluto/dqd/v1" 6 | ) 7 | 8 | var logger = log.With().Str("scope", "Handler").Logger() 9 | 10 | type HandlerErrorCode int 11 | 12 | type HandlerError interface { 13 | error 14 | Code() HandlerErrorCode 15 | } 16 | 17 | type handlerError struct { 18 | code int 19 | error error 20 | } 21 | 22 | func (e *handlerError) Code() HandlerErrorCode { 23 | return HandlerErrorCode(e.code) 24 | } 25 | 26 | func (e *handlerError) Error() string { 27 | return e.error.Error() 28 | } 29 | 30 | func ServerError(err error) HandlerError { 31 | return &handlerError{ 32 | 5, 33 | err, 34 | } 35 | } 36 | 37 | func BadRequestError(err error) HandlerError { 38 | return &handlerError{ 39 | 4, 40 | err, 41 | } 42 | } 43 | 44 | // Handler handles queue messages. 45 | type Handler interface { 46 | v1.HealthChecker 47 | Handle(*v1.RequestContext, v1.Message) (*v1.RawMessage, HandlerError) 48 | } 49 | -------------------------------------------------------------------------------- /handlers/http.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/url" 7 | "time" 8 | 9 | "github.com/rs/zerolog/log" 10 | v1 "github.com/soluto/dqd/v1" 11 | "gopkg.in/h2non/gentleman.v2" 12 | "gopkg.in/h2non/gentleman.v2/plugins/timeout" 13 | ) 14 | 15 | type httpHandler struct { 16 | baseUrl *url.URL 17 | client *gentleman.Client 18 | } 19 | 20 | type HttpHandlerOptions struct { 21 | Endpoint string 22 | Method string 23 | Host string 24 | Headers map[string]string 25 | } 26 | 27 | var handlerLogger = log.With().Str("scope", "Handler") 28 | 29 | func (h *httpHandler) HealthStatus() v1.HealthStatus { 30 | conn, err := net.Dial("tcp", h.baseUrl.Host) 31 | status := v1.Healthy 32 | if err != nil { 33 | status = v1.Error(err) 34 | } else { 35 | net.Conn(conn).Close() 36 | } 37 | 38 | return v1.HealthStatus{ 39 | "": status, 40 | } 41 | } 42 | 43 | func (h *httpHandler) Handle(ctx *v1.RequestContext, message v1.Message) (*v1.RawMessage, HandlerError) { 44 | res, err := h.client.Post().AddHeader("x-dqd-source", ctx.Source()).JSON(message.Data()).Send() 45 | if err != nil { 46 | return nil, ServerError(err) 47 | } 48 | if err == nil { 49 | if res.ServerError { 50 | return nil, ServerError(fmt.Errorf("invalid server response: %d", res.StatusCode)) 51 | } 52 | if res.ClientError { 53 | return nil, BadRequestError(fmt.Errorf("invalid client response: %d", res.StatusCode)) 54 | } 55 | } 56 | return &v1.RawMessage{ 57 | Data: res.String(), 58 | }, nil 59 | } 60 | 61 | func NewHttpHandler(options *HttpHandlerOptions) Handler { 62 | client := gentleman.New(). 63 | URL(options.Endpoint). 64 | Method(options.Method). 65 | Use(timeout.Request(2 * time.Minute)) 66 | 67 | baseUrl, _ := url.Parse(options.Endpoint) 68 | 69 | if options.Host != "" { 70 | client.AddHeader("Host", options.Host) 71 | } 72 | 73 | if options.Headers != nil { 74 | for header, value := range options.Headers { 75 | client.AddHeader(header, value) 76 | } 77 | } 78 | 79 | return &httpHandler{ 80 | baseUrl, 81 | client, 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /handlers/none.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | v1 "github.com/soluto/dqd/v1" 5 | ) 6 | 7 | type noneHandler struct { 8 | } 9 | 10 | func (h *noneHandler) Available() error { 11 | return nil 12 | } 13 | 14 | func (h *noneHandler) Handle(ctx *v1.RequestContext, message v1.Message) (*v1.RawMessage, HandlerError) { 15 | return &v1.RawMessage{message.Data()}, nil 16 | } 17 | 18 | func (h *noneHandler) HealthStatus() v1.HealthStatus { 19 | return v1.NewHealthStatus(v1.Healthy) 20 | } 21 | 22 | var None = &noneHandler{} 23 | -------------------------------------------------------------------------------- /health/probe.go: -------------------------------------------------------------------------------- 1 | package health 2 | 3 | import v1 "github.com/soluto/dqd/v1" 4 | 5 | type Probe struct { 6 | inputs chan struct { 7 | v1.HealthStatus 8 | string 9 | } 10 | checks chan struct { 11 | v1.HealthChecker 12 | string 13 | } 14 | current v1.HealthStatus 15 | } 16 | 17 | func (p *Probe) UpdateStatus(status v1.HealthStatus, prefix string) { 18 | p.inputs <- struct { 19 | v1.HealthStatus 20 | string 21 | }{status, prefix} 22 | } 23 | 24 | func (p *Probe) SendCheck(checker v1.HealthChecker, prefix string) { 25 | p.checks <- struct { 26 | v1.HealthChecker 27 | string 28 | }{checker, prefix} 29 | } 30 | 31 | func (p *Probe) HealthStatus() v1.HealthStatus { 32 | return p.current 33 | } 34 | 35 | func (p *Probe) run() { 36 | for { 37 | select { 38 | case m := <-p.checks: 39 | p.current.Add(m.HealthChecker.HealthStatus(), m.string) 40 | case m := <-p.inputs: 41 | p.current.Add(m.HealthStatus, m.string) 42 | } 43 | } 44 | } 45 | 46 | func MakeProbe() *Probe { 47 | p := &Probe{ 48 | inputs: make(chan struct { 49 | v1.HealthStatus 50 | string 51 | }), 52 | checks: make(chan struct { 53 | v1.HealthChecker 54 | string 55 | }), 56 | current: v1.HealthStatus{}, 57 | } 58 | go p.run() 59 | return p 60 | } 61 | -------------------------------------------------------------------------------- /integration/docker/docker-compose.base.yaml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | dqd: 5 | build: "../.." 6 | image: soluto/dqd 7 | volumes: 8 | - ./pipes/${PIPES_CONFIG:-default}.yaml:/etc/dqd/pipes.yaml 9 | ports: 10 | - 9999:9999 11 | - 8888:8888 -------------------------------------------------------------------------------- /integration/docker/docker-compose.producer.yaml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | producer: 5 | build: "./producer" 6 | network_mode: service:dqd 7 | depends_on: 8 | - dqd 9 | environment: 10 | - SOURCE=${PRODUCER_SOURCE:-messages} 11 | - MESSAGES_COUNT=${MESSAGES_COUNT:-1000} -------------------------------------------------------------------------------- /integration/docker/docker-compose.worker.yaml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | worker: 5 | build: "./worker" 6 | network_mode: service:dqd 7 | depends_on: 8 | - dqd 9 | environment: 10 | - MESSAGES_COUNT=${MESSAGES_COUNT:-1000} -------------------------------------------------------------------------------- /integration/docker/pipes/default.yaml: -------------------------------------------------------------------------------- 1 | pipe: 2 | source: messages 3 | handler: 4 | http: 5 | rate: 6 | window: 5s -------------------------------------------------------------------------------- /integration/docker/pipes/multi.yaml: -------------------------------------------------------------------------------- 1 | pipes: 2 | - source: sqs 3 | handler: 4 | http: 5 | path: "/?no-increment=true" 6 | output: azure 7 | rate: 8 | window: 5s 9 | - source: azure 10 | handler: 11 | http: 12 | path: "/?error=0.1" 13 | onError: 14 | writeTo: 15 | source: azure-error 16 | rate: 17 | window: 5s 18 | - source: azure-error 19 | handler: 20 | http: 21 | headers: 22 | x-client-id: testing 23 | path: "/?require-header=X-Client-Id,testing" 24 | -------------------------------------------------------------------------------- /integration/docker/pipes/with-error.yaml: -------------------------------------------------------------------------------- 1 | pipes: 2 | - source: messages 3 | handler: 4 | http: 5 | path: "/?error=0.1" 6 | onError: 7 | writeTo: 8 | source: messages-error 9 | rate: 10 | window: 5s 11 | - source: messages-error 12 | handler: 13 | http: -------------------------------------------------------------------------------- /integration/docker/producer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.10.1 2 | 3 | ENV VEGETA_VERSION 12.8.3 4 | 5 | RUN set -ex \ 6 | && apk add --no-cache ca-certificates \ 7 | && apk add --no-cache --virtual .build-deps \ 8 | openssl \ 9 | && wget -q "https://github.com/tsenart/vegeta/releases/download/v$VEGETA_VERSION/vegeta-$VEGETA_VERSION-linux-amd64.tar.gz" -O /tmp/vegeta.tar.gz \ 10 | && cd bin \ 11 | && tar xzf /tmp/vegeta.tar.gz \ 12 | && rm /tmp/vegeta.tar.gz \ 13 | && apk del .build-deps 14 | 15 | COPY entry.sh / 16 | RUN chmod +x entry.sh 17 | 18 | ENV TARGET=http://localhost:9999 19 | ENV DATA="{""test"": ""abcd""}" 20 | ENV MESSAGES_COUNT=1000 21 | ENV DURATION=10s 22 | 23 | ENTRYPOINT [ "/entry.sh" ] -------------------------------------------------------------------------------- /integration/docker/producer/entry.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | echo $DATA > /test.data 3 | sleep 3 4 | echo "attacking $TARGET/$SOURCE at $MESSAGES_COUNT/$DURATION" 5 | echo "POST $TARGET/$SOURCE" | /bin/vegeta attack -rate "$MESSAGES_COUNT/$DURATION" -duration $DURATION -body /test.data "$@" | vegeta report 6 | tail -f /dev/null -------------------------------------------------------------------------------- /integration/docker/providers/azure-local/config.yaml: -------------------------------------------------------------------------------- 1 | sources: 2 | azure: 3 | type: azure-queue 4 | sasToken: "?st=2020-04-27T13%3A57%3A25Z&se=2021-04-28T13%3A57%3A00Z&sp=raup&sv=2018-03-28&sig=ZzcxmPYHU7O1B4ixnpsKtxqSAGKsHqUf1cI%2FNbu6Qew%3D" 5 | connection: http://azure:10001/devstoreaccount1 6 | maxDequeueCount: 0 7 | queue: dqd 8 | azure-error: 9 | type: azure-queue 10 | sasToken: "?st=2020-04-27T15%3A35%3A59Z&se=2021-04-28T15%3A35%3A00Z&sp=raup&sv=2018-03-28&sig=mpll9GgIR3SULQFAOgCYkg%2FEq1LIbP8Z%2FBkgn40QRM4%3D" 11 | connection: http://azure:10001/devstoreaccount1 12 | queue: dqd-error 13 | messages: 14 | type: azure-queue 15 | sasToken: "?st=2020-04-27T13%3A57%3A25Z&se=2021-04-28T13%3A57%3A00Z&sp=raup&sv=2018-03-28&sig=ZzcxmPYHU7O1B4ixnpsKtxqSAGKsHqUf1cI%2FNbu6Qew%3D" 16 | connection: http://azure:10001/devstoreaccount1 17 | maxDequeueCount: 0 18 | queue: dqd 19 | messages-error: 20 | type: azure-queue 21 | sasToken: "?st=2020-04-27T15%3A35%3A59Z&se=2021-04-28T15%3A35%3A00Z&sp=raup&sv=2018-03-28&sig=mpll9GgIR3SULQFAOgCYkg%2FEq1LIbP8Z%2FBkgn40QRM4%3D" 22 | connection: http://azure:10001/devstoreaccount1 23 | queue: dqd-error 24 | 25 | -------------------------------------------------------------------------------- /integration/docker/providers/azure-local/data/__azurite_db_blob__.json: -------------------------------------------------------------------------------- 1 | {"filename":"/data/__azurite_db_blob__.json","collections":[{"name":"$SERVICES_COLLECTION$","data":[],"idIndex":[],"binaryIndices":{},"constraints":null,"uniqueNames":["accountName"],"transforms":{},"objType":"$SERVICES_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[]},{"name":"$CONTAINERS_COLLECTION$","data":[],"idIndex":[],"binaryIndices":{"accountName":{"name":"accountName","dirty":false,"values":[]},"name":{"name":"name","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$CONTAINERS_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[]},{"name":"$BLOBS_COLLECTION$","data":[],"idIndex":[],"binaryIndices":{"accountName":{"name":"accountName","dirty":false,"values":[]},"containerName":{"name":"containerName","dirty":false,"values":[]},"name":{"name":"name","dirty":false,"values":[]},"snapshot":{"name":"snapshot","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$BLOBS_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[]},{"name":"$BLOCKS_COLLECTION$","data":[],"idIndex":[],"binaryIndices":{"accountName":{"name":"accountName","dirty":false,"values":[]},"containerName":{"name":"containerName","dirty":false,"values":[]},"blobName":{"name":"blobName","dirty":false,"values":[]},"name":{"name":"name","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$BLOCKS_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[]}],"databaseVersion":1.5,"engineVersion":1.5,"autosave":true,"autosaveInterval":5000,"autosaveHandle":null,"throttledSaves":true,"options":{"autosave":true,"autosaveInterval":5000,"serializationMethod":"normal","destructureDelimiter":"$<\n"},"persistenceMethod":"fs","persistenceAdapter":null,"verbose":false,"events":{"init":[null],"loaded":[],"flushChanges":[],"close":[],"changes":[],"warning":[]},"ENV":"NODEJS"} -------------------------------------------------------------------------------- /integration/docker/providers/azure-local/data/__azurite_db_blob_extent__.json: -------------------------------------------------------------------------------- 1 | {"filename":"/data/__azurite_db_blob_extent__.json","collections":[{"name":"$EXTENTS_COLLECTION$","data":[],"idIndex":[],"binaryIndices":{"id":{"name":"id","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$EXTENTS_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[]}],"databaseVersion":1.5,"engineVersion":1.5,"autosave":true,"autosaveInterval":5000,"autosaveHandle":null,"throttledSaves":true,"options":{"autosave":true,"autosaveInterval":5000,"serializationMethod":"normal","destructureDelimiter":"$<\n"},"persistenceMethod":"fs","persistenceAdapter":null,"verbose":false,"events":{"init":[null],"loaded":[],"flushChanges":[],"close":[],"changes":[],"warning":[]},"ENV":"NODEJS"} -------------------------------------------------------------------------------- /integration/docker/providers/azure-local/data/__azurite_db_queue__.json: -------------------------------------------------------------------------------- 1 | {"filename":"/data/__azurite_db_queue__.json","collections":[{"name":"$SERVICES_COLLECTION$","data":[],"idIndex":[],"binaryIndices":{},"constraints":null,"uniqueNames":["accountName"],"transforms":{},"objType":"$SERVICES_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"ttl":null,"maxId":0,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[]},{"name":"$QUEUES_COLLECTION$","data":[{"accountName":"devstoreaccount1","name":"dqd","metadata":{},"meta":{"revision":0,"created":1587996735272,"version":0},"$loki":1},{"accountName":"devstoreaccount1","name":"dqd-error","metadata":{},"meta":{"revision":0,"created":1588002644182,"version":0},"$loki":3}],"idIndex":[1,3],"binaryIndices":{"accountName":{"name":"accountName","dirty":false,"values":[1,0]},"name":{"name":"name","dirty":false,"values":[0,1]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$QUEUES_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"ttl":null,"maxId":3,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[]},{"name":"$MESSAGES_COLLECTION$","data":[],"idIndex":[],"binaryIndices":{"accountName":{"name":"accountName","dirty":false,"values":[]},"queueName":{"name":"queueName","dirty":false,"values":[]},"messageId":{"name":"messageId","dirty":false,"values":[]},"visibleTime":{"name":"visibleTime","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$MESSAGES_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"ttl":null,"maxId":2908,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[]}],"databaseVersion":1.5,"engineVersion":1.5,"autosave":true,"autosaveInterval":5000,"autosaveHandle":null,"throttledSaves":true,"options":{"autosave":true,"autosaveInterval":5000,"serializationMethod":"normal","destructureDelimiter":"$<\n"},"persistenceMethod":"fs","persistenceAdapter":null,"verbose":false,"events":{"init":[null],"loaded":[],"flushChanges":[],"close":[],"changes":[],"warning":[]},"ENV":"NODEJS"} -------------------------------------------------------------------------------- /integration/docker/providers/azure-local/data/__azurite_db_queue_extent__.json: -------------------------------------------------------------------------------- 1 | {"filename":"/data/__azurite_db_queue_extent__.json","collections":[{"name":"$EXTENTS_COLLECTION$","data":[],"idIndex":[],"binaryIndices":{"id":{"name":"id","dirty":false,"values":[]}},"constraints":null,"uniqueNames":[],"transforms":{},"objType":"$EXTENTS_COLLECTION$","dirty":false,"cachedIndex":null,"cachedBinaryIndex":null,"cachedData":null,"adaptiveBinaryIndices":true,"transactional":false,"cloneObjects":false,"cloneMethod":"parse-stringify","asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"autoupdate":false,"serializableIndices":true,"ttl":null,"maxId":5,"DynamicViews":[],"events":{"insert":[],"update":[],"pre-insert":[],"pre-update":[],"close":[],"flushbuffer":[],"error":[],"delete":[null],"warning":[null]},"changes":[]}],"databaseVersion":1.5,"engineVersion":1.5,"autosave":true,"autosaveInterval":5000,"autosaveHandle":null,"throttledSaves":true,"options":{"autosave":true,"autosaveInterval":5000,"serializationMethod":"normal","destructureDelimiter":"$<\n"},"persistenceMethod":"fs","persistenceAdapter":null,"verbose":false,"events":{"init":[null],"loaded":[],"flushChanges":[],"close":[],"changes":[],"warning":[]},"ENV":"NODEJS"} -------------------------------------------------------------------------------- /integration/docker/providers/azure-local/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | azure: 5 | image: mcr.microsoft.com/azure-storage/azurite 6 | logging: 7 | driver: none 8 | volumes: 9 | - ./providers/azure-local/data:/data 10 | ports: 11 | - "10001:10001" 12 | dqd: 13 | depends_on: 14 | - azure 15 | volumes: 16 | - ./providers/azure-local/config.yaml:/etc/dqd/azure.yaml 17 | -------------------------------------------------------------------------------- /integration/docker/providers/sqs-local/config.yaml: -------------------------------------------------------------------------------- 1 | sources: 2 | messages: 3 | type: sqs 4 | endpoint: http://sqs:9324 5 | url: http://sqs:9324/queue/default 6 | region: us-east-1 7 | sqs: 8 | type: sqs 9 | endpoint: http://sqs:9324 10 | url: http://sqs:9324/queue/default 11 | region: us-east-1 -------------------------------------------------------------------------------- /integration/docker/providers/sqs-local/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | sqs: 5 | container_name: sqs 6 | image: roribio16/alpine-sqs 7 | #logging: 8 | # driver: none 9 | ports: 10 | - "9325:9325" 11 | dqd: 12 | depends_on: 13 | - sqs 14 | environment: 15 | - AWS_REGION=us-east-1 16 | - AWS_ACCESS_KEY_ID="" 17 | - AWS_SECRET_ACCESS_KEY="" 18 | volumes: 19 | - ./providers/sqs-local/config.yaml:/etc/dqd/sqs.yaml 20 | -------------------------------------------------------------------------------- /integration/docker/worker/Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax = docker/dockerfile:1.0-experimental 2 | FROM golang:1.14.0-alpine as builder 3 | RUN apk update && apk add --no-cache git ca-certificates && update-ca-certificates 4 | 5 | WORKDIR /src 6 | COPY go.mod go.sum ./ 7 | RUN go mod download 8 | RUN go mod verify 9 | COPY . . 10 | 11 | RUN --mount=type=cache,target=/root/.cache/go-build go build -o /go/bin/worker 12 | 13 | ENTRYPOINT ["/go/bin/worker"] -------------------------------------------------------------------------------- /integration/docker/worker/go.mod: -------------------------------------------------------------------------------- 1 | module worker 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/gorilla/mux v1.7.4 7 | github.com/spf13/viper v1.6.3 8 | ) 9 | -------------------------------------------------------------------------------- /integration/docker/worker/go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 3 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 4 | github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= 5 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 6 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 7 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 8 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 9 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 10 | github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= 11 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 12 | github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= 13 | github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 14 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 15 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 16 | github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 17 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 18 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 19 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 20 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 21 | github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= 22 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 23 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 24 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 25 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 26 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 27 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 28 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 29 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 30 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 31 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 32 | github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 33 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 34 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 35 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 36 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 37 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 38 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 39 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= 40 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 41 | github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= 42 | github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 43 | github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 44 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= 45 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 46 | github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= 47 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 48 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 49 | github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 50 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 51 | github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= 52 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 53 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 54 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 55 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 56 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 57 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 58 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 59 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 60 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 61 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 62 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 63 | github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= 64 | github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 65 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 66 | github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= 67 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 68 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 69 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 70 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 71 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 72 | github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= 73 | github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= 74 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 75 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 76 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 77 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 78 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 79 | github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= 80 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 81 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 82 | github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 83 | github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 84 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 85 | github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 86 | github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= 87 | github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= 88 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 89 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= 90 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 91 | github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= 92 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 93 | github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= 94 | github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 95 | github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= 96 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 97 | github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= 98 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 99 | github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= 100 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 101 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 102 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 103 | github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs= 104 | github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw= 105 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 106 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 107 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 108 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 109 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 110 | github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= 111 | github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= 112 | github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 113 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 114 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 115 | go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= 116 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 117 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 118 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 119 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 120 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 121 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 122 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 123 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 124 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 125 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 126 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 127 | golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 128 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 129 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 130 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 131 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 132 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 133 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 134 | golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 135 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 136 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= 137 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 138 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 139 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 140 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 141 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 142 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 143 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 144 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 145 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 146 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 147 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 148 | google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 149 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 150 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 151 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 152 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 153 | gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= 154 | gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 155 | gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= 156 | gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 157 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 158 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 159 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 160 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 161 | -------------------------------------------------------------------------------- /integration/docker/worker/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "math/rand" 7 | "net/http" 8 | "os" 9 | "strconv" 10 | "strings" 11 | "sync/atomic" 12 | "time" 13 | 14 | "github.com/gorilla/mux" 15 | "github.com/spf13/viper" 16 | ) 17 | 18 | func main() { 19 | rand.Seed(10) 20 | viper.SetDefault("MESSAGES_COUNT", -1) 21 | viper.AutomaticEnv() 22 | ec := viper.GetInt64("MESSAGES_COUNT") 23 | fmt.Printf("Expecting %v messages", ec) 24 | router := mux.NewRouter() 25 | n := int64(0) 26 | t := int64(0) 27 | m := make(chan bool) 28 | router.Methods("post").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 29 | m <- true 30 | mNumber := atomic.AddInt64(&n, 1) 31 | body, _ := ioutil.ReadAll(r.Body) 32 | fmt.Printf("got %v:%v:%v", mNumber, r.Header.Get("x-dqd-source"), string(body)) 33 | time.Sleep(100 * time.Millisecond) 34 | sErrRate := r.URL.Query().Get("error") 35 | if sErrRate != "" { 36 | errRate, _ := strconv.ParseFloat(sErrRate, 64) 37 | if rand.Float64() < errRate { 38 | w.WriteHeader(500) 39 | return 40 | } 41 | } 42 | 43 | requireHeader := r.URL.Query().Get("require-header") 44 | if requireHeader != "" { 45 | header := strings.Split(requireHeader, ",") 46 | if r.Header.Get(header[0]) != header[1] { 47 | w.WriteHeader(500) 48 | return 49 | } 50 | } 51 | 52 | incremnt := !(r.URL.Query().Get("no-increment") == "true") 53 | w.WriteHeader(200) 54 | p := t 55 | if incremnt { 56 | p = atomic.AddInt64(&t, 1) 57 | fmt.Printf("handled %v message\n", p) 58 | } 59 | if p != -1 && p == ec { 60 | time.AfterFunc(1*time.Second, func() { os.Exit(0) }) 61 | } 62 | w.Write(body) 63 | }) 64 | println("Listening") 65 | go func() { 66 | <-m 67 | for { 68 | select { 69 | case <-m: 70 | case <-time.After(10 * time.Second): 71 | os.Exit(1) 72 | } 73 | } 74 | 75 | }() 76 | http.ListenAndServe("0.0.0.0:80", router) 77 | } 78 | -------------------------------------------------------------------------------- /integration/envs/Tiltfile: -------------------------------------------------------------------------------- 1 | docker_compose("./docker-compose.yaml") 2 | docker_build("soluto/dqd", "../..") -------------------------------------------------------------------------------- /integration/envs/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | sqs: 5 | image: roribio16/alpine-sqs 6 | logging: 7 | driver: none 8 | ports: 9 | - "9325:9325" 10 | dqd: 11 | build: "../.." 12 | image: soluto/dqd 13 | depends_on: 14 | - sqs 15 | environment: 16 | - AWS_REGION=us-east-1 17 | - AWS_ACCESS_KEY_ID="" 18 | - AWS_SECRET_ACCESS_KEY="" 19 | volumes: 20 | - ./dqd.yaml:/etc/dqd/dqd.yaml 21 | ports: 22 | - 9999:9999 23 | - 8888:8888 -------------------------------------------------------------------------------- /integration/envs/dqd.yaml: -------------------------------------------------------------------------------- 1 | source: 2 | type: sqs 3 | endpoint: http://sqs:9324 4 | url: http://sqs:9324/queue/default 5 | region: us-east-1 6 | pipe: 7 | handler: 8 | none: {} 9 | output: stdout -------------------------------------------------------------------------------- /integration/tests/clean-azure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | git restore -s@ -SW -- ../docker/providers/azure-local/data || true 3 | rm -f ../docker/providers/azure-local/data/__queuestorage__/* || true -------------------------------------------------------------------------------- /integration/tests/error-azure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ./clean-azure.sh 3 | docker-compose -f ../docker/docker-compose.base.yaml down --remove-orphans 4 | PIPES_CONFIG=with-error MESSAGES_COUNT=500 COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose --project-directory ../docker -f ../docker/docker-compose.base.yaml -f ../docker/docker-compose.producer.yaml -f ../docker/docker-compose.worker.yaml -f ../docker/providers/azure-local/docker-compose.yaml up --remove-orphans --build --exit-code-from="worker" 5 | ./clean-azure.sh -------------------------------------------------------------------------------- /integration/tests/multi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ./clean-azure.sh 3 | docker-compose -f ../docker/docker-compose.base.yaml down --remove-orphans 4 | PRODUCER_SOURCE=sqs PIPES_CONFIG=multi MESSAGES_COUNT=500 COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose --project-directory ../docker -f ../docker/docker-compose.base.yaml -f ../docker/docker-compose.producer.yaml -f ../docker/docker-compose.worker.yaml -f ../docker/providers/azure-local/docker-compose.yaml -f ../docker/providers/sqs-local/docker-compose.yaml up --remove-orphans --build --exit-code-from="worker" 5 | ./clean-azure.sh -------------------------------------------------------------------------------- /integration/tests/naive-azure.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ./clean-azure.sh 3 | docker-compose -f ../docker/docker-compose.base.yaml down --remove-orphans 4 | MESSAGES_COUNT=500 COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose --project-directory ../docker -f ../docker/docker-compose.base.yaml -f ../docker/docker-compose.producer.yaml -f ../docker/docker-compose.worker.yaml -f ../docker/providers/azure-local/docker-compose.yaml up --remove-orphans --build --exit-code-from="worker" 5 | ./clean-azure.sh -------------------------------------------------------------------------------- /integration/tests/naive-sqs.sh: -------------------------------------------------------------------------------- 1 | docker-compose -f ../docker/docker-compose.base.yaml down --remove-orphans 2 | MESSAGES_COUNT=500 COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose --project-directory ../docker -f ../docker/docker-compose.base.yaml -f ../docker/docker-compose.producer.yaml -f ../docker/docker-compose.worker.yaml -f ../docker/providers/sqs-local/docker-compose.yaml up --remove-orphans --build --exit-code-from="worker" -------------------------------------------------------------------------------- /listeners/http.go: -------------------------------------------------------------------------------- 1 | package listeners 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | 9 | "github.com/julienschmidt/httprouter" 10 | "github.com/rs/zerolog/log" 11 | v1 "github.com/soluto/dqd/v1" 12 | "github.com/spf13/viper" 13 | ) 14 | 15 | var logger = log.With().Str("scope", "HttpListener").Logger() 16 | 17 | type HttpListener struct { 18 | address string 19 | router *httprouter.Router 20 | } 21 | 22 | func Http(address string) Listener { 23 | return &HttpListener{ 24 | address: address, 25 | router: httprouter.New(), 26 | } 27 | } 28 | 29 | func (h *HttpListener) Add(source *v1.Source, options *viper.Viper) { 30 | p := source.CreateProducer() 31 | logger.Info().Str("source", source.Name).Msg("adding source route") 32 | h.router.POST(fmt.Sprintf("/%v", source.Name), func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { 33 | msg, err := ioutil.ReadAll(r.Body) 34 | if err != nil { 35 | w.WriteHeader(500) 36 | return 37 | } 38 | err = p.Produce(r.Context(), &v1.RawMessage{ 39 | Data: string(msg), 40 | }) 41 | if err != nil { 42 | logger.Warn().Err(err).Msg("Error producing item") 43 | w.WriteHeader(500) 44 | return 45 | } 46 | }) 47 | } 48 | 49 | func (h *HttpListener) Listen(ctx context.Context) error { 50 | srv := &http.Server{Addr: h.address} 51 | e := make(chan error, 1) 52 | go func() { 53 | srv.Handler = h.router 54 | e <- srv.ListenAndServe() 55 | }() 56 | 57 | select { 58 | case err := <-e: 59 | return err 60 | case <-ctx.Done(): 61 | return nil 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /listeners/listener.go: -------------------------------------------------------------------------------- 1 | package listeners 2 | 3 | import ( 4 | "context" 5 | 6 | v1 "github.com/soluto/dqd/v1" 7 | "github.com/spf13/viper" 8 | ) 9 | 10 | type Dispose func() error 11 | 12 | type Listener interface { 13 | Add(source *v1.Source, options *viper.Viper) 14 | Listen(context context.Context) error 15 | } 16 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/rs/zerolog" 8 | "github.com/rs/zerolog/log" 9 | 10 | "github.com/soluto/dqd/api" 11 | "github.com/soluto/dqd/cmd" 12 | "github.com/soluto/dqd/config" 13 | "github.com/soluto/dqd/listeners" 14 | "github.com/soluto/dqd/pipe" 15 | "github.com/soluto/dqd/utils" 16 | v1 "github.com/soluto/dqd/v1" 17 | ) 18 | 19 | var logger = log.With().Str("scope", "Main").Logger() 20 | 21 | func GetHealthChecker(workers []*pipe.Worker) v1.HealthChecker { 22 | checkers := make(map[string]v1.HealthChecker) 23 | for _, w := range workers { 24 | checkers[w.Name] = w 25 | } 26 | return v1.CombineHealthCheckers(checkers) 27 | } 28 | 29 | func main() { 30 | conf, err := cmd.Load() 31 | if err != nil { 32 | cmd.ConfigurationError(err) 33 | } 34 | conf.SetDefault("logLevel", 1) 35 | conf.SetDefault("apiPort", 8888) 36 | logLevel := conf.GetInt("logLevel") 37 | zerolog.SetGlobalLevel(zerolog.Level(logLevel)) 38 | 39 | apiPort := conf.GetInt("metricsPort") 40 | if apiPort == 0 { 41 | apiPort = conf.GetInt("apiPort") 42 | } 43 | 44 | app, err := config.CreateApp(conf) 45 | if err != nil { 46 | cmd.ConfigurationError(err) 47 | } 48 | 49 | ctx := utils.ContextWithSignal(context.Background()) 50 | 51 | for _, worker := range app.Workers { 52 | go func(worker *pipe.Worker) { 53 | err := worker.Start(ctx) 54 | if err != nil { 55 | panic(err) 56 | } 57 | }(worker) 58 | } 59 | 60 | for _, listener := range app.Listeners { 61 | go func(listener listeners.Listener) { 62 | err := listener.Listen(ctx) 63 | if err != nil { 64 | panic(err) 65 | } 66 | }(listener) 67 | } 68 | 69 | if len(app.Workers) == 0 && len(app.Sources) == 0 { 70 | cmd.ConfigurationError(fmt.Errorf("no workers or sources are defiend")) 71 | } 72 | 73 | go api.Start(ctx, apiPort, GetHealthChecker(app.Workers)) 74 | 75 | select { 76 | case <-ctx.Done(): 77 | logger.Info().Msg("Shutting Down") 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/prometheus/client_golang/prometheus/promhttp" 10 | "github.com/rs/zerolog/log" 11 | ) 12 | 13 | var logger = log.With().Str("scope", "Metrics").Logger() 14 | 15 | var WorkerMaxConcurrencyGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 16 | Namespace: "worker", 17 | Subsystem: "concurrent", 18 | Name: "max", 19 | Help: "max concurrent messages", 20 | }, []string{"source"}) 21 | var WorkerBatchSizeGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 22 | Namespace: "worker", 23 | Subsystem: "concurrent", 24 | Name: "size", 25 | Help: "concurrent message handling", 26 | }, []string{"source"}) 27 | 28 | var HandlerProcessingHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ 29 | Namespace: "worker", 30 | Name: "handler_processing", 31 | Help: "handler processing time", 32 | }, []string{"pipe", "source", "success"}) 33 | 34 | var PipeProcessingMessagesHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ 35 | Namespace: "worker", 36 | Name: "pipe_processing", 37 | Help: "Pipe processing messages", 38 | }, []string{"pipe", "source", "success"}) 39 | 40 | func StartTimer(h *prometheus.HistogramVec) func(...string) { 41 | start := time.Now() 42 | return func(labels ...string) { 43 | total := time.Since(start) 44 | h.WithLabelValues(labels...).Observe(float64(total) / float64(time.Second)) 45 | } 46 | } 47 | 48 | func Start(metricsPort int) { 49 | prometheus.MustRegister( 50 | HandlerProcessingHistogram, 51 | PipeProcessingMessagesHistogram, 52 | WorkerBatchSizeGauge, 53 | WorkerMaxConcurrencyGauge, 54 | ) 55 | 56 | http.Handle("/metrics", promhttp.Handler()) 57 | logger.Info().Msgf("listening port for metrics: %v", metricsPort) 58 | err := http.ListenAndServe(fmt.Sprintf(":%v", metricsPort), nil) 59 | if err != nil { 60 | logger.Error().Err(err).Str("scope", "Metrics").Msg("Failed starting prometheus service") 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /pipe/type.go: -------------------------------------------------------------------------------- 1 | package pipe 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/rs/zerolog" 7 | "github.com/rs/zerolog/log" 8 | "github.com/soluto/dqd/handlers" 9 | "github.com/soluto/dqd/health" 10 | v1 "github.com/soluto/dqd/v1" 11 | ) 12 | 13 | type WorkerOption func(w *Worker) 14 | 15 | type Worker struct { 16 | Name string 17 | sources []*v1.Source 18 | output *v1.Source 19 | errorSource *v1.Source 20 | handler handlers.Handler 21 | logger *zerolog.Logger 22 | maxDequeueCount int64 23 | fixedRate bool 24 | dynamicRateBatchWindow time.Duration 25 | concurrencyStartingPoint int 26 | minConcurrency int 27 | writeToErrorSource bool 28 | probe *health.Probe 29 | } 30 | 31 | func WithDynamicRate(start, min int, windowSize time.Duration) WorkerOption { 32 | return WorkerOption(func(w *Worker) { 33 | w.fixedRate = false 34 | w.concurrencyStartingPoint = start 35 | w.minConcurrency = min 36 | w.dynamicRateBatchWindow = windowSize 37 | }) 38 | } 39 | 40 | func WithFixedRate(rate int) WorkerOption { 41 | return WorkerOption(func(w *Worker) { 42 | w.fixedRate = true 43 | w.concurrencyStartingPoint = rate 44 | }) 45 | } 46 | 47 | func WithErrorSource(source *v1.Source) WorkerOption { 48 | return WorkerOption(func(w *Worker) { 49 | w.writeToErrorSource = true 50 | w.errorSource = source 51 | }) 52 | } 53 | 54 | func WithOutput(source *v1.Source) WorkerOption { 55 | return WorkerOption(func(w *Worker) { 56 | w.output = source 57 | }) 58 | } 59 | 60 | func NewWorker(name string, sources []*v1.Source, handler handlers.Handler, opts ...WorkerOption) *Worker { 61 | l := log.With().Str("scope", "Worker").Str("pipe", name).Logger() 62 | w := &Worker{ 63 | Name: name, 64 | sources: sources, 65 | handler: handler, 66 | logger: &l, 67 | probe: health.MakeProbe(), 68 | } 69 | for _, o := range opts { 70 | o(w) 71 | } 72 | return w 73 | } 74 | -------------------------------------------------------------------------------- /pipe/worker.go: -------------------------------------------------------------------------------- 1 | package pipe 2 | 3 | import ( 4 | "context" 5 | "strconv" 6 | "sync/atomic" 7 | "time" 8 | 9 | "github.com/soluto/dqd/metrics" 10 | v1 "github.com/soluto/dqd/v1" 11 | ) 12 | 13 | func (w *Worker) handleErrorRequest(ctx *v1.RequestContext, err error, errProducer v1.Producer) { 14 | m := ctx.Message() 15 | w.logger.Warn().Err(err).Msg("Failed to handle messge") 16 | if !m.Abort(err) { 17 | if w.writeToErrorSource && errProducer != nil { 18 | err = errProducer.Produce(ctx, &v1.RawMessage{Data: m.Data()}) 19 | } 20 | if err != nil { 21 | w.logger.Error().Err(err).Msg("Failed to abort or recover message") 22 | } 23 | } 24 | } 25 | 26 | func (w *Worker) handleRequest(ctx *v1.RequestContext) (_ *v1.RawMessage, err error) { 27 | start := time.Now() 28 | defer func() { 29 | source := ctx.Source() 30 | t := float64(time.Since(start)) / float64(time.Second) 31 | metrics.HandlerProcessingHistogram.WithLabelValues(w.Name, source, strconv.FormatBool(err == nil)).Observe(t) 32 | }() 33 | return w.handler.Handle(ctx, ctx.Message()) 34 | } 35 | 36 | func (w *Worker) handleResults(ctx context.Context, results chan *v1.RequestContext) error { 37 | var outputP v1.Producer 38 | var errorP v1.Producer 39 | if w.output != nil { 40 | outputP = w.output.CreateProducer() 41 | } 42 | if w.errorSource != nil { 43 | errorP = w.errorSource.CreateProducer() 44 | } 45 | done := make(chan error) 46 | defer close(done) 47 | for reqCtx := range results { 48 | select { 49 | case err := <-done: 50 | return err 51 | case <-ctx.Done(): 52 | return nil 53 | default: 54 | } 55 | go func(reqCtx *v1.RequestContext) { 56 | m, err := reqCtx.Result() 57 | defer func() { 58 | defer func() { 59 | t := float64(time.Since(reqCtx.DequeueTime())) / float64(time.Second) 60 | metrics.PipeProcessingMessagesHistogram.WithLabelValues(w.Name, reqCtx.Source(), strconv.FormatBool(err == nil)).Observe(t) 61 | }() 62 | if err != nil { 63 | w.handleErrorRequest(reqCtx, err, errorP) 64 | } 65 | }() 66 | 67 | if err != nil { 68 | return 69 | } 70 | err = reqCtx.Complete() 71 | if err != nil { 72 | return 73 | } 74 | if m != nil && outputP != nil { 75 | err := outputP.Produce(reqCtx, m) 76 | if err != nil { 77 | return 78 | } 79 | } 80 | }(reqCtx) 81 | } 82 | return nil 83 | } 84 | 85 | func (w *Worker) HealthStatus() v1.HealthStatus { 86 | return w.probe.HealthStatus() 87 | } 88 | 89 | func (w *Worker) waitForHandlerToBeReady(ctx context.Context) { 90 | for { 91 | if ctx.Err() != nil { 92 | return 93 | } 94 | status := w.handler.HealthStatus() 95 | w.probe.UpdateStatus(status, "handler") 96 | if status.IsHealthy() { 97 | return 98 | } 99 | time.Sleep(time.Duration(time.Second)) 100 | } 101 | } 102 | 103 | func (w *Worker) readMessages(ctx context.Context, messages chan *v1.RequestContext, results chan *v1.RequestContext) error { 104 | maxConcurrencyGauge := metrics.WorkerMaxConcurrencyGauge.WithLabelValues(w.Name) 105 | batchSizeGauge := metrics.WorkerBatchSizeGauge.WithLabelValues(w.Name) 106 | 107 | var count, lastBatch int64 108 | maxItems := int64(w.concurrencyStartingPoint) 109 | minConcurrency := int64(w.minConcurrency) 110 | 111 | maxConcurrencyGauge.Set(float64(maxItems)) 112 | 113 | w.waitForHandlerToBeReady(ctx) 114 | if ctx.Err() == context.Canceled { 115 | return nil 116 | } 117 | 118 | //TODO #15 Consider replacing this code with a goroutine pool library 119 | go func() { 120 | for message := range messages { 121 | select { 122 | case <-ctx.Done(): 123 | return 124 | default: 125 | } 126 | for count >= maxItems { 127 | time.Sleep(10 * time.Millisecond) 128 | } 129 | 130 | atomic.AddInt64(&count, 1) 131 | 132 | go func(r *v1.RequestContext) { 133 | result, err := w.handleRequest(r) 134 | atomic.AddInt64(&count, -1) 135 | if !w.fixedRate { 136 | atomic.AddInt64(&lastBatch, 1) 137 | } 138 | select { 139 | case <-ctx.Done(): 140 | default: 141 | results <- r.WithResult(result, err) 142 | } 143 | 144 | }(message) 145 | } 146 | }() 147 | 148 | // Handle throughput 149 | if !w.fixedRate { 150 | go func() { 151 | var prev int64 152 | timer := time.NewTimer(w.dynamicRateBatchWindow) 153 | shouldUpscale := true 154 | w.logger.Debug().Int64("concurrency", maxItems).Msg("Using dynamic concurrency") 155 | for { 156 | timer.Reset(w.dynamicRateBatchWindow) 157 | 158 | select { 159 | case <-ctx.Done(): 160 | return 161 | case <-timer.C: 162 | } 163 | 164 | curr := atomic.SwapInt64(&lastBatch, 0) 165 | batchSizeGauge.Set(float64(curr)) 166 | 167 | if curr == 0 { 168 | continue 169 | } 170 | if curr < prev { 171 | shouldUpscale = !shouldUpscale 172 | } 173 | if shouldUpscale { 174 | atomic.AddInt64(&maxItems, 1) 175 | } else if maxItems > minConcurrency { 176 | atomic.AddInt64(&maxItems, -1) 177 | } 178 | maxConcurrencyGauge.Set(float64(maxItems)) 179 | 180 | prev = curr 181 | w.logger.Debug().Int64("concurrency", maxItems).Float64("rate", float64(curr)/w.dynamicRateBatchWindow.Seconds()).Msg("tuning concurrency") 182 | } 183 | }() 184 | } 185 | done := make(chan error) 186 | defer close(done) 187 | 188 | for _, s := range w.sources { 189 | go func(ss *v1.Source) { 190 | w.logger.Info().Str("source", ss.Name).Msg("Start reading from source") 191 | consumer := ss.CreateConsumer() 192 | 193 | err := consumer.Iter(ctx, v1.NextMessage(func(m v1.Message) { 194 | select { 195 | case <-ctx.Done(): 196 | default: 197 | messages <- v1.CreateRequestContext(ctx, ss.Name, m) 198 | } 199 | })) 200 | select { 201 | case <-ctx.Done(): 202 | default: 203 | done <- err 204 | } 205 | }(s) 206 | } 207 | select { 208 | case err := <-done: 209 | return err 210 | case <-ctx.Done(): 211 | return nil 212 | } 213 | } 214 | 215 | func (w *Worker) Start(ctx context.Context) error { 216 | w.logger.Info().Msg("Starting pipe") 217 | messages := make(chan *v1.RequestContext, w.minConcurrency) 218 | defer close(messages) 219 | results := make(chan *v1.RequestContext, w.minConcurrency) 220 | defer close(results) 221 | done := make(chan error) 222 | 223 | innerContext, cancel := context.WithCancel(ctx) 224 | defer cancel() 225 | 226 | go func() { 227 | err := w.readMessages(innerContext, messages, results) 228 | select { 229 | case <-ctx.Done(): 230 | default: 231 | done <- err 232 | } 233 | }() 234 | 235 | go func() { 236 | err := w.handleResults(innerContext, results) 237 | select { 238 | case <-ctx.Done(): 239 | default: 240 | done <- err 241 | } 242 | }() 243 | 244 | select { 245 | 246 | case <-ctx.Done(): 247 | return nil 248 | case err := <-done: 249 | return err 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /providers/azure/azure.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/Azure/azure-pipeline-go/pipeline" 9 | "github.com/Azure/azure-storage-queue-go/azqueue" 10 | "github.com/jpillora/backoff" 11 | "github.com/rs/zerolog" 12 | 13 | "context" 14 | "net/url" 15 | 16 | v1 "github.com/soluto/dqd/v1" 17 | "github.com/spf13/viper" 18 | ) 19 | 20 | const ( 21 | serverTimeout = 5 22 | ) 23 | 24 | // Message represents a message in a queue. 25 | type AzureMessage struct { 26 | *azqueue.DequeuedMessage 27 | azureClient *azureClient 28 | } 29 | 30 | type azureClient struct { 31 | messagesURL azqueue.MessagesURL 32 | MaxDequeueCount int64 33 | visibilityTimeout time.Duration 34 | logger *zerolog.Logger 35 | } 36 | 37 | func (m *AzureMessage) Data() string { 38 | decoded, err := base64.StdEncoding.DecodeString(m.Text) 39 | if err == nil { 40 | return string(decoded) 41 | } 42 | return m.Text 43 | } 44 | 45 | func (m *AzureMessage) Complete() error { 46 | res, err := m.azureClient.messagesURL.NewMessageIDURL(azqueue.MessageID(m.Id())).Delete(context.Background(), azqueue.PopReceipt(m.PopReceipt)) 47 | if err != nil { 48 | return err 49 | } 50 | if res.StatusCode() >= 400 { 51 | return fmt.Errorf("error deleting message") 52 | } 53 | return nil 54 | } 55 | 56 | func (m *AzureMessage) Id() string { 57 | return m.ID.String() 58 | } 59 | 60 | func (m *AzureMessage) Abort(error) bool { 61 | return m.DequeueCount < m.azureClient.MaxDequeueCount 62 | } 63 | 64 | type ClientOptions struct { 65 | StorageAccount, SasToken, Name string 66 | ServerTimeoutInSeconds int 67 | VisibilityTimeoutInSeconds int64 68 | } 69 | 70 | func (c *azureClient) Produce(context context.Context, m *v1.RawMessage) error { 71 | _, err := c.messagesURL.Enqueue(context, m.Data, time.Duration(0), time.Duration(0)) 72 | return err 73 | } 74 | 75 | func (c *azureClient) Iter(ctx context.Context, next v1.NextMessage) error { 76 | backoff := &backoff.Backoff{} 77 | 78 | Main: 79 | for { 80 | select { 81 | case <-ctx.Done(): 82 | break Main 83 | default: 84 | } 85 | 86 | messages, err := c.messagesURL.Dequeue(context.Background(), 32, c.visibilityTimeout) 87 | if err != nil { 88 | return err 89 | } 90 | messagesCount := messages.NumMessages() 91 | 92 | if messagesCount == 0 { 93 | c.logger.Debug().Msg("Reached empty queue") 94 | time.Sleep(backoff.Duration()) 95 | continue Main 96 | } 97 | backoff.Reset() 98 | 99 | for i := int32(0); i < messages.NumMessages(); i++ { 100 | select { 101 | case <-ctx.Done(): 102 | break Main 103 | default: 104 | } 105 | azM := messages.Message(i) 106 | message := &AzureMessage{ 107 | azM, 108 | c, 109 | } 110 | next(message) 111 | } 112 | } 113 | return nil 114 | } 115 | 116 | func translateLogLevel(l pipeline.LogLevel) zerolog.Level { 117 | switch l { 118 | case pipeline.LogDebug: 119 | return zerolog.DebugLevel 120 | case pipeline.LogError: 121 | return zerolog.ErrorLevel 122 | case pipeline.LogFatal: 123 | return zerolog.FatalLevel 124 | case pipeline.LogInfo: 125 | return zerolog.InfoLevel 126 | case pipeline.LogNone: 127 | return zerolog.Disabled 128 | case pipeline.LogPanic: 129 | return zerolog.PanicLevel 130 | case pipeline.LogWarning: 131 | return zerolog.WarnLevel 132 | } 133 | return zerolog.Disabled 134 | } 135 | 136 | func createAuzreQueueClient(cfg *viper.Viper, logger *zerolog.Logger) *azureClient { 137 | cfg.SetDefault("visibilityTimeoutInSeconds", 60) 138 | cfg.SetDefault("maxDequeueCount", 5) 139 | 140 | storageAccount := cfg.GetString("storageAccount") 141 | queueName := cfg.GetString("queue") 142 | sasToken := cfg.GetString("sasToken") 143 | accountKey := cfg.GetString("storageAccountKey") 144 | visibilityTimeout := time.Duration(cfg.GetInt64("visibilityTimeoutInSeconds")) * time.Second 145 | 146 | credentials := azqueue.NewAnonymousCredential() 147 | if accountKey != "" && storageAccount != "" { 148 | var err error 149 | credentials, err = azqueue.NewSharedKeyCredential(storageAccount, accountKey) 150 | if err != nil { 151 | logger.Fatal().Msg("Error using SharedKeyCredentials") 152 | panic("Error using SharedKeyCredentials") 153 | } 154 | } 155 | 156 | pipeline := azqueue.NewPipeline(credentials, azqueue.PipelineOptions{ 157 | Log: pipeline.LogOptions{ 158 | Log: func(level pipeline.LogLevel, message string) { 159 | logger.WithLevel(translateLogLevel(level)).Msg(message) 160 | }, 161 | }, 162 | Retry: azqueue.RetryOptions{ 163 | Policy: azqueue.RetryPolicyExponential, 164 | MaxTries: 4, 165 | TryTimeout: 30 * time.Second, 166 | }, 167 | }) 168 | 169 | var sURL string 170 | if storageAccount != "" { 171 | sURL = fmt.Sprintf("https://%s.queue.core.windows.net", storageAccount) 172 | } else { 173 | sURL = cfg.GetString("connection") 174 | } 175 | sURL = fmt.Sprintf("%s%s", sURL, sasToken) 176 | u, _ := url.Parse(sURL) 177 | 178 | messagesURL := azqueue.NewServiceURL(*u, pipeline).NewQueueURL(queueName).NewMessagesURL() 179 | 180 | return &azureClient{ 181 | messagesURL: messagesURL, 182 | MaxDequeueCount: cfg.GetInt64("maxDequeueCount"), 183 | visibilityTimeout: visibilityTimeout, 184 | logger: logger, 185 | } 186 | } 187 | 188 | type AzureQueueClientFactory struct { 189 | } 190 | 191 | func (factory *AzureQueueClientFactory) CreateConsumer(cfg *viper.Viper, logger *zerolog.Logger) v1.Consumer { 192 | return createAuzreQueueClient(cfg, logger) 193 | } 194 | 195 | func (factory *AzureQueueClientFactory) CreateProducer(cfg *viper.Viper, logger *zerolog.Logger) v1.Producer { 196 | return createAuzreQueueClient(cfg, logger) 197 | } 198 | 199 | func (c *azureClient) HealthStatus() v1.HealthStatus { 200 | return v1.NewHealthStatus(v1.Healthy) 201 | } 202 | -------------------------------------------------------------------------------- /providers/servicebus/servicebus.go: -------------------------------------------------------------------------------- 1 | package servicebus 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | azservicebus "github.com/Azure/azure-service-bus-go" 8 | "github.com/rs/zerolog" 9 | v1 "github.com/soluto/dqd/v1" 10 | "github.com/spf13/viper" 11 | ) 12 | 13 | type ServiceBusClient struct { 14 | topic *azservicebus.Topic 15 | subscription *azservicebus.Subscription 16 | logger zerolog.Logger 17 | preFetchCount int 18 | removeSerializationInfo bool 19 | } 20 | 21 | type ServiceBusMessage struct { 22 | message *azservicebus.Message 23 | removeSerializationInfo bool 24 | } 25 | 26 | func createServiceBusClient(cfg *viper.Viper, logger *zerolog.Logger) *ServiceBusClient { 27 | cfg.SetDefault("prefetchCount", 1) 28 | namespace, err := azservicebus.NewNamespace(azservicebus.NamespaceWithConnectionString(cfg.GetString("connectionString"))) 29 | topicName := cfg.GetString("topic") 30 | subscriptionName := cfg.GetString("subscription") 31 | topic, err := namespace.NewTopic(topicName) 32 | if err != nil { 33 | panic("failed to initalize service bus client") 34 | } 35 | l := logger.With().Str("topic", topicName).Str("subscription", subscriptionName).Logger() 36 | subscription, err := topic.NewSubscription(subscriptionName) 37 | return &ServiceBusClient{ 38 | topic, 39 | subscription, 40 | l, 41 | cfg.GetInt("prefetchCount"), 42 | cfg.GetBool("removeSerializationInfoInJson"), 43 | } 44 | } 45 | 46 | func (m *ServiceBusMessage) Id() string { 47 | return m.message.ID 48 | } 49 | 50 | func (m *ServiceBusMessage) Data() string { 51 | data := string(m.message.Data) 52 | if m.removeSerializationInfo { 53 | return strings.TrimRightFunc(strings.TrimLeftFunc(data, func(r rune) bool { return r != '{' && r != '[' }), func(r rune) bool { return r != '}' && r != ']' }) 54 | } 55 | return data 56 | } 57 | 58 | func (m *ServiceBusMessage) Complete() error { 59 | return m.message.Complete(context.Background()) 60 | } 61 | 62 | func (m *ServiceBusMessage) Abort(error) bool { 63 | m.message.Abandon(context.Background()) 64 | return true 65 | } 66 | 67 | func (sb *ServiceBusClient) Iter(ctx context.Context, next v1.NextMessage) error { 68 | rec, err := sb.subscription.NewReceiver(ctx, azservicebus.ReceiverWithReceiveMode(azservicebus.PeekLockMode), azservicebus.ReceiverWithPrefetchCount(10)) 69 | if err != nil { 70 | return err 71 | } 72 | for { 73 | select { 74 | case <-ctx.Done(): 75 | break 76 | default: 77 | 78 | } 79 | 80 | err = rec.ReceiveOne(ctx, azservicebus.HandlerFunc(func(ctx context.Context, m *azservicebus.Message) error { 81 | message := &ServiceBusMessage{ 82 | m, 83 | sb.removeSerializationInfo, 84 | } 85 | next(message) 86 | return nil 87 | })) 88 | if err != nil { 89 | return err 90 | } 91 | } 92 | } 93 | 94 | func (c *ServiceBusClient) Produce(ctx context.Context, m *v1.RawMessage) error { 95 | return c.topic.Send(ctx, azservicebus.NewMessageFromString(m.Data)) 96 | } 97 | 98 | type ServiceBusClientFactory struct { 99 | } 100 | 101 | func (factory *ServiceBusClientFactory) CreateConsumer(cfg *viper.Viper, logger *zerolog.Logger) v1.Consumer { 102 | return createServiceBusClient(cfg, logger) 103 | } 104 | 105 | func (factory *ServiceBusClientFactory) CreateProducer(cfg *viper.Viper, logger *zerolog.Logger) v1.Producer { 106 | return createServiceBusClient(cfg, logger) 107 | } 108 | 109 | func (sb *ServiceBusClient) HealthStatus() v1.HealthStatus { 110 | return v1.NewHealthStatus(v1.Healthy) 111 | } 112 | -------------------------------------------------------------------------------- /providers/sqs/sqs.go: -------------------------------------------------------------------------------- 1 | package sqs 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "time" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/aws/aws-sdk-go/service/sqs" 11 | "github.com/jpillora/backoff" 12 | "github.com/rs/zerolog" 13 | v1 "github.com/soluto/dqd/v1" 14 | "github.com/spf13/viper" 15 | ) 16 | 17 | type SQSClient struct { 18 | sqs sqs.SQS 19 | url string 20 | visibilityTimeoutInSeconds int64 21 | maxNumberOfMessages int64 22 | unwrapSnsMessage bool 23 | logger *zerolog.Logger 24 | } 25 | 26 | type SQSMessage struct { 27 | *sqs.Message 28 | client *SQSClient 29 | } 30 | 31 | type SnsMessage struct { 32 | Message string `json:"Message"` 33 | } 34 | 35 | func createSQSClient(cfg *viper.Viper, logger *zerolog.Logger) *SQSClient { 36 | cfg.SetDefault("visibilityTimeoutInSeconds", 600) 37 | cfg.SetDefault("maxNumberOfMessages", 10) 38 | cfg.SetDefault("unwrapSnsMessage", false) 39 | 40 | awsConfig := aws.NewConfig().WithRegion(cfg.GetString("region")) 41 | 42 | endpoint := cfg.GetString("endpoint") 43 | if endpoint != "" { 44 | awsConfig.Endpoint = &endpoint 45 | } 46 | svc := sqs.New(session.New(), awsConfig) 47 | return &SQSClient{ 48 | *svc, 49 | cfg.GetString("url"), 50 | cfg.GetInt64("visibilityTimeoutInSeconds"), 51 | cfg.GetInt64("maxNumberOfMessages"), 52 | cfg.GetBool("unwrapSnsMessage"), 53 | logger, 54 | } 55 | } 56 | 57 | func (m *SQSMessage) Id() string { 58 | return *m.MessageId 59 | } 60 | 61 | func (m *SQSMessage) Data() string { 62 | if !m.client.unwrapSnsMessage { 63 | return *m.Body 64 | } 65 | 66 | var snsMessage SnsMessage 67 | err := json.Unmarshal([]byte(*m.Body), &snsMessage) 68 | 69 | if err != nil { 70 | m.client.logger.Warn().Err(err).Str("Body", *m.Body).Msg("Failed deserializing SNS style message, sending along original message instead") 71 | return *m.Body 72 | } 73 | 74 | return snsMessage.Message 75 | } 76 | 77 | func (m *SQSMessage) Complete() error { 78 | _, err := m.client.sqs.DeleteMessage(&sqs.DeleteMessageInput{ 79 | QueueUrl: &m.client.url, 80 | ReceiptHandle: m.ReceiptHandle, 81 | }) 82 | return err 83 | } 84 | 85 | func (m *SQSMessage) Abort(error) bool { 86 | return true 87 | } 88 | 89 | func (c *SQSClient) Iter(ctx context.Context, next v1.NextMessage) error { 90 | errorBackoff := &backoff.Backoff{} 91 | emptyBackoff := &backoff.Backoff{} 92 | Main: 93 | for { 94 | select { 95 | case <-ctx.Done(): 96 | break Main 97 | default: 98 | } 99 | messages, err := c.sqs.ReceiveMessage(&sqs.ReceiveMessageInput{ 100 | QueueUrl: &c.url, 101 | MaxNumberOfMessages: &c.maxNumberOfMessages, 102 | VisibilityTimeout: &c.visibilityTimeoutInSeconds, 103 | }) 104 | 105 | if err != nil { 106 | c.logger.Debug().Err(err).Msg("Error reading from queue") 107 | time.Sleep(errorBackoff.Duration()) 108 | if errorBackoff.Attempt() >= 10 { 109 | return err 110 | } 111 | continue Main 112 | } 113 | errorBackoff.Reset() 114 | 115 | if len(messages.Messages) == 0 { 116 | c.logger.Debug().Msg("Reached empty queue") 117 | time.Sleep(emptyBackoff.Duration()) 118 | continue Main 119 | } 120 | emptyBackoff.Reset() 121 | 122 | for _, sqsM := range messages.Messages { 123 | select { 124 | case <-ctx.Done(): 125 | break Main 126 | default: 127 | } 128 | message := &SQSMessage{ 129 | sqsM, 130 | c, 131 | } 132 | next(message) 133 | } 134 | } 135 | return nil 136 | } 137 | 138 | func (c *SQSClient) Produce(context context.Context, m *v1.RawMessage) error { 139 | backoff := &backoff.Backoff{ 140 | Max: 10 * time.Second, 141 | Min: 100 * time.Millisecond, 142 | } 143 | act := func() error { 144 | _, err := c.sqs.SendMessage(&sqs.SendMessageInput{ 145 | MessageBody: &m.Data, 146 | QueueUrl: &c.url, 147 | }) 148 | return err 149 | } 150 | err := act() 151 | for err != nil { 152 | err = act() 153 | if backoff.Attempt() > 4 { 154 | return err 155 | } 156 | time.Sleep(backoff.Duration()) 157 | } 158 | return err 159 | } 160 | 161 | type SQSClientFactory struct { 162 | } 163 | 164 | func (factory *SQSClientFactory) CreateConsumer(cfg *viper.Viper, logger *zerolog.Logger) v1.Consumer { 165 | return createSQSClient(cfg, logger) 166 | } 167 | 168 | func (factory *SQSClientFactory) CreateProducer(cfg *viper.Viper, logger *zerolog.Logger) v1.Producer { 169 | return createSQSClient(cfg, logger) 170 | } 171 | 172 | func (h *SQSClient) HealthStatus() v1.HealthStatus { 173 | return v1.NewHealthStatus(v1.Healthy) 174 | } 175 | -------------------------------------------------------------------------------- /utils/context.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | ) 9 | 10 | func ContextWithSignal(ctx context.Context) context.Context { 11 | newCtx, cancel := context.WithCancel(ctx) 12 | signals := make(chan os.Signal) 13 | signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) 14 | go func() { 15 | select { 16 | case <-signals: 17 | cancel() 18 | } 19 | }() 20 | return newCtx 21 | } 22 | -------------------------------------------------------------------------------- /utils/env.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "strconv" 7 | 8 | "github.com/rs/zerolog/log" 9 | ) 10 | 11 | var logger = log.With().Str("scope", "Main").Logger() 12 | 13 | func GetenvInt(s string, defaultValue int64) int64 { 14 | value := os.Getenv(s) 15 | i, err := strconv.ParseInt(value, 10, 64) 16 | if err != nil { 17 | return defaultValue 18 | } 19 | return i 20 | } 21 | 22 | func GetenvRequired(s string) string { 23 | value := os.Getenv(s) 24 | if value == "" { 25 | logger.Fatal().Msg("Env var " + s + " is required") 26 | } 27 | return value 28 | } 29 | 30 | func GetenvOrFile(varName, fileVarName string, required bool) string { 31 | value := os.Getenv(varName) 32 | if value != "" { 33 | return value 34 | } 35 | fileName := os.Getenv(fileVarName) 36 | dat, err := ioutil.ReadFile(fileName) 37 | if err == nil { 38 | return string(dat) 39 | } 40 | if required { 41 | logger.Fatal().Msg("Env var " + varName + " or " + fileVarName + " is required") 42 | } 43 | return "" 44 | } 45 | -------------------------------------------------------------------------------- /utils/sources.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/rs/zerolog" 8 | v1 "github.com/soluto/dqd/v1" 9 | "github.com/spf13/viper" 10 | ) 11 | 12 | type ioClient struct { 13 | file *os.File 14 | } 15 | 16 | type IoSourceFactory struct { 17 | } 18 | 19 | func (c *ioClient) HealthStatus() v1.HealthStatus { 20 | return v1.NewHealthStatus(v1.Healthy) 21 | } 22 | 23 | func (c *ioClient) Produce(context context.Context, m *v1.RawMessage) error { 24 | _, err := c.file.WriteString(m.Data) 25 | return err 26 | } 27 | 28 | func (*IoSourceFactory) CreateConsumer(config *viper.Viper, logger *zerolog.Logger) v1.Consumer { 29 | return nil 30 | } 31 | 32 | func (*IoSourceFactory) CreateProducer(config *viper.Viper, logger *zerolog.Logger) v1.Producer { 33 | s := config.GetString("file") 34 | var file *os.File 35 | if s == "" { 36 | file = os.Stdout 37 | } else { 38 | file, _ = os.Open(s) 39 | } 40 | 41 | return &ioClient{ 42 | file, 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /utils/viper.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | 7 | "github.com/spf13/cast" 8 | "github.com/spf13/viper" 9 | ) 10 | 11 | func NormalizeEntityConfig(v *viper.Viper, singular string, plural string) error { 12 | one := v.Get(singular) 13 | multiple := v.Get(plural) 14 | if one != nil { 15 | if multiple != nil { 16 | return fmt.Errorf("failed to normalize %v,%v multiple definitions", singular, plural) 17 | } 18 | v.Set(plural, map[string]interface{}{ 19 | "default": one, 20 | }) 21 | v.Set(singular, nil) 22 | } else if multiple != nil { 23 | if reflect.TypeOf(multiple).Kind() == reflect.Slice { 24 | newData := map[string]interface{}{} 25 | for i, item := range multiple.([]interface{}) { 26 | newData[fmt.Sprintf("%v%v", plural, i)] = item 27 | } 28 | v.Set(plural, newData) 29 | } 30 | } 31 | return nil 32 | 33 | } 34 | 35 | func ViperSubSlice(v *viper.Viper, key string) []*viper.Viper { 36 | data := v.Get(key) 37 | if data == nil { 38 | return nil 39 | } 40 | switch reflect.TypeOf(data).Kind() { 41 | case reflect.Slice: 42 | var vList []*viper.Viper 43 | for _, item := range data.([]interface{}) { 44 | sub := viper.New() 45 | sub.MergeConfigMap(cast.ToStringMap(item)) 46 | vList = append(vList, sub) 47 | } 48 | return vList 49 | } 50 | return nil 51 | 52 | } 53 | 54 | func ViperSubMap(v *viper.Viper, key string) map[string]*viper.Viper { 55 | data := v.Get(key) 56 | if data == nil { 57 | return nil 58 | } 59 | 60 | if reflect.TypeOf(data).Kind() == reflect.Map { 61 | subMap := map[string]*viper.Viper{} 62 | for key, item := range data.(map[string]interface{}) { 63 | sub := viper.New() 64 | sub.MergeConfigMap(cast.ToStringMap(item)) 65 | subMap[key] = sub 66 | } 67 | return subMap 68 | } 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /v1/common.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/rs/zerolog" 7 | "github.com/rs/zerolog/log" 8 | "github.com/spf13/viper" 9 | ) 10 | 11 | type NextMessage func(Message) 12 | 13 | type RawMessage struct { 14 | Data string 15 | } 16 | 17 | type Message interface { 18 | Id() string 19 | Data() string 20 | Complete() error 21 | Abort(error) bool 22 | } 23 | 24 | type Consumer interface { 25 | HealthChecker 26 | Iter(ctx context.Context, next NextMessage) error 27 | } 28 | 29 | type ConsumerFactory interface { 30 | CreateConsumer(config *viper.Viper, logger *zerolog.Logger) Consumer 31 | } 32 | 33 | type Producer interface { 34 | HealthChecker 35 | Produce(context context.Context, m *RawMessage) error 36 | } 37 | 38 | type ProducerFactory interface { 39 | CreateProducer(config *viper.Viper, logger *zerolog.Logger) Producer 40 | } 41 | 42 | type Source struct { 43 | consumerFactory ConsumerFactory 44 | producerFactory ProducerFactory 45 | config *viper.Viper 46 | Name string 47 | } 48 | 49 | func NewSource(cf ConsumerFactory, pf ProducerFactory, config *viper.Viper, name string) *Source { 50 | return &Source{ 51 | cf, 52 | pf, 53 | config, 54 | name, 55 | } 56 | } 57 | 58 | func (s Source) CreateConsumer() Consumer { 59 | l := log.With().Fields(map[string]interface{}{ 60 | "scope": "Consumer", 61 | "source": s.Name, 62 | }).Logger() 63 | return s.consumerFactory.CreateConsumer(s.config, &l) 64 | } 65 | 66 | func (s Source) CreateProducer() Producer { 67 | l := log.With().Fields(map[string]interface{}{ 68 | "scope": "Consumer", 69 | "source": s.Name, 70 | }).Logger() 71 | return s.producerFactory.CreateProducer(s.config, &l) 72 | } 73 | -------------------------------------------------------------------------------- /v1/health.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type healthStatusValue string 8 | 9 | const ( 10 | Healthy = healthStatusValue("Healthy") 11 | Init = healthStatusValue("Init") 12 | ) 13 | 14 | func Error(err error) healthStatusValue { 15 | return healthStatusValue(fmt.Sprintf("Error - %v", err.Error())) 16 | } 17 | 18 | type HealthStatus map[string]healthStatusValue 19 | 20 | type HealthChecker interface { 21 | HealthStatus() HealthStatus 22 | } 23 | 24 | func (h HealthStatus) IsHealthy() bool { 25 | for _, v := range h { 26 | if v != Healthy { 27 | return false 28 | } 29 | } 30 | return true 31 | } 32 | 33 | func (h HealthStatus) Add(h2 HealthStatus, prefix string) HealthStatus { 34 | for k, v := range h2 { 35 | h[fmt.Sprintf("%v.%v", prefix, k)] = v 36 | } 37 | return h 38 | } 39 | 40 | type CompositeHealthChecker struct { 41 | checkers map[string]HealthChecker 42 | } 43 | 44 | func (c *CompositeHealthChecker) HealthStatus() HealthStatus { 45 | s := HealthStatus{} 46 | for k, c := range c.checkers { 47 | s.Add(c.HealthStatus(), k) 48 | } 49 | return s 50 | } 51 | 52 | func CombineHealthCheckers(checkers map[string]HealthChecker) HealthChecker { 53 | return &CompositeHealthChecker{ 54 | checkers, 55 | } 56 | } 57 | 58 | func NewHealthStatus(status healthStatusValue) HealthStatus { 59 | return HealthStatus{ 60 | "": status, 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /v1/request-context.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | type contextKey string 9 | 10 | func (c contextKey) String() string { 11 | return string(c) 12 | } 13 | 14 | var ( 15 | ContextKeyMessage = contextKey("message") 16 | ContextKeyResult = contextKey("result") 17 | ContextKeySource = contextKey("source") 18 | ContextKeyStart = contextKey("start") 19 | ) 20 | 21 | type RequestContext struct { 22 | context.Context 23 | } 24 | 25 | func CreateRequestContext(ctx context.Context, source string, m Message) *RequestContext { 26 | return &RequestContext{ 27 | context.WithValue( 28 | context.WithValue( 29 | context.WithValue(ctx, 30 | ContextKeySource, source), 31 | ContextKeyMessage, m), 32 | ContextKeyStart, time.Now()), 33 | } 34 | } 35 | 36 | func (r *RequestContext) Abort(e error) bool { 37 | return r.Message().Abort(e) 38 | } 39 | 40 | func (r *RequestContext) Complete() error { 41 | return r.Message().Complete() 42 | } 43 | 44 | func (r *RequestContext) Message() Message { 45 | m, _ := r.Value(ContextKeyMessage).(Message) 46 | return m 47 | } 48 | 49 | func (r *RequestContext) Source() string { 50 | s, _ := r.Value(ContextKeySource).(string) 51 | return s 52 | } 53 | 54 | func (r *RequestContext) DequeueTime() time.Time { 55 | s, _ := r.Value(ContextKeyStart).(time.Time) 56 | return s 57 | } 58 | 59 | func (r *RequestContext) WithResult(m *RawMessage, err error) *RequestContext { 60 | if err == nil { 61 | return &RequestContext{context.WithValue(r.Context, ContextKeyResult, m)} 62 | } else { 63 | return &RequestContext{context.WithValue(r.Context, ContextKeyResult, err)} 64 | } 65 | } 66 | 67 | func (r *RequestContext) Result() (m *RawMessage, err error) { 68 | result := r.Value(ContextKeyResult) 69 | if result == nil { 70 | return nil, nil 71 | } 72 | if m, ok := result.(*RawMessage); ok { 73 | return m, nil 74 | } 75 | if err, ok := result.(error); ok { 76 | return nil, err 77 | } 78 | return nil, nil 79 | } 80 | --------------------------------------------------------------------------------