├── .VERSION ├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .github └── workflows │ ├── deploy-docker.yml │ ├── deploy-schemas.yml │ ├── lint.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── .vscode └── settings.json ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── buz │ ├── app.go │ └── main.go └── healthcheck │ └── main.go ├── deploy ├── Dockerfile └── terraform │ ├── aws │ └── lambda │ │ ├── Dockerfile.tftpl │ │ ├── README.md │ │ ├── config.yml.tftpl │ │ ├── data.tf │ │ ├── iam.tf │ │ ├── locals.tf │ │ ├── main.tf │ │ ├── outputs.tf │ │ ├── provider.tf │ │ └── variables.tf │ └── gcp │ ├── .terraform-version │ └── cloud_run │ ├── README.md │ ├── config.yml.tftpl │ ├── locals.tf │ ├── main.tf │ ├── outputs.tf │ ├── provider.tf │ ├── schema.json │ └── variables.tf ├── examples ├── cli │ └── curl.sh ├── devel │ └── buz │ │ └── simple.conf.yml └── quickstart │ ├── buz │ └── quickstart.conf.yml │ ├── docker-compose.yml │ └── nginx │ └── nginx.conf ├── go.mod ├── go.sum ├── img └── buzz.png ├── pkg ├── annotator │ ├── annotator.go │ └── annotator_test.go ├── backend │ ├── azuredw │ │ └── sink.go │ ├── backendutils │ │ └── sink.go │ ├── bigquery │ │ └── sink.go │ ├── blackhole │ │ └── sink.go │ ├── clickhousedb │ │ ├── dsn.go │ │ ├── dsn_test.go │ │ ├── registry.go │ │ └── sink.go │ ├── elasticsearch │ │ └── sink.go │ ├── eventbridge │ │ └── sink.go │ ├── file │ │ ├── registry.go │ │ └── sink.go │ ├── firebolt │ │ └── sink.go │ ├── gcs │ │ └── registry.go │ ├── http │ │ ├── registry.go │ │ └── sink.go │ ├── kafka │ │ ├── registry.go │ │ └── sink.go │ ├── kinesis │ │ └── sink.go │ ├── kinesisFirehose │ │ └── sink.go │ ├── minio │ │ └── registry.go │ ├── mongodb │ │ ├── registry.go │ │ └── sink.go │ ├── mysqldb │ │ ├── dsn.go │ │ ├── dsn_test.go │ │ ├── registry.go │ │ └── sink.go │ ├── nats │ │ └── sink.go │ ├── natsJetstream │ │ └── sink.go │ ├── postgresdb │ │ ├── dsn.go │ │ ├── dsn_test.go │ │ ├── registry.go │ │ └── sink.go │ ├── pubnub │ │ └── sink.go │ ├── pubsub │ │ └── sink.go │ ├── pulsar │ │ └── sink.go │ ├── rabbitmq │ │ └── sink.go │ ├── redshift │ │ └── sink.go │ ├── s3 │ │ └── registry.go │ ├── splunk │ │ └── sink.go │ └── stdout │ │ ├── sink.go │ │ └── sink_test.go ├── config │ ├── app.go │ ├── app_test.go │ ├── cloudevents.go │ ├── cloudevents_test.go │ ├── config.go │ ├── config_test.go │ ├── generic_test.go │ ├── input.go │ ├── manifold.go │ ├── middleware.go │ ├── middleware_test.go │ ├── pixel.go │ ├── registry.go │ ├── registry_test.go │ ├── relay.go │ ├── selfDescribing.go │ ├── sink.go │ ├── sink_test.go │ ├── snowplow.go │ ├── snowplow_test.go │ ├── squawkbox.go │ ├── tele.go │ ├── tele_test.go │ ├── webhook.go │ └── webhook_test.go ├── constants │ ├── backend.go │ ├── buz.go │ ├── buz_test.go │ ├── context.go │ ├── context_test.go │ ├── routes.go │ └── routes_test.go ├── db │ ├── db.go │ ├── model.go │ └── util.go ├── env │ ├── env.go │ └── env_test.go ├── envelope │ ├── annotation.go │ ├── context.go │ ├── envelope.go │ ├── envelope_test.go │ ├── event.go │ ├── event_test.go │ ├── output.go │ ├── output_test.go │ ├── selfDescribing.go │ ├── selfDescribing_test.go │ └── validation.go ├── handler │ ├── buz.go │ ├── buz_test.go │ ├── health.go │ ├── health_test.go │ ├── overview.go │ ├── overview_test.go │ ├── stats.go │ └── stats_test.go ├── input │ └── input.go ├── inputApp │ └── inputApp.go ├── manifold │ ├── channelManifold.go │ ├── kernelBufferManifold.go │ ├── manifold.go │ └── simpleManifold.go ├── meta │ └── collector.go ├── middleware │ ├── auth.go │ ├── auth_test.go │ ├── cors.go │ ├── cors_test.go │ ├── identity.go │ ├── identity_test.go │ ├── jsonLogger.go │ ├── jsonLogger_test.go │ ├── rateLimiter.go │ ├── rateLimiter_test.go │ ├── timeout.go │ └── timeout_test.go ├── params │ └── handler.go ├── protocol │ ├── cloudevents │ │ ├── envelopeBuilder.go │ │ ├── event.go │ │ ├── eventBuilder.go │ │ └── input.go │ ├── pixel │ │ ├── envelopeBuilder.go │ │ ├── eventBuilder.go │ │ └── input.go │ ├── protocol.go │ ├── protocol_test.go │ ├── selfdescribing │ │ ├── envelopeBuilder.go │ │ ├── eventBuilder.go │ │ └── input.go │ ├── snowplow │ │ ├── envelopeBuilder.go │ │ ├── event.go │ │ ├── eventBuilder.go │ │ ├── eventBuilder_test.go │ │ ├── event_test.go │ │ └── input.go │ └── webhook │ │ ├── envelopeBuilder.go │ │ ├── eventBuilder.go │ │ └── input.go ├── registry │ ├── backend.go │ ├── handler.go │ ├── handler_test.go │ ├── param.go │ └── registry.go ├── request │ ├── request.go │ └── request_test.go ├── response │ ├── response.go │ ├── response_test.go │ └── retry.go ├── sink │ ├── sink.go │ └── sink_test.go ├── stats │ ├── protocolStats.go │ ├── routes.go │ └── routes_test.go ├── tele │ ├── tele.go │ └── tele_test.go ├── testUtil │ └── gin.go ├── util │ ├── hash.go │ ├── hash_test.go │ ├── http.go │ ├── http_test.go │ ├── identity.go │ ├── identity_test.go │ ├── mapUrlParams.go │ ├── mapUrlParams_test.go │ ├── pprint.go │ ├── pprint_test.go │ ├── schema.go │ ├── schema_test.go │ ├── structToMap.go │ ├── structToMap_test.go │ ├── time.go │ └── time_test.go └── validator │ ├── invalidMessages.go │ └── validator.go └── schemas ├── com.github └── hook │ └── repository │ └── v1.0.json ├── com.stripe └── hook │ └── v1.0.json └── io.silverton ├── amplitude └── event │ └── v1.0.json ├── buz ├── example │ ├── generic │ │ └── sample │ │ │ └── v1.0.json │ ├── gettingStarted │ │ └── v1.0.json │ └── productView │ │ └── v1.0.json ├── hook │ └── arbitrary │ │ └── v1.0.json ├── internal │ ├── config │ │ └── app │ │ │ └── v1.0.json │ ├── contexts │ │ └── httpHeaders │ │ │ └── v1.0.json │ ├── envelope │ │ └── v1.0.json │ ├── meta │ │ └── v1.0.json │ └── tele │ │ ├── beat │ │ └── v1.0.json │ │ ├── meta │ │ └── v1.0.json │ │ ├── shutdown │ │ └── v1.0.json │ │ └── startup │ │ └── v1.0.json └── pixel │ ├── arbitrary │ └── v1.0.json │ ├── linkClick │ └── v1.0.json │ └── pageView │ └── v1.0.json ├── segment └── common │ └── v1.0.json └── snowplow ├── page_ping └── v1.0.json ├── page_view └── v1.0.json ├── payload_data └── v1.4.json ├── struct └── v1.0.json ├── transaction └── v1.0.json └── transaction_item └── v1.0.json /.VERSION: -------------------------------------------------------------------------------- 1 | v0.19.0 -------------------------------------------------------------------------------- /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/go/.devcontainer/base.Dockerfile 2 | 3 | # [Choice] Go version (use -bullseye variants on local arm64/Apple Silicon): 1, 1.19, 1.18, 1-bullseye, 1.19-bullseye, 1.18-bullseye, 1-buster, 1.19-buster, 1.18-buster 4 | ARG VARIANT="1.19-bullseye" 5 | FROM mcr.microsoft.com/vscode/devcontainers/go:0-${VARIANT} 6 | 7 | # [Choice] Node.js version: none, lts/*, 18, 16, 14 8 | ARG NODE_VERSION="none" 9 | RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi 10 | 11 | # [Optional] Uncomment this section to install additional OS packages. 12 | # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ 13 | # && apt-get -y install --no-install-recommends 14 | 15 | # [Optional] Uncomment the next lines to use go get to install anything else you need 16 | # USER vscode 17 | # RUN go get -x 18 | 19 | # [Optional] Uncomment this line to install global node packages. 20 | # RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 21 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: 2 | // https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/go 3 | { 4 | "name": "Go", 5 | "build": { 6 | "dockerfile": "Dockerfile", 7 | "args": { 8 | // Update the VARIANT arg to pick a version of Go: 1, 1.19, 1.18 9 | // Append -bullseye or -buster to pin to an OS version. 10 | // Use -bullseye variants on local arm64/Apple Silicon. 11 | "VARIANT": "1.19", 12 | // Options 13 | "NODE_VERSION": "lts/*" 14 | } 15 | }, 16 | "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], 17 | 18 | // Configure tool-specific properties. 19 | "customizations": { 20 | // Configure properties specific to VS Code. 21 | "vscode": { 22 | // Set *default* container specific settings.json values on container create. 23 | "settings": { 24 | "go.toolsManagement.checkForUpdates": "local", 25 | "go.useLanguageServer": true, 26 | "go.gopath": "/go" 27 | }, 28 | 29 | // Add the IDs of extensions you want installed when the container is created. 30 | "extensions": [ 31 | "golang.Go" 32 | ] 33 | } 34 | }, 35 | 36 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 37 | // "forwardPorts": [], 38 | 39 | // Use 'postCreateCommand' to run commands after the container is created. 40 | // "postCreateCommand": "go version", 41 | 42 | // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. 43 | "remoteUser": "vscode" 44 | } 45 | -------------------------------------------------------------------------------- /.github/workflows/deploy-docker.yml: -------------------------------------------------------------------------------- 1 | name: Publish Container Image 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | env: 9 | GH_REGISTRY: ghcr.io 10 | GCP_REGISTRY: us-east1-docker.pkg.dev 11 | IMAGE_NAME: ${{ github.repository }} 12 | 13 | jobs: 14 | build-and-push-image: 15 | runs-on: ubuntu-latest 16 | permissions: 17 | contents: read 18 | packages: write 19 | id-token: write 20 | 21 | steps: 22 | - name: Checkout repository 23 | uses: actions/checkout@v3 24 | 25 | - name: Get Version from file 26 | id: get-version 27 | uses: juliangruber/read-file-action@v1 28 | with: 29 | path: ./.VERSION 30 | 31 | - name: Login To Github Docker Registry 32 | uses: docker/login-action@v1 33 | with: 34 | registry: ${{ env.GH_REGISTRY }} 35 | username: ${{ github.actor }} 36 | password: ${{ secrets.GITHUB_TOKEN }} 37 | 38 | - name: Set up build cache 39 | uses: actions/cache@v2 40 | with: 41 | path: /tmp/.build-cache 42 | key: ${{ runner.os }}-build-${{ github.sha }} 43 | restore-keys: | 44 | ${{ runner.os }}-build- 45 | 46 | - name: Set up Buildx 47 | id: buildx 48 | uses: docker/setup-buildx-action@v1 49 | 50 | - name: Build and push Docker image 51 | uses: docker/build-push-action@v2 52 | with: 53 | context: . 54 | file: ./deploy/Dockerfile 55 | builder: ${{ steps.buildx.outputs.name }} 56 | platforms: linux/amd64, linux/arm64 57 | push: true 58 | cache-from: type=local,src=/tmp/.build-cache 59 | cache-to: type=local,dest=/tmp/.build-cache 60 | tags: | 61 | ${{ env.GH_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.get-version.outputs.content }} 62 | labels: ${{ steps.meta.outputs.labels }} 63 | -------------------------------------------------------------------------------- /.github/workflows/deploy-schemas.yml: -------------------------------------------------------------------------------- 1 | name: Publish Schemas to Registry 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | deploy-schemas: 10 | runs-on: ubuntu-latest 11 | permissions: 12 | contents: read 13 | id-token: write 14 | steps: 15 | - name: Checkout repository 16 | uses: actions/checkout@v3.5.2 17 | # - name: Auth to GCP 18 | # uses: google-github-actions/auth@v0 19 | # with: 20 | # credentials_json: ${{ secrets.HONEYPOT_CI_CREDENTIALS }} 21 | 22 | # - name: Upload schemas 23 | # uses: google-github-actions/upload-cloud-storage@v0 24 | # with: 25 | # path: schemas 26 | # destination: registry.silverton.io 27 | # parent: false 28 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | branches: 7 | - main 8 | pull_request: 9 | permissions: 10 | contents: read 11 | jobs: 12 | golangci: 13 | name: lint 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/setup-go@v3 17 | with: 18 | go-version: 1.19 19 | - uses: actions/checkout@v3 20 | - name: golangci-lint 21 | uses: golangci/golangci-lint-action@v3 22 | with: 23 | version: v1.49 24 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | env: 9 | DISCORD_WEBHOOK_ID: ${{ secrets.DISCORD_WEBHOOK_ID }} 10 | DISCORD_WEBHOOK_TOKEN: ${{ secrets.DISCORD_WEBHOOK_TOKEN }} 11 | 12 | jobs: 13 | tag-release: 14 | runs-on: ubuntu-latest 15 | permissions: 16 | contents: write 17 | pull-requests: write 18 | steps: 19 | - name: Checkout repository 20 | uses: actions/checkout@v3.5.2 21 | with: 22 | fetch-depth: 0 23 | 24 | - name: Get Version from file 25 | id: get-version 26 | uses: juliangruber/read-file-action@v1.1.6 27 | with: 28 | path: ./.VERSION 29 | 30 | - name: Configure Git 31 | run: | 32 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 33 | git config user.name "$GITHUB_ACTOR" 34 | 35 | - name: Set Reftag 36 | id: tag-version 37 | uses: mathieudutour/github-tag-action@v6.1 38 | with: 39 | github_token: ${{ secrets.GITHUB_TOKEN }} 40 | custom_tag: ${{ steps.get-version.outputs.content }} 41 | tag_prefix: "" 42 | 43 | cut-release: 44 | runs-on: ubuntu-latest 45 | needs: tag-release 46 | permissions: 47 | contents: write 48 | pull-requests: write 49 | steps: 50 | - name: Checkout repository 51 | uses: actions/checkout@v3.5.2 52 | with: 53 | fetch-depth: 0 54 | 55 | - name: Configure Git 56 | run: | 57 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 58 | git config user.name "$GITHUB_ACTOR" 59 | 60 | - name: GoReleaser 61 | uses: goreleaser/goreleaser-action@v4 62 | with: 63 | version: latest 64 | args: release --rm-dist 65 | env: 66 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 67 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | branches: 7 | - main 8 | pull_request: 9 | permissions: 10 | contents: read 11 | jobs: 12 | build: 13 | name: build & test 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/setup-go@v3 17 | with: 18 | go-version: 1.19 19 | - uses: actions/checkout@v3 20 | - name: determine go cache paths 21 | id: go-cache-path 22 | run: | 23 | echo "::set-output name=build::$(go env GOCACHE)" 24 | echo "::set-output name=module::$(go env GOMODCACHE)" 25 | shell: bash 26 | - name: setup go cache 27 | uses: actions/cache@v3 28 | with: 29 | path: | 30 | ${{ steps.go-cache-path.outputs.build }} 31 | ${{ steps.go-cache-path.outputs.module }} 32 | key: ${{ runner.os }}-go-buz-cache-${{ hashFiles('**/go.sum') }} 33 | restore-keys: | 34 | ${{ runner.os }}-go-buz-cache- 35 | - name: build 36 | run: make build 37 | - name: test 38 | run: make test 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_STORE 2 | config.yml 3 | scratch/* 4 | testprofile.out 5 | cmd/scratch/* 6 | # File sink 7 | buz_events.json 8 | buz_invalid_events.json 9 | # Other 10 | notes/* 11 | *.sql 12 | examples/quickstart/minio/* 13 | *.bin 14 | *.meta 15 | .idea/* 16 | 17 | # Terraform 18 | .terraform 19 | .terraform.lock.hcl 20 | .terraform.tfstate.lock* 21 | terraform.* 22 | backend.tf 23 | build/buz 24 | *.build 25 | target/* 26 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 5m 3 | go: "1.19" 4 | skip-files: 5 | - ".*\\_test\\.go$" # ignore test files for now 6 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | builds: 2 | - id: buz 3 | main: ./cmd/buz/ 4 | binary: buz 5 | goos: [windows, darwin, linux, freebsd, openbsd] 6 | goarch: [amd64, arm64] 7 | ldflags: > 8 | -s -w 9 | -X main.VERSION={{.Version}} 10 | archives: 11 | - id: buz 12 | builds: [buz] 13 | format: tar.gz 14 | files: 15 | - README.md 16 | - LICENSE 17 | dist: target/dist 18 | release: 19 | github: 20 | owner: silverton-io 21 | name: buz 22 | prerelease: auto 23 | disable: false 24 | announce: 25 | discord: 26 | enabled: true 27 | message_template: "🐝 {{.Tag}} now available at {{ .ReleaseURL }}" 28 | author: "Queen Bee" 29 | icon_url: "https://raw.githubusercontent.com/silverton-io/buz/main/img/buzz.png" 30 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "yaml.schemas": { 3 | // "https://registry.buz.dev/s/io.silverton/buz/internal/config/app/v1.0.json": "/*conf*.yml" 4 | "./schemas/io.silverton/buz/internal/config/app/v1.0.json": "/*conf*.yml*" 5 | } 6 | } -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: run debug bootstrap bootstrap-destinations build-docker buildx-deploy lint test test-cover-pkg help 2 | S=silverton 3 | REGISTRY:=us-east1-docker.pkg.dev/silverton-io/docker 4 | VERSION:=$(shell cat .VERSION) 5 | BUZ_DIR="./cmd/buz/" 6 | TEST_PROFILE=testprofile.out 7 | 8 | build: 9 | go build -ldflags="-X main.VERSION=$(VERSION)" -o buz $(BUZ_DIR) 10 | 11 | run: ## Run buz locally 12 | go run -ldflags="-X 'main.VERSION=x.x.dev'" $(BUZ_DIR) 13 | 14 | debug: ## Run buz locally with debug 15 | DEBUG=1 go run -ldflags="-X 'main.VERSION=x.x.dev'" $(BUZ_DIR) 16 | 17 | bootstrap: ## Bootstrap development environment 18 | test -f config.yml || cp ./examples/devel/buz/simple.conf.yml config.yml; 19 | make debug 20 | 21 | bootstrap-destinations: ## Bootstrap various containerized database/stream systems 22 | docker-compose -f examples/devel/docker-compose.yml up -d 23 | 24 | build-docker: ## Build local buz image 25 | docker build -f deploy/Dockerfile -t buz:$(VERSION) . 26 | 27 | buildx-deploy: ## Build multi-platform buz image and push it to edge repo 28 | docker buildx create --name $(S) || true; 29 | docker buildx use $(S) 30 | docker buildx build --platform linux/arm64,linux/amd64 -f deploy/Dockerfile -t $(REGISTRY)/buz:$(VERSION)-edge . --push 31 | 32 | lint: ## Lint go code 33 | @golangci-lint run --config .golangci.yml 34 | 35 | test: ## Run tests against pkg 36 | @go test ./pkg/... 37 | 38 | test-cover-pkg: ## Run tests against pkg, output test profile, and open profile in browser 39 | go test ./pkg/... -v -coverprofile=$(TEST_PROFILE) || true 40 | go tool cover -html=$(TEST_PROFILE) || true 41 | 42 | help: ## Display makefile help 43 | @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[$$()% a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 44 | -------------------------------------------------------------------------------- /cmd/buz/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package main 6 | 7 | func main() { 8 | app := App{} 9 | app.Initialize() 10 | app.Run() 11 | } 12 | -------------------------------------------------------------------------------- /cmd/healthcheck/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "fmt" 10 | "io" 11 | "net/http" 12 | "os" 13 | ) 14 | 15 | func main() { 16 | protocol := flag.String("protocol", "http", "The protocol to poll for healthcheck") 17 | host := flag.String("host", "localhost", "The host to poll for healthcheck") 18 | port := flag.String("port", "8080", "The port to poll for healthcheck") 19 | path := flag.String("path", "/health", "The path to poll for healthcheck") 20 | flag.Parse() 21 | healthcheckPath := *protocol + "://" + *host + ":" + *port + *path 22 | fmt.Println("Checking service health via: " + healthcheckPath) 23 | resp, err := http.Get(healthcheckPath) 24 | if err != nil { 25 | fmt.Println("error when getting response: " + err.Error()) 26 | os.Exit(1) 27 | } 28 | if resp == nil { 29 | fmt.Println("empty response") 30 | os.Exit(1) 31 | } 32 | if resp.StatusCode != 200 { 33 | fmt.Println("non-200 status code: " + fmt.Sprint(resp.StatusCode)) 34 | os.Exit(1) 35 | } 36 | b, err := io.ReadAll(resp.Body) 37 | if err != nil { 38 | fmt.Println("could not read response body") 39 | } 40 | fmt.Println(string(b)) 41 | os.Exit(0) 42 | } 43 | -------------------------------------------------------------------------------- /deploy/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.19-bullseye AS build 2 | 3 | WORKDIR /app 4 | 5 | COPY . /app/ 6 | 7 | RUN go mod download 8 | RUN CGO_ENABLED=0 GOOS=linux go build -o healthcheck ./cmd/healthcheck/*.go 9 | RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-X main.VERSION=$(cat .VERSION)" -o buz ./cmd/buz/*.go 10 | 11 | FROM busybox 12 | 13 | LABEL maintainer="Jake Thomas " 14 | LABEL org.opencontainers.image.description "A lightweight, Snowplow-compatible streaming event collection system." 15 | 16 | WORKDIR /app 17 | COPY --from=build /app/.VERSION . 18 | COPY --from=build /app/buz . 19 | COPY --from=build /app/healthcheck . 20 | COPY --from=build /etc/ssl/certs /etc/ssl/certs 21 | COPY --from=build /app/examples/quickstart/buz/quickstart.conf.yml /etc/buz/config.yml 22 | 23 | EXPOSE 8080 24 | ENV BUZ_CONFIG_PATH=/etc/buz/config.yml 25 | 26 | ENTRYPOINT [ "./buz" ] 27 | -------------------------------------------------------------------------------- /deploy/terraform/aws/lambda/Dockerfile.tftpl: -------------------------------------------------------------------------------- 1 | FROM ${sourceImage} 2 | 3 | COPY config.yml.build /etc/buz/config.yml 4 | 5 | ENTRYPOINT [ "./buz" ] 6 | -------------------------------------------------------------------------------- /deploy/terraform/aws/lambda/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/silverton-io/buz/daa3a8032ef9460568d1e88a3c6585ff5928e786/deploy/terraform/aws/lambda/README.md -------------------------------------------------------------------------------- /deploy/terraform/aws/lambda/config.yml.tftpl: -------------------------------------------------------------------------------- 1 | version: 1.1 2 | 3 | app: 4 | name: ${system} 5 | env: ${env} 6 | port: ${port} 7 | trackerDomain: ${trackerDomain} 8 | enableConfigRoute: false 9 | serverless: true 10 | 11 | middleware: 12 | timeout: 13 | enabled: false 14 | ms: 2000 15 | rateLimiter: 16 | enabled: false 17 | period: S 18 | limit: 10 19 | identity: 20 | cookie: 21 | enabled: true 22 | name: nuid 23 | secure: true 24 | ttlDays: 365 25 | domain: .${cookieDomain} 26 | path: / 27 | sameSite: None 28 | fallback: 00000000-0000-4000-A000-000000000000 29 | cors: 30 | enabled: true 31 | allowOrigin: 32 | - "*" 33 | allowCredentials: true 34 | allowMethods: 35 | - POST 36 | - OPTIONS 37 | - GET 38 | maxAge: 86400 39 | requestLogger: 40 | enabled: false 41 | auth: 42 | enabled: false 43 | tokens: 44 | - "" 45 | 46 | inputs: 47 | snowplow: 48 | enabled: true 49 | standardRoutesEnabled: true 50 | openRedirectsEnabled: true 51 | getPath: /snowplow/g 52 | postPath: /snowplow/p 53 | redirectPath: /snowplow/r 54 | cloudevents: 55 | enabled: true 56 | path: /cloudevents 57 | selfDescribing: 58 | enabled: true 59 | path: /self-describing 60 | contexts: 61 | rootKey: contexts 62 | payload: 63 | rootKey: payload 64 | schemaKey: schema 65 | dataKey: data 66 | webhook: 67 | enabled: true 68 | path: /webhook 69 | pixel: 70 | enabled: true 71 | path: /pixel 72 | 73 | registry: 74 | backend: 75 | type: s3 76 | bucket: ${schemaBucket} 77 | path: / 78 | ttlSeconds: 300 79 | maxSizeBytes: 104857600 80 | purge: 81 | enabled: true 82 | path: /c/purge 83 | http: 84 | enabled: true 85 | 86 | sinks: 87 | - name: primary 88 | type: kinesis-firehose 89 | deliveryRequired: true 90 | defaultOutput: ${defaultOutput} 91 | deadletterOutput: ${deadletterOutput} 92 | 93 | squawkBox: 94 | enabled: true 95 | 96 | tele: 97 | enabled: true 98 | -------------------------------------------------------------------------------- /deploy/terraform/aws/lambda/data.tf: -------------------------------------------------------------------------------- 1 | data "aws_caller_identity" "current" {} 2 | data "aws_region" "current" {} 3 | 4 | 5 | data "aws_iam_policy_document" "firehose_assume_role" { 6 | statement { 7 | effect = "Allow" 8 | actions = ["sts:AssumeRole"] 9 | 10 | principals { 11 | type = "Service" 12 | identifiers = ["firehose.amazonaws.com"] 13 | } 14 | } 15 | } 16 | 17 | data "aws_iam_policy_document" "firehose_bucket" { 18 | statement { 19 | effect = "Allow" 20 | 21 | actions = [ 22 | "s3:AbortMultipartUpload", 23 | "s3:GetBucketLocation", 24 | "s3:GetObject", 25 | "s3:ListBucket", 26 | "s3:ListBucketMultipartUploads", 27 | "s3:PutObject", 28 | ] 29 | 30 | resources = [ 31 | aws_s3_bucket.events.arn, 32 | "${aws_s3_bucket.events.arn}/*", 33 | ] 34 | } 35 | } 36 | 37 | data "aws_iam_policy_document" "lambda_assume_policy" { 38 | statement { 39 | actions = ["sts:AssumeRole"] 40 | 41 | principals { 42 | type = "Service" 43 | identifiers = ["lambda.amazonaws.com"] 44 | } 45 | } 46 | } 47 | 48 | data "aws_iam_policy_document" "lambda_role_policy" { 49 | statement { 50 | actions = ["firehose:Put*"] 51 | effect = "Allow" 52 | resources = [ 53 | aws_kinesis_firehose_delivery_stream.default.arn, 54 | aws_kinesis_firehose_delivery_stream.deadletter.arn 55 | ] 56 | } 57 | 58 | statement { 59 | actions = ["s3:Get*"] 60 | effect = "Allow" 61 | resources = [ 62 | aws_s3_bucket.buz_schemas.arn, 63 | "${aws_s3_bucket.buz_schemas.arn}/*", 64 | ] 65 | } 66 | 67 | statement { 68 | effect = "Allow" 69 | actions = ["logs:CreateLogGroup"] 70 | resources = [ 71 | "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}" 72 | ] 73 | } 74 | 75 | statement { 76 | effect = "Allow" 77 | actions = ["logs:CreateLogStream", "logs:PutLogEvents"] 78 | resources = [ 79 | "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/${local.service_name}:*" 80 | ] 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /deploy/terraform/aws/lambda/iam.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_role" "firehose" { 2 | name = "${local.system_env_base}firehose" 3 | assume_role_policy = data.aws_iam_policy_document.firehose_assume_role.json 4 | } 5 | 6 | resource "aws_iam_role_policy" "firehose_bucket" { 7 | name = "${local.system_env_base}firehose" 8 | role = aws_iam_role.firehose.name 9 | policy = data.aws_iam_policy_document.firehose_bucket.json 10 | } 11 | 12 | # Lambda 13 | resource "aws_iam_role" "lambda_role" { 14 | name = "${local.system_env_base}lambda" 15 | path = "/" 16 | assume_role_policy = data.aws_iam_policy_document.lambda_assume_policy.json 17 | } 18 | 19 | resource "aws_iam_policy" "lambda_policy" { 20 | name = "${local.system_env_base}lambda" 21 | policy = data.aws_iam_policy_document.lambda_role_policy.json 22 | } 23 | 24 | resource "aws_iam_role_policy_attachment" "lambda_role_attachment" { 25 | role = aws_iam_role.lambda_role.name 26 | policy_arn = aws_iam_policy.lambda_policy.arn 27 | } 28 | -------------------------------------------------------------------------------- /deploy/terraform/aws/lambda/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | domain_parts = split(".", var.buz_domain) 3 | cookie_domain = join(".", slice(local.domain_parts, 1, length(local.domain_parts))) # Assumes Buz is running on a subdomain and the cookie should be on root 4 | buz_debug_var = "DEBUG" 5 | buz_config_var = "BUZ_CONFIG_PATH" 6 | buz_config_path = "/etc/buz/config.yml" 7 | system_env_base = "${var.system}-${var.env}-" 8 | artifact_repository = "${local.system_env_base}img" 9 | image = "buz:${var.buz_version}" 10 | buz_source_image = "${var.buz_image_repo}/${local.image}" 11 | service_name = "${local.system_env_base}collector" 12 | config = "${local.system_env_base}config" 13 | schema_bucket = "${local.system_env_base}${var.schema_bucket_name}" 14 | events_bucket = "${local.system_env_base}${var.events_bucket_name}" 15 | default_output = "${local.system_env_base}events" 16 | deadletter_output = "${local.system_env_base}invalid-events" 17 | metadata_extraction_params = "{isValid:.isValid,vendor:.vendor,namespace:.namespace,version:.version}" 18 | s3_dynamic_prefix = "isValid=!{partitionKeyFromQuery:isValid}/vendor=!{partitionKeyFromQuery:vendor}/namespace=!{partitionKeyFromQuery:namespace}/version=!{partitionKeyFromQuery:version}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/" 19 | } 20 | -------------------------------------------------------------------------------- /deploy/terraform/aws/lambda/outputs.tf: -------------------------------------------------------------------------------- 1 | output "buz_version" { 2 | value = var.buz_version 3 | } 4 | 5 | # output "buz_url" { 6 | # value = aws_apigatewayv2_api.lambda.api_endpoint 7 | # } 8 | 9 | output "schema_bucket" { 10 | value = local.schema_bucket 11 | } 12 | 13 | output "events_bucket" { 14 | value = local.events_bucket 15 | } 16 | 17 | output "default_output" { 18 | value = local.default_output 19 | } 20 | 21 | output "deadletter_output" { 22 | value = local.deadletter_output 23 | } 24 | 25 | output "buz_function_url" { 26 | value = aws_lambda_function_url.buz.function_url 27 | } 28 | 29 | output "buz_cloudfront_url" { 30 | value = aws_cloudfront_distribution.buz.domain_name 31 | } 32 | -------------------------------------------------------------------------------- /deploy/terraform/aws/lambda/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = "~> 4.0" 8 | } 9 | } 10 | } 11 | 12 | provider "aws" { 13 | region = var.aws_region 14 | profile = var.aws_profile 15 | } 16 | -------------------------------------------------------------------------------- /deploy/terraform/gcp/.terraform-version: -------------------------------------------------------------------------------- 1 | latest:^1.1.0 -------------------------------------------------------------------------------- /deploy/terraform/gcp/cloud_run/README.md: -------------------------------------------------------------------------------- 1 | # Terraform GCP 2 | Deploy Buz to Google Cloud via Terraform. 3 | 4 | ## Prerequisites 5 | 6 | You will need `terraform` and `gcloud`. 7 | 8 | ## Spinning up Buz 9 | 10 | **Auth Gcloud** 11 | 12 | ``` 13 | gcloud auth application-default login 14 | ``` 15 | 16 | 17 | **Apply Terraform** 18 | 19 | ``` 20 | terraform apply 21 | ``` 22 | 23 | **[Optional] - Create and populate terraform.tfvars** 24 | 25 | If you don't want to pass terraform variables interactively, you can optionally create a `terraform.tfvars` file in this directory and populate it: 26 | 27 | ``` 28 | gcp_project = "my-project-23456" 29 | gcp_region = "us-central1" 30 | system = "buz" 31 | buz_domain = "track.yourdomain.com" 32 | buz_version = "v0.x.x" 33 | ``` 34 | -------------------------------------------------------------------------------- /deploy/terraform/gcp/cloud_run/config.yml.tftpl: -------------------------------------------------------------------------------- 1 | version: 1.1 2 | 3 | app: 4 | name: ${system} 5 | env: ${env} 6 | port: ${port} 7 | trackerDomain: ${trackerDomain} 8 | enableConfigRoute: false 9 | serverless: false 10 | 11 | middleware: 12 | timeout: 13 | enabled: false 14 | ms: 2000 15 | rateLimiter: 16 | enabled: false 17 | period: S 18 | limit: 10 19 | identity: 20 | cookie: 21 | enabled: true 22 | name: nuid 23 | secure: true 24 | ttlDays: 365 25 | domain: .${cookieDomain} 26 | path: / 27 | sameSite: None 28 | fallback: 00000000-0000-4000-A000-000000000000 29 | cors: 30 | enabled: true 31 | allowOrigin: 32 | - "*" 33 | allowCredentials: true 34 | allowMethods: 35 | - POST 36 | - OPTIONS 37 | - GET 38 | maxAge: 86400 39 | requestLogger: 40 | enabled: false 41 | auth: 42 | enabled: false 43 | tokens: 44 | - "" 45 | 46 | inputs: 47 | snowplow: 48 | enabled: true 49 | standardRoutesEnabled: true 50 | openRedirectsEnabled: true 51 | getPath: /snowplow/g 52 | postPath: /snowplow/p 53 | redirectPath: /snowplow/r 54 | cloudevents: 55 | enabled: true 56 | path: /cloudevents 57 | selfDescribing: 58 | enabled: true 59 | path: /self-describing 60 | contexts: 61 | rootKey: contexts 62 | payload: 63 | rootKey: payload 64 | schemaKey: schema 65 | dataKey: data 66 | webhook: 67 | enabled: true 68 | path: /webhook 69 | pixel: 70 | enabled: true 71 | path: /pixel 72 | 73 | registry: 74 | backend: 75 | type: gcs 76 | bucket: ${schemaBucket} 77 | path: / 78 | ttlSeconds: 300 79 | maxSizeBytes: 104857600 80 | purge: 81 | enabled: true 82 | path: /c/purge 83 | http: 84 | enabled: true 85 | 86 | sinks: 87 | - name: primary 88 | type: bigquery 89 | deliveryRequired: true 90 | project: ${project} 91 | dataset: ${dataset} 92 | defaultOutput: ${defaultOutput} 93 | deadletterOutput: ${deadletterOutput} 94 | 95 | squawkBox: 96 | enabled: true 97 | 98 | tele: 99 | enabled: true -------------------------------------------------------------------------------- /deploy/terraform/gcp/cloud_run/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | buz_config_var = "BUZ_CONFIG_PATH" 3 | buz_config_dir = "/etc/buz/" 4 | buz_config_path = "${local.buz_config_dir}config.yml" 5 | activate_apis = [ 6 | "artifactregistry.googleapis.com", 7 | "run.googleapis.com", 8 | "secretmanager.googleapis.com" 9 | ] 10 | domain_parts = split(".", var.buz_domain) 11 | cookie_domain = join(".", slice(local.domain_parts, 1, length(local.domain_parts))) # Assumes Buz is running on a subdomain and the cookie should be on root 12 | system_env_base = "${var.system}-${var.env}-" 13 | artifact_repository = "${local.system_env_base}repository" 14 | artifact_registry_location = "${var.gcp_region}-docker.pkg.dev" 15 | artifact_registry_root = "${local.artifact_registry_location}/${var.gcp_project}" 16 | artifact_registry_repository = "${local.system_env_base}repository" 17 | buz_source_image = "ghcr.io/silverton-io/buz:${var.buz_version}" 18 | buz_image = "${local.artifact_registry_root}/${local.artifact_registry_repository}/buz:${var.buz_version}" 19 | service_name = "${local.system_env_base}collector" 20 | config = "${local.system_env_base}config" 21 | schema_bucket = "${local.system_env_base}${var.schema_bucket_name}" 22 | default_output = "${local.system_env_base}events" 23 | default_subscription = "${local.system_env_base}events" 24 | default_table = var.default_bigquery_table 25 | deadletter_output = "${local.system_env_base}invalid-events" 26 | deadletter_subscription = "${local.system_env_base}invalid-events" 27 | deadletter_table = var.deadletter_bigquery_table 28 | } 29 | -------------------------------------------------------------------------------- /deploy/terraform/gcp/cloud_run/outputs.tf: -------------------------------------------------------------------------------- 1 | output "gcp_project" { 2 | value = var.gcp_project 3 | } 4 | 5 | output "gcp_region" { 6 | value = var.gcp_region 7 | } 8 | 9 | output "buz_domain" { 10 | value = var.buz_domain 11 | } 12 | 13 | output "buz_version" { 14 | value = var.buz_version 15 | } 16 | 17 | output "schema_bucket" { 18 | value = local.schema_bucket 19 | } 20 | 21 | output "default_topic" { 22 | value = local.default_output 23 | } 24 | 25 | output "deadletter_topic" { 26 | value = local.deadletter_output 27 | } 28 | 29 | output "buz_service_id" { 30 | value = google_cloud_run_service.buz.id 31 | } 32 | 33 | output "buz_service_status" { 34 | value = google_cloud_run_service.buz.status 35 | } 36 | 37 | output "bigquery_dataset" { 38 | value = var.bigquery_dataset_name 39 | } 40 | 41 | output "default_table" { 42 | value = local.default_table 43 | } 44 | 45 | output "deadletter_table" { 46 | value = local.deadletter_table 47 | } 48 | -------------------------------------------------------------------------------- /deploy/terraform/gcp/cloud_run/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | 4 | required_providers { 5 | google = { 6 | source = "hashicorp/google" 7 | version = ">= 4.33.0" 8 | } 9 | } 10 | } 11 | 12 | provider "google" { 13 | project = var.gcp_project 14 | region = var.gcp_region 15 | } 16 | -------------------------------------------------------------------------------- /deploy/terraform/gcp/cloud_run/schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "data", 4 | "type": "STRING" 5 | }, 6 | { 7 | "name": "subscription_name", 8 | "type": "STRING" 9 | }, 10 | { 11 | "name": "publish_time", 12 | "type": "TIMESTAMP" 13 | }, 14 | { 15 | "name": "message_id", 16 | "type": "STRING" 17 | }, 18 | { 19 | "name": "attributes", 20 | "type": "STRING" 21 | } 22 | ] -------------------------------------------------------------------------------- /examples/quickstart/nginx/nginx.conf: -------------------------------------------------------------------------------- 1 | user nginx; 2 | worker_processes auto; 3 | 4 | error_log /var/log/nginx/error.log notice; 5 | pid /var/run/nginx.pid; 6 | 7 | 8 | events { 9 | worker_connections 1024; 10 | } 11 | 12 | 13 | http { 14 | include /etc/nginx/mime.types; 15 | default_type application/octet-stream; 16 | 17 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 18 | '$status $body_bytes_sent "$http_referer" ' 19 | '"$http_user_agent" "$http_x_forwarded_for"'; 20 | 21 | access_log /var/log/nginx/access.log main; 22 | 23 | sendfile on; 24 | 25 | keepalive_timeout 65; 26 | 27 | server { 28 | listen 8081; 29 | server_name localhost; 30 | location = / { 31 | proxy_pass http://buz-quickstart-ui:8080; 32 | } 33 | location / { 34 | proxy_pass http://buz:8080; 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /img/buzz.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/silverton-io/buz/daa3a8032ef9460568d1e88a3c6585ff5928e786/img/buzz.png -------------------------------------------------------------------------------- /pkg/annotator/annotator.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package annotator 6 | 7 | import ( 8 | "github.com/rs/zerolog/log" 9 | "github.com/silverton-io/buz/pkg/envelope" 10 | "github.com/silverton-io/buz/pkg/registry" 11 | "github.com/silverton-io/buz/pkg/validator" 12 | "github.com/tidwall/gjson" 13 | ) 14 | 15 | type schemaMetadata struct { 16 | Vendor string 17 | Namespace string 18 | Version string 19 | DisableValidation bool 20 | } 21 | 22 | func getSchemaMetadata(schema []byte) schemaMetadata { 23 | schemaContents := gjson.ParseBytes(schema) 24 | vendor := schemaContents.Get("self.vendor").String() 25 | namespace := schemaContents.Get("self.namespace").String() 26 | version := schemaContents.Get("self.version").String() 27 | disableValidation := schemaContents.Get("disableValidation").Bool() 28 | log.Debug().Msgf("🟡 vendor: %s, namespace: %s, version: %s, disableValidation: %t", vendor, namespace, version, disableValidation) 29 | return schemaMetadata{ 30 | Vendor: vendor, 31 | Namespace: namespace, 32 | Version: version, 33 | DisableValidation: disableValidation, 34 | } 35 | } 36 | 37 | func Annotate(envelopes []envelope.Envelope, registry *registry.Registry) []envelope.Envelope { 38 | var e []envelope.Envelope 39 | for _, envelope := range envelopes { 40 | log.Debug().Msg("🟡 annotating event") 41 | isValid, validationError, schemaContents := validator.Validate(envelope, registry) 42 | m := getSchemaMetadata(schemaContents) 43 | if m.Namespace != "" { 44 | envelope.Vendor = m.Vendor 45 | envelope.Namespace = m.Namespace 46 | envelope.Version = m.Version 47 | } 48 | if m.DisableValidation { 49 | // If schema-level validation is disabled 50 | // consider the payload valid. 51 | valid := true 52 | envelope.IsValid = valid 53 | } else { 54 | envelope.IsValid = isValid 55 | if !isValid { 56 | // Annotate the envelope with associated validation errors 57 | envelope.ValidationError = &validationError 58 | } 59 | } 60 | e = append(e, envelope) 61 | } 62 | return e 63 | } 64 | -------------------------------------------------------------------------------- /pkg/annotator/annotator_test.go: -------------------------------------------------------------------------------- 1 | package annotator 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestGetSchemaMetadata(t *testing.T) { 10 | testData := []struct { 11 | name string 12 | input []byte 13 | expected schemaMetadata 14 | }{ 15 | { 16 | name: "Valid JSON", 17 | input: []byte(`{"self":{"vendor":"testVendor","namespace":"testNamespace","version":"testVersion"},"disableValidation":true}`), 18 | expected: schemaMetadata{ 19 | Vendor: "testVendor", 20 | Namespace: "testNamespace", 21 | Version: "testVersion", 22 | DisableValidation: true, 23 | }, 24 | }, 25 | } 26 | 27 | for _, tc := range testData { 28 | t.Run(tc.name, func(t *testing.T) { 29 | result := getSchemaMetadata(tc.input) 30 | assert.Equal(t, tc.expected, result) 31 | }) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /pkg/backend/azuredw/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package sink 6 | 7 | // FIXME! Implement azure dw sink 8 | -------------------------------------------------------------------------------- /pkg/backend/blackhole/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package blackhole 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/rs/zerolog/log" 11 | "github.com/silverton-io/buz/pkg/backend/backendutils" 12 | "github.com/silverton-io/buz/pkg/config" 13 | "github.com/silverton-io/buz/pkg/envelope" 14 | ) 15 | 16 | type Sink struct { 17 | metadata backendutils.SinkMetadata 18 | } 19 | 20 | func (s *Sink) Metadata() backendutils.SinkMetadata { 21 | return s.metadata 22 | } 23 | 24 | func (s *Sink) Initialize(conf config.Sink) error { 25 | s.metadata = backendutils.NewSinkMetadataFromConfig(conf) 26 | return nil 27 | } 28 | 29 | func (s *Sink) StartWorker() error { 30 | // Blackhole. No worker necessary 31 | return nil 32 | } 33 | 34 | func (s *Sink) Enqueue(envelopes []envelope.Envelope) error { 35 | log.Debug().Interface("metadata", s.Metadata()).Msg("enqueueing envelopes") 36 | // This is a blackhole. It does nothing but dequeue 37 | ctx := context.Background() 38 | err := s.Dequeue(ctx, envelopes, "nothingness") 39 | if err != nil { 40 | log.Error().Err(err).Interface("metadata", s.Metadata()).Msg("could not dequeue") 41 | } 42 | return nil 43 | } 44 | 45 | func (s *Sink) Dequeue(ctx context.Context, envelopes []envelope.Envelope, output string) error { 46 | log.Debug().Interface("metadata", s.Metadata()).Msg("dequeueing envelopes") 47 | // This is a blackhole. It does nothing. 48 | return nil 49 | } 50 | 51 | func (s *Sink) Shutdown() error { 52 | log.Debug().Interface("metadata", s.metadata).Msg("🟢 shutting down sink") 53 | return nil 54 | } 55 | -------------------------------------------------------------------------------- /pkg/backend/clickhousedb/dsn.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package clickhousedb 6 | 7 | import ( 8 | "strconv" 9 | 10 | "github.com/silverton-io/buz/pkg/db" 11 | ) 12 | 13 | // generateDsn generates a Clickhouse dsn from the provided connection params 14 | func generateDsn(params db.ConnectionParams) string { 15 | // "tcp://localhost:9000?database=gorm&username=gorm&password=gorm&read_timeout=10&write_timeout=20" 16 | port := strconv.FormatUint(uint64(params.Port), 10) 17 | return "tcp://" + params.Host + ":" + port + "?database=" + params.Db + "&username=" + params.User + "&password=" + params.Pass 18 | } 19 | -------------------------------------------------------------------------------- /pkg/backend/clickhousedb/dsn_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package clickhousedb 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/silverton-io/buz/pkg/db" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestGenerateDsn(t *testing.T) { 15 | p := db.ConnectionParams{ 16 | Host: "myhost", 17 | Port: 9000, 18 | Db: "db", 19 | User: "usr", 20 | Pass: "pass", 21 | } 22 | 23 | dsn := generateDsn(p) 24 | expectedDsn := "tcp://myhost:9000?database=db&username=usr&password=pass" 25 | 26 | assert.Equal(t, expectedDsn, dsn) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/backend/clickhousedb/registry.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package clickhousedb 6 | 7 | import ( 8 | "github.com/rs/zerolog/log" 9 | "github.com/silverton-io/buz/pkg/config" 10 | "github.com/silverton-io/buz/pkg/db" 11 | "gorm.io/driver/clickhouse" 12 | "gorm.io/gorm" 13 | ) 14 | 15 | type RegistryBackend struct { 16 | gormDb *gorm.DB 17 | registryTable string 18 | } 19 | 20 | func (b *RegistryBackend) Initialize(conf config.Backend) error { 21 | connParams := db.ConnectionParams{ 22 | Host: conf.DbHost, 23 | Port: conf.DbPort, 24 | Db: conf.DbName, 25 | User: conf.DbUser, 26 | Pass: conf.DbPass, 27 | } 28 | connString := generateDsn(connParams) 29 | gormDb, err := gorm.Open(clickhouse.Open(connString), &gorm.Config{}) 30 | if err != nil { 31 | log.Error().Err(err).Msg("🔴 could not open clickhouse connection") 32 | return err 33 | } 34 | b.gormDb, b.registryTable = gormDb, conf.RegistryTable 35 | ensureErr := db.EnsureTable(b.gormDb, b.registryTable, db.RegistryTable{}) 36 | return ensureErr 37 | } 38 | 39 | func (b *RegistryBackend) GetRemote(schema string) (contents []byte, err error) { 40 | var s db.RegistryTable 41 | b.gormDb.Table(b.registryTable).Where("name = ?", schema).First(&s) 42 | err = b.gormDb.Error 43 | if err != nil { 44 | return nil, err 45 | } 46 | return s.Contents, nil 47 | } 48 | 49 | func (b *RegistryBackend) Close() { 50 | log.Info().Msg("🟢 closing clickhouse schema cache backend") 51 | } 52 | -------------------------------------------------------------------------------- /pkg/backend/clickhousedb/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package clickhousedb 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/rs/zerolog/log" 11 | "github.com/silverton-io/buz/pkg/backend/backendutils" 12 | "github.com/silverton-io/buz/pkg/config" 13 | "github.com/silverton-io/buz/pkg/db" 14 | "github.com/silverton-io/buz/pkg/envelope" 15 | "gorm.io/driver/clickhouse" 16 | "gorm.io/gorm" 17 | ) 18 | 19 | type Sink struct { 20 | metadata backendutils.SinkMetadata 21 | gormDb *gorm.DB 22 | } 23 | 24 | func (s *Sink) Initialize(conf config.Sink) error { 25 | log.Debug().Msg("🟡 initializing clickhouse sink") 26 | s.metadata = backendutils.NewSinkMetadataFromConfig(conf) 27 | connParams := db.ConnectionParams{ 28 | Host: conf.Hosts[0], // Only use the first configured host 29 | Port: conf.Port, 30 | Db: conf.Name, 31 | User: conf.User, 32 | Pass: conf.Password, 33 | } 34 | connString := generateDsn(connParams) 35 | gormDb, err := gorm.Open(clickhouse.Open(connString), &gorm.Config{}) 36 | if err != nil { 37 | log.Error().Err(err).Msg("🔴 could not open clickhouse connection") 38 | return err 39 | } 40 | s.gormDb = gormDb 41 | for _, tbl := range []string{s.metadata.DefaultOutput, s.metadata.DeadletterOutput} { 42 | ensureErr := db.EnsureTable(s.gormDb, tbl, &envelope.StringEnvelope{}) 43 | if ensureErr != nil { 44 | return ensureErr 45 | } 46 | } 47 | return nil 48 | } 49 | 50 | func (s *Sink) BatchPublish(ctx context.Context, envelopes []envelope.Envelope) error { 51 | // Get shards 52 | // err := s.gormDb.Table(s.validTable).Create(envelopes).Error 53 | // return err 54 | return nil 55 | } 56 | 57 | func (s *Sink) Close() { 58 | log.Debug().Msg("🟡 closing mysql sink") 59 | db, _ := s.gormDb.DB() 60 | db.Close() 61 | } 62 | -------------------------------------------------------------------------------- /pkg/backend/file/registry.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package file 6 | 7 | import ( 8 | "os" 9 | "path/filepath" 10 | 11 | "github.com/rs/zerolog/log" 12 | "github.com/silverton-io/buz/pkg/config" 13 | ) 14 | 15 | type RegistryBackend struct { 16 | path string 17 | } 18 | 19 | func (b *RegistryBackend) Initialize(conf config.Backend) error { 20 | log.Debug().Msg("🟡 initializing filesystem registry backend") 21 | b.path = conf.Path 22 | // No-op 23 | return nil 24 | } 25 | 26 | func (b *RegistryBackend) GetRemote(schema string) (contents []byte, err error) { 27 | schemaLocation := filepath.Join(b.path, schema) 28 | content, err := os.ReadFile(schemaLocation) 29 | if err != nil { 30 | log.Error().Err(err).Msg("🔴 could not get schema from filesystem registry backend: " + schemaLocation) 31 | return nil, err 32 | } 33 | return content, nil 34 | } 35 | 36 | func (b *RegistryBackend) Close() { 37 | log.Debug().Msg("🟡 closing filesystem registry backend") 38 | // No-op 39 | } 40 | -------------------------------------------------------------------------------- /pkg/backend/file/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package file 6 | 7 | import ( 8 | "context" 9 | "encoding/json" 10 | "os" 11 | 12 | "github.com/rs/zerolog/log" 13 | "github.com/silverton-io/buz/pkg/backend/backendutils" 14 | "github.com/silverton-io/buz/pkg/config" 15 | "github.com/silverton-io/buz/pkg/envelope" 16 | ) 17 | 18 | type Sink struct { 19 | metadata backendutils.SinkMetadata 20 | input chan []envelope.Envelope 21 | shutdown chan int 22 | } 23 | 24 | func (s *Sink) Metadata() backendutils.SinkMetadata { 25 | return s.metadata 26 | } 27 | 28 | func (s *Sink) Initialize(conf config.Sink) error { 29 | log.Debug().Msg("🟡 initializing file sink") 30 | s.metadata = backendutils.NewSinkMetadataFromConfig(conf) 31 | s.input = make(chan []envelope.Envelope, 10000) 32 | s.shutdown = make(chan int, 1) 33 | return nil 34 | } 35 | 36 | func (s *Sink) StartWorker() error { 37 | err := backendutils.StartSinkWorker(s.input, s.shutdown, s) 38 | return err 39 | } 40 | 41 | func (s *Sink) batchPublish(ctx context.Context, filePath string, envelopes []envelope.Envelope) error { 42 | f, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 43 | if err != nil { 44 | log.Error().Err(err).Msg("🔴 could not open file") 45 | return err 46 | } 47 | defer f.Close() // nolint 48 | for _, envelope := range envelopes { 49 | b, err := json.Marshal(envelope) 50 | if err != nil { 51 | log.Error().Err(err).Msg("🔴 could not marshal envelope") 52 | return err 53 | } 54 | newline := []byte("\n") 55 | b = append(b, newline...) 56 | if _, err := f.Write(b); err != nil { 57 | return err 58 | } 59 | } 60 | return nil 61 | } 62 | 63 | func (s *Sink) Enqueue(envelopes []envelope.Envelope) error { 64 | log.Debug().Interface("metadata", s.Metadata()).Msg("enqueueing envelopes") 65 | s.input <- envelopes 66 | return nil 67 | } 68 | 69 | func (s *Sink) Dequeue(ctx context.Context, envelopes []envelope.Envelope, output string) error { 70 | log.Debug().Interface("metadata", s.Metadata()).Msg("dequeueing envelopes") 71 | err := s.batchPublish(ctx, output, envelopes) 72 | return err 73 | } 74 | 75 | func (s *Sink) Shutdown() error { 76 | log.Debug().Interface("metadata", s.metadata).Msg("🟢 shutting down sink") 77 | s.shutdown <- 1 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /pkg/backend/firebolt/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package sink 6 | 7 | // TODO! Implement me! 8 | -------------------------------------------------------------------------------- /pkg/backend/gcs/registry.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package gcs 6 | 7 | import ( 8 | "context" 9 | "io" 10 | "path/filepath" 11 | 12 | "cloud.google.com/go/storage" 13 | "github.com/rs/zerolog/log" 14 | "github.com/silverton-io/buz/pkg/config" 15 | ) 16 | 17 | type RegistryBackend struct { 18 | bucket string 19 | path string 20 | client *storage.Client 21 | } 22 | 23 | func (b *RegistryBackend) Initialize(config config.Backend) error { 24 | ctx := context.Background() 25 | log.Debug().Msg("🟡 initializing gcs schema cache backend") 26 | client, err := storage.NewClient(ctx) 27 | if err != nil { 28 | log.Error().Err(err).Msg("🔴 could not initialize gcs schema cache backend") 29 | return err 30 | } 31 | b.client, b.bucket, b.path = client, config.Bucket, config.Path 32 | return nil 33 | } 34 | 35 | func (b *RegistryBackend) GetRemote(schema string) (contents []byte, err error) { 36 | ctx := context.Background() 37 | var schemaLocation string 38 | if b.path == "/" { 39 | schemaLocation = schema 40 | } else { 41 | schemaLocation = filepath.Join(b.path, schema) 42 | } 43 | log.Debug().Msg("🟡 getting file from gcs backend " + schemaLocation) 44 | reader, err := b.client.Bucket(b.bucket).Object(schemaLocation).NewReader(ctx) 45 | if err != nil { 46 | log.Error().Err(err).Msg("🔴 could not get file from gcs: " + schemaLocation) 47 | return nil, err 48 | } 49 | data, _ := io.ReadAll(reader) 50 | return data, nil 51 | } 52 | 53 | func (b *RegistryBackend) Close() { 54 | log.Debug().Msg("🟡 closing gcs schema cache backend") 55 | b.client.Close() 56 | } 57 | -------------------------------------------------------------------------------- /pkg/backend/http/registry.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package http 6 | 7 | import ( 8 | "net/url" 9 | 10 | "github.com/rs/zerolog/log" 11 | "github.com/silverton-io/buz/pkg/config" 12 | "github.com/silverton-io/buz/pkg/request" 13 | ) 14 | 15 | type RegistryBackend struct { 16 | protocol string 17 | host string 18 | path string 19 | } 20 | 21 | func (b *RegistryBackend) Initialize(conf config.Backend) error { 22 | log.Debug().Msg("🟡 initializing http schema cache backend") 23 | b.protocol = conf.Type 24 | b.host = conf.Host // FIXME! String trailing / if it's present (or validate it upstream) 25 | b.path = conf.Path // FIXME! Strip leading / if it's present (or validate it upstream) 26 | return nil 27 | } 28 | 29 | func (b *RegistryBackend) GetRemote(schema string) (contents []byte, err error) { 30 | schemaLocation, _ := url.Parse(b.protocol + "://" + b.host + "/" + b.path + "/" + schema) // FIXME!! There's gotta be a better way here. 31 | content, err := request.Get(*schemaLocation) 32 | if err != nil { 33 | log.Error().Err(err).Msg("🔴 could not get schema from http schema cache backend") 34 | return nil, err 35 | } 36 | return content, nil 37 | } 38 | 39 | func (b *RegistryBackend) Close() { 40 | log.Debug().Msg("🟡 closing http schema cache backend") 41 | // Knock off auth tokens? TBD 42 | } 43 | -------------------------------------------------------------------------------- /pkg/backend/http/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package http 6 | 7 | import ( 8 | "context" 9 | "net/http" 10 | "net/url" 11 | 12 | "github.com/rs/zerolog/log" 13 | "github.com/silverton-io/buz/pkg/backend/backendutils" 14 | "github.com/silverton-io/buz/pkg/config" 15 | "github.com/silverton-io/buz/pkg/envelope" 16 | "github.com/silverton-io/buz/pkg/request" 17 | ) 18 | 19 | type Sink struct { 20 | metadata backendutils.SinkMetadata 21 | input chan []envelope.Envelope 22 | shutdown chan int 23 | } 24 | 25 | func (s *Sink) Metadata() backendutils.SinkMetadata { 26 | return s.metadata 27 | } 28 | 29 | func (s *Sink) Initialize(conf config.Sink) error { 30 | _, err := url.Parse(conf.DefaultOutput) 31 | if err != nil { 32 | log.Error().Err(err).Msg("🔴 " + conf.DefaultOutput + " is not a valid url") 33 | return err 34 | } 35 | _, err = url.Parse(conf.DeadletterOutput) 36 | if err != nil { 37 | log.Error().Err(err).Msg("🔴 " + conf.DeadletterOutput + " is not a valid url") 38 | return err 39 | } 40 | s.metadata = backendutils.NewSinkMetadataFromConfig(conf) 41 | s.input = make(chan []envelope.Envelope, 10000) 42 | s.shutdown = make(chan int, 1) 43 | return nil 44 | } 45 | 46 | func (s *Sink) StartWorker() error { 47 | err := backendutils.StartSinkWorker(s.input, s.shutdown, s) 48 | return err 49 | } 50 | 51 | func (s *Sink) Enqueue(envelopes []envelope.Envelope) error { 52 | log.Debug().Interface("metadata", s.Metadata()).Msg("enqueueing envelopes") 53 | s.input <- envelopes 54 | return nil 55 | } 56 | 57 | func (s *Sink) Dequeue(ctx context.Context, envelopes []envelope.Envelope, output string) error { 58 | log.Debug().Interface("metadata", s.Metadata()).Msg("dequeueing envelopes") 59 | url, err := url.Parse(output) 60 | if err != nil { 61 | log.Error().Err(err).Msg("🔴 " + output + " is not a valid url") 62 | return err 63 | } 64 | _, err = request.PostEnvelopes(*url, envelopes, http.Header{}) 65 | if err != nil { 66 | log.Error().Err(err).Interface("metadata", s.Metadata()).Msg("🔴 could not dequeue payloads") 67 | } 68 | return err 69 | } 70 | 71 | func (s *Sink) Shutdown() error { 72 | log.Debug().Interface("metadata", s.metadata).Msg("🟢 shutting down sink") 73 | s.shutdown <- 1 74 | return nil 75 | } 76 | -------------------------------------------------------------------------------- /pkg/backend/kafka/registry.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package kafka 6 | 7 | // TODO! Implement me!! 8 | -------------------------------------------------------------------------------- /pkg/backend/minio/registry.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package minio 6 | 7 | import ( 8 | "context" 9 | "io" 10 | "path/filepath" 11 | 12 | "github.com/minio/minio-go/v7" 13 | "github.com/minio/minio-go/v7/pkg/credentials" 14 | "github.com/rs/zerolog/log" 15 | "github.com/silverton-io/buz/pkg/config" 16 | ) 17 | 18 | type RegistryBackend struct { 19 | endpoint string 20 | accessKeyId string 21 | secretAccessKey string 22 | // TODO: add support for token 23 | // TODO: add support for useSsl 24 | bucket string 25 | path string 26 | client *minio.Client 27 | } 28 | 29 | func (b *RegistryBackend) Initialize(conf config.Backend) error { 30 | log.Debug().Msg("🟡 initializing minio schema cache backend") 31 | b.endpoint = conf.MinioEndpoint 32 | b.accessKeyId = conf.AccessKeyId 33 | b.secretAccessKey = conf.SecretAccessKey 34 | b.bucket, b.path = conf.Bucket, conf.Path 35 | client, err := minio.New(b.endpoint, &minio.Options{ 36 | Creds: credentials.NewStaticV4(b.accessKeyId, b.secretAccessKey, ""), 37 | }) 38 | if err != nil { 39 | log.Error().Err(err).Msg("🔴 could not initialize minio client") 40 | return err 41 | } 42 | b.client = client 43 | return nil 44 | } 45 | 46 | func (b *RegistryBackend) GetRemote(schema string) (contents []byte, err error) { 47 | ctx := context.Background() 48 | var schemaLocation string 49 | if b.path == "/" { 50 | schemaLocation = schema 51 | } else { 52 | schemaLocation = filepath.Join(b.path, schema) 53 | } 54 | log.Debug().Msg("🟡 getting file from minio backend " + schemaLocation) 55 | obj, err := b.client.GetObject(ctx, b.bucket, schemaLocation, minio.GetObjectOptions{}) 56 | if err != nil { 57 | log.Error().Err(err).Msg("🔴 could not get file from minio: " + schemaLocation) 58 | return nil, err 59 | } 60 | contents, err = io.ReadAll(obj) 61 | if err != nil { 62 | log.Error().Err(err).Msg("🔴 could not read contents from file: " + schemaLocation) 63 | return nil, err 64 | } 65 | return contents, nil 66 | } 67 | 68 | func (b *RegistryBackend) Close() { 69 | log.Debug().Msg("🟡 closing minio schema cache backend") 70 | } 71 | -------------------------------------------------------------------------------- /pkg/backend/mongodb/registry.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package mongodb 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/rs/zerolog/log" 11 | "github.com/silverton-io/buz/pkg/config" 12 | "go.mongodb.org/mongo-driver/bson" 13 | "go.mongodb.org/mongo-driver/bson/primitive" 14 | "go.mongodb.org/mongo-driver/mongo" 15 | "go.mongodb.org/mongo-driver/mongo/options" 16 | ) 17 | 18 | type MongoSchemaDocument struct { 19 | ID primitive.ObjectID `bson:"_id"` 20 | Name string `bson:"name"` 21 | Contents string `bson:"contents"` 22 | } 23 | 24 | type RegistryBackend struct { 25 | client *mongo.Client 26 | registryCollection *mongo.Collection 27 | } 28 | 29 | func (b *RegistryBackend) Initialize(conf config.Backend) error { 30 | ctx := context.Background() 31 | opt := options.ClientOptions{ 32 | Hosts: conf.MongoHosts, 33 | } 34 | if conf.MongoUser != "" { 35 | c := options.Credential{ 36 | Username: conf.MongoUser, 37 | Password: conf.MongoPass, 38 | } 39 | opt.Auth = &c 40 | } 41 | client, err := mongo.Connect(ctx, &opt) 42 | if err != nil { 43 | log.Error().Err(err).Msg("🔴 could not connect to mongodb") 44 | } 45 | b.client = client 46 | registryCollection := b.client.Database(conf.MongoDbName).Collection(conf.RegistryCollection) 47 | b.registryCollection = registryCollection 48 | return nil 49 | } 50 | 51 | func (b *RegistryBackend) GetRemote(schema string) (contents []byte, err error) { 52 | ctx := context.Background() 53 | var doc = MongoSchemaDocument{} 54 | err = b.registryCollection.FindOne(ctx, bson.M{"name": schema}).Decode(&doc) 55 | if err != nil { 56 | log.Error().Err(err).Msg("🔴 could not decode document") 57 | return nil, err 58 | } 59 | if err != nil { 60 | log.Error().Err(err).Msg("🔴 could not marshal document") 61 | return nil, err 62 | } 63 | return []byte(doc.Contents), nil 64 | } 65 | 66 | func (b *RegistryBackend) Close() { 67 | log.Info().Msg("🟢 closing mongodb schema cache backend") 68 | } 69 | -------------------------------------------------------------------------------- /pkg/backend/mysqldb/dsn.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package mysqldb 6 | 7 | import ( 8 | "strconv" 9 | 10 | "github.com/silverton-io/buz/pkg/db" 11 | ) 12 | 13 | // GenerateMysqlDsn generates a Mysql Dsn from the provided connection params 14 | func generateDsn(params db.ConnectionParams) string { 15 | port := strconv.FormatUint(uint64(params.Port), 10) 16 | return params.User + ":" + params.Pass + "@tcp(" + params.Host + ":" + port + ")/" + params.Db + "?parseTime=true" 17 | } 18 | -------------------------------------------------------------------------------- /pkg/backend/mysqldb/dsn_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package mysqldb 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/silverton-io/buz/pkg/db" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestGenerateDsn(t *testing.T) { 15 | p := db.ConnectionParams{ 16 | Host: "host", 17 | Port: 3306, 18 | Db: "db", 19 | User: "usr", 20 | Pass: "pass", 21 | } 22 | 23 | expectedDsn := "usr:pass@tcp(host:3306)/db?parseTime=true" 24 | dsn := generateDsn(p) 25 | 26 | assert.Equal(t, expectedDsn, dsn) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/backend/mysqldb/registry.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package mysqldb 6 | 7 | import ( 8 | "encoding/json" 9 | 10 | "github.com/rs/zerolog/log" 11 | "github.com/silverton-io/buz/pkg/config" 12 | "github.com/silverton-io/buz/pkg/db" 13 | "gorm.io/driver/mysql" 14 | "gorm.io/gorm" 15 | ) 16 | 17 | type RegistryBackend struct { 18 | gormDb *gorm.DB 19 | registryTable string 20 | } 21 | 22 | func (b *RegistryBackend) Initialize(conf config.Backend) error { 23 | connParams := db.ConnectionParams{ 24 | Host: conf.DbHost, 25 | Port: conf.DbPort, 26 | Db: conf.DbName, 27 | User: conf.DbUser, 28 | Pass: conf.DbPass, 29 | } 30 | connString := generateDsn(connParams) 31 | gormDb, err := gorm.Open(mysql.Open(connString)) 32 | if err != nil { 33 | log.Error().Err(err).Msg("🔴 could not open mysql connection") 34 | return err 35 | } 36 | b.gormDb, b.registryTable = gormDb, conf.RegistryTable 37 | ensureErr := db.EnsureTable(b.gormDb, b.registryTable, db.RegistryTable{}) 38 | return ensureErr 39 | } 40 | 41 | func (b *RegistryBackend) GetRemote(schema string) (contents []byte, err error) { 42 | var s db.RegistryTable 43 | b.gormDb.Table(b.registryTable).Where("name = ?", schema).First(&s) 44 | err = b.gormDb.Error 45 | if err != nil { 46 | log.Error().Err(err).Msg("🔴 gorm error") 47 | return nil, err 48 | } 49 | contents, err = json.Marshal(s.Contents) 50 | if err != nil { 51 | log.Error().Err(err).Msg("🔴 could not marshal schema contents") 52 | } 53 | return contents, nil 54 | } 55 | 56 | func (b *RegistryBackend) Close() { 57 | log.Info().Msg("🟢 closing mysql schema cache backend") 58 | } 59 | -------------------------------------------------------------------------------- /pkg/backend/nats/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package nats 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/nats-io/nats.go" 11 | "github.com/rs/zerolog/log" 12 | "github.com/silverton-io/buz/pkg/backend/backendutils" 13 | "github.com/silverton-io/buz/pkg/config" 14 | "github.com/silverton-io/buz/pkg/envelope" 15 | ) 16 | 17 | type Sink struct { 18 | metadata backendutils.SinkMetadata 19 | conn *nats.Conn 20 | encodedConn *nats.EncodedConn 21 | input chan []envelope.Envelope 22 | shutdown chan int 23 | // FIXME! Add .creds/token/tls cert/nkey auth 24 | } 25 | 26 | func (s *Sink) Metadata() backendutils.SinkMetadata { 27 | return s.metadata 28 | } 29 | 30 | func (s *Sink) Initialize(conf config.Sink) error { 31 | log.Debug().Msg("🟡 initializing nats sink") 32 | s.metadata = backendutils.NewSinkMetadataFromConfig(conf) 33 | conn, err := nats.Connect(conf.Hosts[0], nats.UserInfo(conf.User, conf.Password)) 34 | if err != nil { 35 | log.Error().Err(err).Msg("🔴 could not open nats connection") 36 | return err 37 | } 38 | encodedConn, err := nats.NewEncodedConn(conn, nats.JSON_ENCODER) 39 | if err != nil { 40 | log.Error().Err(err).Msg("🔴 could not open encoded connection") 41 | return err 42 | } 43 | s.conn, s.encodedConn = conn, encodedConn 44 | s.input = make(chan []envelope.Envelope, 10000) 45 | s.shutdown = make(chan int, 1) 46 | return nil 47 | } 48 | 49 | func (s *Sink) StartWorker() error { 50 | err := backendutils.StartSinkWorker(s.input, s.shutdown, s) 51 | return err 52 | } 53 | 54 | func (s *Sink) Enqueue(envelopes []envelope.Envelope) error { 55 | log.Debug().Interface("metadata", s.Metadata()).Msg("enqueueing envelopes") 56 | s.input <- envelopes 57 | return nil 58 | } 59 | 60 | func (s *Sink) Dequeue(ctx context.Context, envelopes []envelope.Envelope, output string) error { 61 | log.Debug().Interface("metadata", s.Metadata()).Msg("dequeueing envelopes") 62 | for _, e := range envelopes { 63 | err := s.encodedConn.Publish(output, &e) 64 | if err != nil { 65 | return err 66 | } 67 | } 68 | return nil 69 | } 70 | 71 | func (s *Sink) Shutdown() error { 72 | log.Debug().Interface("metadata", s.metadata).Msg("🟢 shutting down sink") 73 | s.shutdown <- 1 74 | s.conn.Close() 75 | s.encodedConn.Close() 76 | return nil 77 | } 78 | -------------------------------------------------------------------------------- /pkg/backend/natsJetstream/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package natsJetstream 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/nats-io/nats.go" 11 | "github.com/rs/zerolog/log" 12 | "github.com/silverton-io/buz/pkg/backend/backendutils" 13 | "github.com/silverton-io/buz/pkg/config" 14 | "github.com/silverton-io/buz/pkg/envelope" 15 | ) 16 | 17 | type Sink struct { 18 | metadata backendutils.SinkMetadata 19 | conn *nats.Conn 20 | jetstream nats.JetStreamContext 21 | // FIXME! Add .creds/token/tls cert/nkey auth 22 | } 23 | 24 | func (s *Sink) Initialize(conf config.Sink) error { 25 | log.Debug().Msg("🟡 initializing nats jetstream sink") 26 | s.metadata = backendutils.NewSinkMetadataFromConfig(conf) 27 | conn, err := nats.Connect(conf.Hosts[0], nats.UserInfo(conf.User, conf.Password)) 28 | if err != nil { 29 | log.Error().Err(err).Msg("🔴 could not open nats connection") 30 | return err 31 | } 32 | js, err := conn.JetStream() 33 | 34 | if err != nil { 35 | log.Error().Err(err).Msg("🔴 could not use jetstream context") 36 | return err 37 | } 38 | s.conn, s.jetstream = conn, js 39 | return nil 40 | } 41 | 42 | func (s *Sink) BatchPublish(ctx context.Context, envelopes []envelope.Envelope) error { 43 | for _, e := range envelopes { // FIXME -> shard 44 | contents, err := e.AsByte() 45 | if err != nil { 46 | log.Error().Err(err).Msg("🔴 could not marshal envelope") 47 | return err 48 | } 49 | _, err = s.jetstream.Publish(s.metadata.DefaultOutput, contents) 50 | if err != nil { 51 | log.Error().Err(err).Msg("🔴 could not publish valid envelope to jetstream") 52 | return err 53 | } 54 | } 55 | return nil 56 | } 57 | 58 | func (s *Sink) Close() { 59 | log.Debug().Msg("🟡 closing nats sink") 60 | s.conn.Close() 61 | } 62 | -------------------------------------------------------------------------------- /pkg/backend/postgresdb/dsn.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package postgresdb 6 | 7 | import ( 8 | "strconv" 9 | 10 | "github.com/silverton-io/buz/pkg/db" 11 | ) 12 | 13 | // GenerateDsn generates a dsn from the provided connection params 14 | func GenerateDsn(params db.ConnectionParams) string { 15 | // postgresql://[user[:password]@][netloc][:port][/dbname] 16 | p := strconv.FormatUint(uint64(params.Port), 10) 17 | return "postgresql://" + params.User + ":" + params.Pass + "@" + params.Host + ":" + p + "/" + params.Db 18 | } 19 | -------------------------------------------------------------------------------- /pkg/backend/postgresdb/dsn_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package postgresdb 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/silverton-io/buz/pkg/db" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestGeneratePostgresDsn(t *testing.T) { 15 | p := db.ConnectionParams{ 16 | Host: "host", 17 | Port: 5432, 18 | Db: "db", 19 | User: "usr", 20 | Pass: "pass", 21 | } 22 | 23 | expectedDsn := "postgresql://usr:pass@host:5432/db" 24 | dsn := GenerateDsn(p) 25 | 26 | assert.Equal(t, expectedDsn, dsn) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/backend/postgresdb/registry.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package postgresdb 6 | 7 | import ( 8 | "github.com/rs/zerolog/log" 9 | "github.com/silverton-io/buz/pkg/config" 10 | "github.com/silverton-io/buz/pkg/db" 11 | "gorm.io/driver/postgres" 12 | "gorm.io/gorm" 13 | ) 14 | 15 | type RegistryBackend struct { 16 | gormDb *gorm.DB 17 | registryTable string 18 | } 19 | 20 | func (b *RegistryBackend) Initialize(conf config.Backend) error { 21 | connParams := db.ConnectionParams{ 22 | Host: conf.DbHost, 23 | Port: conf.DbPort, 24 | Db: conf.DbName, 25 | User: conf.DbUser, 26 | Pass: conf.DbPass, 27 | } 28 | connString := GenerateDsn(connParams) 29 | gormDb, err := gorm.Open(postgres.Open(connString), &gorm.Config{}) 30 | if err != nil { 31 | log.Error().Err(err).Msg("🔴 could not open pg connection") 32 | return err 33 | } 34 | b.gormDb, b.registryTable = gormDb, conf.RegistryTable 35 | ensureErr := db.EnsureTable(b.gormDb, b.registryTable, db.RegistryTable{}) 36 | return ensureErr 37 | } 38 | 39 | func (b *RegistryBackend) GetRemote(schema string) (contents []byte, err error) { 40 | var s db.RegistryTable 41 | b.gormDb.Table(b.registryTable).Where("name = ?", schema).First(&s) 42 | err = b.gormDb.Error 43 | if err != nil { 44 | return nil, err 45 | } 46 | return s.Contents, nil 47 | } 48 | 49 | func (b *RegistryBackend) Close() { 50 | log.Info().Msg("🟢 closing postgres schema cache backend") 51 | } 52 | -------------------------------------------------------------------------------- /pkg/backend/pubnub/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package pubnub 6 | 7 | import ( 8 | "context" 9 | "net/url" 10 | "path" 11 | 12 | "github.com/rs/zerolog/log" 13 | "github.com/silverton-io/buz/pkg/backend/backendutils" 14 | "github.com/silverton-io/buz/pkg/config" 15 | "github.com/silverton-io/buz/pkg/envelope" 16 | "github.com/silverton-io/buz/pkg/request" 17 | ) 18 | 19 | const ( 20 | PUBNUB_PUBLISH_URL string = "ps.pndsn.com/publish" 21 | PUBNUB_HTTP_PROTOCOL string = "https" 22 | ) 23 | 24 | type Sink struct { 25 | metadata backendutils.SinkMetadata 26 | pubKey string 27 | subKey string 28 | input chan []envelope.Envelope 29 | shutdown chan int 30 | // store int // nolint: unused 31 | // callback string // nolint: unused 32 | } 33 | 34 | func (s *Sink) Metadata() backendutils.SinkMetadata { 35 | return s.metadata 36 | } 37 | 38 | func (s *Sink) Initialize(conf config.Sink) error { 39 | log.Debug().Msg("🟡 initializing pubnub sink") 40 | s.metadata = backendutils.NewSinkMetadataFromConfig(conf) 41 | s.pubKey, s.subKey = conf.PubnubPubKey, conf.PubnubSubKey 42 | s.input = make(chan []envelope.Envelope, 10000) 43 | s.shutdown = make(chan int, 1) 44 | return nil 45 | } 46 | 47 | func (s *Sink) StartWorker() error { 48 | err := backendutils.StartSinkWorker(s.input, s.shutdown, s) 49 | return err 50 | } 51 | 52 | func (s *Sink) Enqueue(envelopes []envelope.Envelope) error { 53 | log.Debug().Interface("metadata", s.Metadata()).Msg("enqueueing envelopes") 54 | s.input <- envelopes 55 | return nil 56 | } 57 | 58 | func (s *Sink) buildPublishUrl(channel string) *url.URL { 59 | p := path.Join(PUBNUB_PUBLISH_URL, s.pubKey, s.subKey, "1", channel, "0") 60 | p = "https://" + p + "?uuid=" + s.metadata.Id.String() 61 | u, err := url.Parse(p) 62 | if err != nil { 63 | log.Error().Err(err).Msg("🔴 could not parse publish url") 64 | } 65 | return u 66 | } 67 | 68 | func (s *Sink) Dequeue(ctx context.Context, envelopes []envelope.Envelope, output string) error { 69 | u := s.buildPublishUrl(output) 70 | _, err := request.PostEnvelopes(*u, envelopes, nil) 71 | return err 72 | } 73 | 74 | func (s *Sink) Shutdown() error { 75 | log.Debug().Interface("metadata", s.metadata).Msg("🟢 shutting down sink") 76 | s.shutdown <- 1 77 | return nil 78 | } 79 | -------------------------------------------------------------------------------- /pkg/backend/pulsar/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package sink 6 | -------------------------------------------------------------------------------- /pkg/backend/rabbitmq/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package sink 6 | -------------------------------------------------------------------------------- /pkg/backend/redshift/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package sink 6 | 7 | // TODO! Implement me. 8 | -------------------------------------------------------------------------------- /pkg/backend/s3/registry.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package s3 6 | 7 | import ( 8 | "context" 9 | "path/filepath" 10 | 11 | "github.com/aws/aws-sdk-go-v2/aws" 12 | awsconf "github.com/aws/aws-sdk-go-v2/config" 13 | "github.com/aws/aws-sdk-go-v2/feature/s3/manager" 14 | "github.com/aws/aws-sdk-go-v2/service/s3" 15 | "github.com/rs/zerolog/log" 16 | "github.com/silverton-io/buz/pkg/config" 17 | ) 18 | 19 | type RegistryBackend struct { 20 | bucket string 21 | path string 22 | client *s3.Client 23 | downloader *manager.Downloader 24 | } 25 | 26 | func (b *RegistryBackend) Initialize(conf config.Backend) error { 27 | log.Debug().Msg("🟡 initializing s3 schema cache backend") 28 | ctx := context.Background() 29 | cfg, err := awsconf.LoadDefaultConfig(ctx) 30 | if err != nil { 31 | log.Error().Err(err).Msg("🔴 could not load aws config") 32 | return err 33 | } 34 | client := s3.NewFromConfig(cfg) 35 | downloader := manager.NewDownloader(client) 36 | b.bucket, b.path, b.client, b.downloader = conf.Bucket, conf.Path, client, downloader 37 | return nil 38 | } 39 | 40 | func (b *RegistryBackend) GetRemote(schema string) (contents []byte, err error) { 41 | ctx := context.Background() 42 | var schemaLocation string 43 | if b.path == "/" { 44 | schemaLocation = schema 45 | } else { 46 | schemaLocation = filepath.Join(b.path, schema) 47 | } 48 | buffer := manager.NewWriteAtBuffer([]byte{}) 49 | log.Debug().Msg("🟡 getting file from s3 backend " + schemaLocation) 50 | _, err = b.downloader.Download(ctx, buffer, &s3.GetObjectInput{ 51 | Bucket: aws.String(b.bucket), 52 | Key: aws.String(schemaLocation), 53 | }) 54 | if err != nil { 55 | log.Error().Err(err).Msg("🔴 could not get file from s3: " + schemaLocation) 56 | return nil, err 57 | } 58 | return buffer.Bytes(), nil 59 | } 60 | 61 | func (b *RegistryBackend) Close() { 62 | log.Debug().Msg("🟡 closing s3 schema cache backend") 63 | // This is no-op 64 | } 65 | -------------------------------------------------------------------------------- /pkg/backend/splunk/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package splunk 6 | 7 | import ( 8 | "context" 9 | "net/http" 10 | "net/url" 11 | 12 | "github.com/rs/zerolog/log" 13 | "github.com/silverton-io/buz/pkg/backend/backendutils" 14 | "github.com/silverton-io/buz/pkg/config" 15 | "github.com/silverton-io/buz/pkg/envelope" 16 | "github.com/silverton-io/buz/pkg/request" 17 | ) 18 | 19 | type Sink struct { 20 | metadata backendutils.SinkMetadata 21 | url *url.URL 22 | token string 23 | input chan []envelope.Envelope 24 | shutdown chan int 25 | } 26 | 27 | func (s *Sink) Metadata() backendutils.SinkMetadata { 28 | return s.metadata 29 | } 30 | 31 | func (s *Sink) Initialize(conf config.Sink) error { 32 | url, err := url.Parse(conf.Url) 33 | if err != nil { 34 | log.Fatal().Err(err).Interface("metadata", s.Metadata()).Msg(conf.Url + " is not a valid url") 35 | } 36 | s.url = url 37 | s.token = conf.Token 38 | s.metadata = backendutils.NewSinkMetadataFromConfig(conf) 39 | s.input = make(chan []envelope.Envelope, 10000) 40 | s.shutdown = make(chan int, 1) 41 | return nil 42 | } 43 | 44 | func (s *Sink) StartWorker() error { 45 | err := backendutils.StartSinkWorker(s.input, s.shutdown, s) 46 | return err 47 | } 48 | 49 | func (s *Sink) Enqueue(envelopes []envelope.Envelope) error { 50 | log.Debug().Interface("metadata", s.Metadata()).Msg("enqueueing envelopes") 51 | s.input <- envelopes 52 | return nil 53 | } 54 | 55 | func (s *Sink) Dequeue(ctx context.Context, envelopes []envelope.Envelope, output string) error { 56 | log.Debug().Interface("metadata", s.Metadata()).Msg("dequeueing envelopes") 57 | splunkHeader := http.Header{ 58 | "Authorization": {"Splunk " + s.token}, 59 | } 60 | resp, err := request.PostEnvelopes(*s.url, envelopes, splunkHeader) 61 | if err != nil { 62 | log.Error().Interface("response", resp).Err(err).Msg("could not post envelopes") 63 | } 64 | return err 65 | } 66 | 67 | func (s *Sink) Shutdown() error { 68 | log.Debug().Interface("metadata", s.metadata).Msg("🟢 shutting down sink") 69 | s.shutdown <- 1 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /pkg/backend/stdout/sink_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package stdout 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/silverton-io/buz/pkg/config" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestConst(t *testing.T) { 15 | var testCases = []struct { 16 | name string 17 | have func(...interface{}) string 18 | want func(...interface{}) string 19 | }{ 20 | {"black", Black, Colorize("\033[1;30m%s\033[0m")}, 21 | {"red", Red, Colorize("\033[1;31m%s\033[0m")}, 22 | {"green", Green, Colorize("\033[1;32m%s\033[0m")}, 23 | {"yellow", Yellow, Colorize("\033[1;33m%s\033[0m")}, 24 | {"purple", Purple, Colorize("\033[1;34m%s\033[0m")}, 25 | {"magenta", Magenta, Colorize("\033[1;35m%s\033[0m")}, 26 | {"teal", Teal, Colorize("\033[1;36m%s\033[0m")}, 27 | {"white", White, Colorize("\033[1;37m%s\033[0m")}, 28 | } 29 | 30 | for _, tc := range testCases { 31 | t.Run(tc.name, func(t *testing.T) { 32 | have := tc.have("hello") 33 | want := tc.want("hello") 34 | assert.Equal(t, want, have) 35 | }) 36 | } 37 | } 38 | 39 | func TestColorize(t *testing.T) { 40 | s := "hello" 41 | want := "\x1b[1;36mhello\x1b[0m" 42 | have := Colorize("\033[1;36m%s\033[0m")(s) 43 | assert.Equal(t, want, have) 44 | } 45 | 46 | func TestStdoutSink(t *testing.T) { 47 | c := config.Sink{ 48 | Type: "stdout", 49 | } 50 | sink := Sink{} 51 | 52 | sink.Initialize(c) 53 | sink.Shutdown() 54 | } 55 | -------------------------------------------------------------------------------- /pkg/config/app.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type App struct { 8 | Version string `json:"version"` 9 | Name string `json:"name"` 10 | Env string `json:"env"` 11 | Port string `json:"port"` 12 | TrackerDomain string `json:"trackerDomain"` 13 | EnableConfigRoute bool `json:"enableConfigRoute"` 14 | Serverless bool `json:"serverless"` 15 | } 16 | -------------------------------------------------------------------------------- /pkg/config/app_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | import "testing" 8 | 9 | func TestApp(t *testing.T) {} 10 | -------------------------------------------------------------------------------- /pkg/config/cloudevents.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type Cloudevents struct { 8 | Enabled bool `json:"enabled"` 9 | Path string `json:"path"` 10 | } 11 | -------------------------------------------------------------------------------- /pkg/config/cloudevents_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | import "testing" 8 | 9 | func TestCloudevents(t *testing.T) {} 10 | -------------------------------------------------------------------------------- /pkg/config/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type Config struct { 8 | App `json:"app"` 9 | Middleware `json:"middleware"` 10 | Inputs `json:"inputs"` 11 | Registry `json:"registry"` 12 | Manifold `json:"manifold,omitempty"` 13 | Sinks []Sink `json:"sinks"` 14 | Squawkbox `json:"squawkBox"` 15 | Tele `json:"tele"` 16 | } 17 | -------------------------------------------------------------------------------- /pkg/config/config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | import "testing" 8 | 9 | func TestConfig(t *testing.T) {} 10 | -------------------------------------------------------------------------------- /pkg/config/generic_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | import "testing" 8 | 9 | func TestSelfDescribingConfig(t *testing.T) {} 10 | 11 | func TestGeneric(t *testing.T) {} 12 | -------------------------------------------------------------------------------- /pkg/config/input.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type Inputs struct { 8 | Snowplow `json:"snowplow"` 9 | Cloudevents `json:"cloudevents"` 10 | SelfDescribing `json:"selfDescribing"` 11 | Webhook `json:"webhook"` 12 | Pixel `json:"pixel"` 13 | } 14 | -------------------------------------------------------------------------------- /pkg/config/manifold.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type Manifold struct { 8 | } 9 | -------------------------------------------------------------------------------- /pkg/config/middleware.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type Middleware struct { 8 | Timeout `json:"timeout"` 9 | RateLimiter `json:"rateLimiter"` 10 | Identity `json:"identity"` 11 | Cors `json:"cors"` 12 | RequestLogger `json:"requestLogger"` 13 | Auth `json:"auth"` 14 | } 15 | 16 | type Timeout struct { 17 | Enabled bool `json:"enabled"` 18 | Ms int `json:"ms"` 19 | } 20 | 21 | type RateLimiter struct { 22 | Enabled bool `json:"enabled"` 23 | Period string `json:"period"` 24 | Limit int64 `json:"limit"` 25 | } 26 | 27 | type Identity struct { 28 | Cookie IdentityCookie `json:"cookie"` 29 | Fallback string `json:"fallback"` 30 | } 31 | 32 | type IdentityCookie struct { 33 | Enabled bool `json:"enabled"` 34 | Name string `json:"name"` 35 | Secure bool `json:"secure"` 36 | TtlDays int `json:"ttlDays"` 37 | Domain string `json:"domain"` 38 | Path string `json:"path"` 39 | SameSite string `json:"sameSite"` 40 | } 41 | 42 | type Cors struct { 43 | Enabled bool `json:"enabled"` 44 | AllowOrigin []string `json:"allowOrigin"` 45 | AllowCredentials bool `json:"allowCredentials"` 46 | AllowMethods []string `json:"allowMethods"` 47 | MaxAge int `json:"maxAge"` 48 | } 49 | 50 | type RequestLogger struct { 51 | Enabled bool `json:"enabled"` 52 | } 53 | 54 | type Auth struct { 55 | Enabled bool `json:"enabled"` 56 | Tokens []string `json:"tokens"` 57 | } 58 | -------------------------------------------------------------------------------- /pkg/config/middleware_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | import "testing" 8 | 9 | func TestMiddleware(t *testing.T) {} 10 | -------------------------------------------------------------------------------- /pkg/config/pixel.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type Pixel struct { 8 | Enabled bool `json:"enabled"` 9 | Path string `json:"path"` 10 | } 11 | -------------------------------------------------------------------------------- /pkg/config/registry.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type Http struct { 8 | Enabled bool `json:"enabled"` 9 | } 10 | 11 | type Purge struct { 12 | Enabled bool `json:"enabled"` 13 | } 14 | 15 | type Backend struct { 16 | Type string `json:"type"` 17 | Path string `json:"path"` 18 | // S3 and Gcs 19 | Bucket string `json:"bucket,omitempty"` 20 | // Gcs 21 | Region string `json:"region,omitempty"` 22 | // Http 23 | Host string `json:"host,omitempty"` 24 | // Db, general 25 | RegistryTable string `json:"registryTable,omitempty"` 26 | // Postgres Database 27 | DbHost string `json:"-"` 28 | DbPort uint16 `json:"-"` 29 | DbName string `json:"-"` 30 | DbUser string `json:"-"` 31 | DbPass string `json:"-"` 32 | // Mongodb 33 | MongoHosts []string `json:"mongoHosts,omitempty"` 34 | MongoPort string `json:"mongoDbPort,omitempty"` 35 | MongoDbName string `json:"mongoDbName,omitempty"` 36 | MongoUser string `json:"-"` 37 | MongoPass string `json:"-"` 38 | RegistryCollection string `json:"registryCollection,omitempty"` 39 | // Minio 40 | MinioEndpoint string `json:"minioEndpoint,omitempty"` 41 | AccessKeyId string `json:"accessKeyId,omitempty"` 42 | SecretAccessKey string `json:"secretAccessKey,omitempty"` 43 | } 44 | 45 | type Registry struct { 46 | Backend `json:"backend"` 47 | TtlSeconds int `json:"ttlSeconds"` 48 | MaxSizeBytes int `json:"maxSizeBytes"` 49 | Purge `json:"purge"` 50 | Http `json:"http"` 51 | } 52 | -------------------------------------------------------------------------------- /pkg/config/registry_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | -------------------------------------------------------------------------------- /pkg/config/relay.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type Relay struct { 8 | Enabled bool `json:"enabled"` 9 | Path string `json:"path"` 10 | } 11 | -------------------------------------------------------------------------------- /pkg/config/selfDescribing.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type SelfDescribingRootConfig struct { 8 | RootKey string `json:"rootKey"` 9 | } 10 | 11 | type SelfDescribingRootAndChildConfig struct { 12 | RootKey string `json:"rootKey"` 13 | SchemaKey string `json:"schemaKey"` 14 | DataKey string `json:"dataKey"` 15 | } 16 | 17 | type SelfDescribing struct { 18 | Enabled bool `json:"enabled"` 19 | Path string `json:"path"` 20 | Contexts SelfDescribingRootConfig `json:"contexts"` 21 | Payload SelfDescribingRootAndChildConfig `json:"payload"` 22 | } 23 | -------------------------------------------------------------------------------- /pkg/config/sink.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type Sink struct { 8 | Name string `json:"name"` 9 | Type string `json:"type"` 10 | DeliveryRequired bool `json:"deliveryRequired"` 11 | DefaultOutput string `json:"defaultOutput"` 12 | DeadletterOutput string `json:"deadletterOutput"` 13 | // GCP 14 | Project string `json:"project,omitempty"` 15 | Dataset string `json:"dataset,omitempty"` 16 | // Kafka 17 | Brokers []string `json:"kakfaBrokers,omitempty"` 18 | // Http / API 19 | Url string `json:"url"` 20 | ApiKey string `json:"-"` 21 | Token string `json:"-"` 22 | // Misc 23 | Region string `json:"-"` 24 | // Database 25 | Hosts []string `json:"-"` 26 | Port uint16 `json:"-"` 27 | Database string `json:"-"` 28 | User string `json:"-"` 29 | Password string `json:"-"` 30 | // Pubnub 31 | PubnubPubKey string `json:"pubnubPubKey,omitempty"` 32 | PubnubSubKey string `json:"pubnubSubKey,omitempty"` 33 | } 34 | -------------------------------------------------------------------------------- /pkg/config/sink_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | -------------------------------------------------------------------------------- /pkg/config/snowplow.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type Snowplow struct { 8 | Enabled bool `json:"enabled"` 9 | StandardRoutesEnabled bool `json:"standardRoutesEnabled"` 10 | OpenRedirectsEnabled bool `json:"openRedirectsEnabled"` 11 | GetPath string `json:"getPath"` 12 | PostPath string `json:"postPath"` 13 | RedirectPath string `json:"redirectPath"` 14 | } 15 | -------------------------------------------------------------------------------- /pkg/config/snowplow_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | -------------------------------------------------------------------------------- /pkg/config/squawkbox.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type Squawkbox struct { 8 | Enabled bool `json:"enabled"` 9 | } 10 | -------------------------------------------------------------------------------- /pkg/config/tele.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type Tele struct { 8 | Enabled bool `json:"enabled,omitempty"` 9 | Host string `json:"host,omitempty"` 10 | } 11 | -------------------------------------------------------------------------------- /pkg/config/tele_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | -------------------------------------------------------------------------------- /pkg/config/webhook.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | 7 | type Webhook struct { 8 | Enabled bool `json:"enabled"` 9 | Path string `json:"path"` 10 | } 11 | -------------------------------------------------------------------------------- /pkg/config/webhook_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package config 6 | -------------------------------------------------------------------------------- /pkg/constants/backend.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package constants 6 | 7 | const ( 8 | // Databases 9 | POSTGRES string = "postgres" 10 | MYSQL string = "mysql" 11 | MATERIALIZE string = "materialize" 12 | CLICKHOUSE string = "clickhouse" 13 | MONGODB string = "mongodb" 14 | ELASTICSEARCH string = "elasticsearch" 15 | TIMESCALE string = "timescale" 16 | BIGQUERY string = "bigquery" 17 | // Streams and Queues 18 | PUBSUB string = "pubsub" 19 | REDPANDA string = "redpanda" 20 | KAFKA string = "kafka" 21 | KINESIS string = "kinesis" 22 | KINESIS_FIREHOSE string = "kinesis-firehose" 23 | NATS string = "nats" 24 | NATS_JETSTREAM string = "nats-jetstream" 25 | EVENTBRIDGE string = "eventbridge" 26 | // Object Stores 27 | GCS string = "gcs" 28 | S3 string = "s3" 29 | MINIO string = "minio" 30 | // System 31 | STDOUT string = "stdout" 32 | BLACKHOLE string = "blackhole" 33 | FILE string = "file" 34 | // Web 35 | HTTP string = "http" 36 | HTTPS string = "https" 37 | // Third Party 38 | INDICATIVE string = "indicative" 39 | AMPLITUDE string = "amplitude" 40 | PUBNUB string = "pubnub" 41 | IGLU string = "iglu" 42 | SPLUNK string = "splunk" 43 | ) 44 | -------------------------------------------------------------------------------- /pkg/constants/buz.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package constants 6 | 7 | const ( 8 | BUZ_SCHEMA_PARAM string = "hps" 9 | BUZ_BASE64_ENCODED_PAYLOAD_PARAM string = "hpbp" 10 | UNKNOWN string = "unknown" 11 | BUZ_EVENTS string = "buz_events" 12 | BUZ_VALID_EVENTS string = "buz_valid_events" 13 | BUZ_INVALID_EVENTS string = "buz_invalid_events" 14 | BUZ string = "buz" 15 | BUZ_REDIRECT_TO_PARAM string = "rto" 16 | ) 17 | -------------------------------------------------------------------------------- /pkg/constants/buz_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package constants 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestBuzConstants(t *testing.T) { 14 | assert.Equal(t, "hps", BUZ_SCHEMA_PARAM) 15 | assert.Equal(t, "hpbp", BUZ_BASE64_ENCODED_PAYLOAD_PARAM) 16 | assert.Equal(t, "unknown", UNKNOWN) 17 | } 18 | -------------------------------------------------------------------------------- /pkg/constants/context.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package constants 6 | 7 | const IDENTITY string = "identity" 8 | -------------------------------------------------------------------------------- /pkg/constants/context_test.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestContextConstants(t *testing.T) { 10 | assert.Equal(t, "identity", IDENTITY) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/constants/routes.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | const ( 4 | STATS_PATH = "/stats" 5 | HEALTH_PATH = "/health" 6 | ROUTE_OVERVIEW_PATH = "/routes" 7 | CONFIG_OVERVIEW_PATH = "/config" 8 | SNOWPLOW_STANDARD_GET_PATH = "/i" 9 | SNOWPLOW_STANDARD_POST_PATH = "/com.snowplowanalytics.snowplow/tp2" 10 | SNOWPLOW_STANDARD_REDIRECT_PATH = "/r/tp2" 11 | ) 12 | -------------------------------------------------------------------------------- /pkg/constants/routes_test.go: -------------------------------------------------------------------------------- 1 | package constants 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestInternalRoutes(t *testing.T) { 10 | assert.Equal(t, "/stats", STATS_PATH) 11 | assert.Equal(t, "/health", HEALTH_PATH) 12 | assert.Equal(t, "/routes", ROUTE_OVERVIEW_PATH) 13 | assert.Equal(t, "/config", CONFIG_OVERVIEW_PATH) 14 | } 15 | 16 | func TestSnowplowRoutes(t *testing.T) { 17 | assert.Equal(t, "/i", SNOWPLOW_STANDARD_GET_PATH) 18 | assert.Equal(t, "/com.snowplowanalytics.snowplow/tp2", SNOWPLOW_STANDARD_POST_PATH) 19 | assert.Equal(t, "/r/tp2", SNOWPLOW_STANDARD_REDIRECT_PATH) 20 | } 21 | -------------------------------------------------------------------------------- /pkg/db/db.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package db 6 | 7 | type ConnectionParams struct { 8 | Host string 9 | Port uint16 10 | Db string 11 | User string 12 | Pass string 13 | } 14 | -------------------------------------------------------------------------------- /pkg/db/model.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package db 6 | 7 | import ( 8 | "time" 9 | 10 | "gorm.io/datatypes" 11 | ) 12 | 13 | type BasePKeylessModel struct { 14 | CreatedAt time.Time `json:"-" sql:"index"` 15 | UpdatedAt time.Time `json:"-" sql:"index"` 16 | DeletedAt *time.Time `json:"-" sql:"index"` 17 | } 18 | 19 | type RegistryTable struct { 20 | BasePKeylessModel 21 | Name string `json:"name" gorm:"index:idx_name"` 22 | Contents datatypes.JSON `json:"contents"` 23 | } 24 | -------------------------------------------------------------------------------- /pkg/db/util.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package db 6 | 7 | import ( 8 | "github.com/rs/zerolog/log" 9 | "gorm.io/gorm" 10 | ) 11 | 12 | // EnsureTable creates a table according to the specified model if it 13 | // does not already exist. 14 | func EnsureTable(gormDb *gorm.DB, tableName string, model interface{}) error { 15 | tblExists := gormDb.Migrator().HasTable(tableName) 16 | if !tblExists { 17 | log.Debug().Msg("🟡 " + tableName + " table doesn't exist - creating") 18 | err := gormDb.Table(tableName).AutoMigrate(model) 19 | if err != nil { 20 | log.Error().Err(err).Msg("🔴 could not create " + tableName + " table") 21 | } 22 | } else { 23 | log.Debug().Msg("🟡 " + tableName + " table already exists - not creating") 24 | } 25 | return nil 26 | } 27 | -------------------------------------------------------------------------------- /pkg/env/env.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package env 6 | 7 | const BUZ_CONFIG_PATH = "BUZ_CONFIG_PATH" 8 | const DEBUG = "DEBUG" 9 | -------------------------------------------------------------------------------- /pkg/env/env_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package env 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestConfigPath(t *testing.T) { 14 | assert.Equal(t, "BUZ_CONFIG_PATH", BUZ_CONFIG_PATH) 15 | } 16 | -------------------------------------------------------------------------------- /pkg/envelope/annotation.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package envelope 6 | 7 | // Reserving this for future use. 8 | type Annotations struct{} 9 | -------------------------------------------------------------------------------- /pkg/envelope/context.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package envelope 6 | 7 | import ( 8 | "database/sql/driver" 9 | "encoding/json" 10 | 11 | "github.com/gin-gonic/gin" 12 | "github.com/silverton-io/buz/pkg/util" 13 | ) 14 | 15 | const HTTP_HEADERS_CONTEXT string = "io.silverton/buz/internal/contexts/httpHeaders/v1.0.json" 16 | 17 | type Contexts map[string]interface{} 18 | 19 | func (c Contexts) Value() (driver.Value, error) { 20 | b, err := json.Marshal(c) 21 | return string(b), err 22 | } 23 | 24 | func (c Contexts) Scan(input interface{}) error { 25 | return json.Unmarshal(input.([]byte), &c) 26 | } 27 | 28 | func BuildContextsFromRequest(c *gin.Context) Contexts { 29 | headers := util.HttpHeadersToMap(c) 30 | context := map[string]interface{}{ 31 | HTTP_HEADERS_CONTEXT: headers, 32 | } 33 | return context 34 | } 35 | 36 | func (c *Contexts) AsByte() ([]byte, error) { 37 | return json.Marshal(c) 38 | } 39 | -------------------------------------------------------------------------------- /pkg/envelope/envelope_test.go: -------------------------------------------------------------------------------- 1 | package envelope 2 | -------------------------------------------------------------------------------- /pkg/envelope/event.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package envelope 6 | 7 | import ( 8 | "database/sql/driver" 9 | "encoding/json" 10 | ) 11 | 12 | type Event interface { 13 | SchemaName() *string 14 | PayloadAsByte() ([]byte, error) 15 | Value() (driver.Value, error) 16 | Scan(input interface{}) error 17 | } 18 | 19 | type Payload map[string]interface{} 20 | 21 | func (p Payload) Value() (driver.Value, error) { 22 | b, err := json.Marshal(p) 23 | return string(b), err 24 | } 25 | 26 | func (p Payload) Scan(input interface{}) error { 27 | return json.Unmarshal(input.([]byte), &p) 28 | } 29 | 30 | func (p *Payload) AsByte() ([]byte, error) { 31 | return json.Marshal(p) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/envelope/event_test.go: -------------------------------------------------------------------------------- 1 | package envelope 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestPayloadAsByte(t *testing.T) { 11 | raw := map[string]interface{}{ 12 | "something": "here", 13 | } 14 | p := Payload{ 15 | "something": "here", 16 | } 17 | b, err := p.AsByte() 18 | raw_bytes, _ := json.Marshal(raw) 19 | assert.Equal(t, raw_bytes, b) 20 | assert.Equal(t, nil, err) 21 | } 22 | -------------------------------------------------------------------------------- /pkg/envelope/output.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package envelope 6 | 7 | import ( 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | type OutputLocation struct { 13 | Path string 14 | DatabaseFqn string 15 | Namespace string 16 | } 17 | 18 | // Build an OutputLocation using Envelope contents 19 | func NewOutputLocationFromEnvelope(e *Envelope) OutputLocation { 20 | // Example output path: isValid=true/vendor=io.silverton/namespace=gettingStarted.example/version=1.1/year=2023/month=3/day=10/ 21 | year, month, day := e.BuzTimestamp.Date() 22 | datePath := "/year=" + strconv.Itoa(year) + "/month=" + strconv.Itoa(int(month)) + "/day=" + strconv.Itoa(day) + "/" 23 | outputPath := "isValid=" + strconv.FormatBool(e.IsValid) + "/vendor=" + e.Vendor + "/namespace=" + e.Namespace + "/version=" + e.Version + datePath 24 | // Example output database fqn: io_silverton.gettingstarted_example_1 25 | outputDbFqn := strings.Replace(e.Vendor, ".", "_", -1) + "." + strings.Replace(e.Namespace, ".", "_", -1) + "_" + strings.Split(e.Version, ".")[0] 26 | // Example output namespace: io.silverton.gettingStarted.example.1.1 27 | outputNamespace := strings.Join([]string{e.Vendor, e.Namespace, e.Version}, ".") 28 | return OutputLocation{ 29 | Path: outputPath, 30 | DatabaseFqn: outputDbFqn, 31 | Namespace: outputNamespace, 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /pkg/envelope/output_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package envelope 6 | 7 | import ( 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestNewOutputLocationFromEnvelope(t *testing.T) { 15 | d, _ := time.Parse("2006-01-02", "2023-03-10") 16 | fakeEnvelope := Envelope{ 17 | IsValid: true, 18 | Vendor: "com.vendor", 19 | Namespace: "customer.activity", 20 | Version: "1.1", 21 | BuzTimestamp: d, 22 | } 23 | expectedPath := "isValid=true/vendor=com.vendor/namespace=customer.activity/version=1.1/year=2023/month=3/day=10/" 24 | expectedDatabaseFqn := "com_vendor.customer_activity_1" 25 | expectedNamespace := "com.vendor.customer.activity.1.1" 26 | 27 | outputLocation := NewOutputLocationFromEnvelope(&fakeEnvelope) 28 | assert.Equal(t, expectedPath, outputLocation.Path) 29 | assert.Equal(t, expectedDatabaseFqn, outputLocation.DatabaseFqn) 30 | assert.Equal(t, expectedNamespace, outputLocation.Namespace) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/envelope/selfDescribing.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package envelope 6 | 7 | import ( 8 | "database/sql/driver" 9 | "encoding/json" 10 | "strings" 11 | ) 12 | 13 | func stripColonSeparatedPrefix(schema string) string { 14 | colonIdx := strings.Index(schema, ":") 15 | if colonIdx == -1 { 16 | return schema 17 | } else { 18 | return schema[colonIdx+1:] 19 | } 20 | } 21 | 22 | type SelfDescribingEvent struct { 23 | Contexts `json:"contexts"` 24 | Payload SelfDescribingPayload `json:"payload"` 25 | } 26 | 27 | type SelfDescribingPayload struct { 28 | Schema string `json:"schema"` 29 | Data map[string]interface{} `json:"data"` 30 | } 31 | 32 | func (e SelfDescribingPayload) SchemaName() *string { 33 | name := stripColonSeparatedPrefix(e.Schema) 34 | return &name 35 | } 36 | 37 | func (e SelfDescribingPayload) PayloadAsByte() ([]byte, error) { 38 | payloadBytes, err := json.Marshal(e.Data) 39 | if err != nil { 40 | return nil, err 41 | } 42 | return payloadBytes, nil 43 | } 44 | 45 | func (e SelfDescribingPayload) AsByte() ([]byte, error) { 46 | eventBytes, err := json.Marshal(e) 47 | if err != nil { 48 | return nil, err 49 | } 50 | return eventBytes, nil 51 | } 52 | 53 | func (e SelfDescribingPayload) AsMap() (map[string]interface{}, error) { 54 | var m map[string]interface{} 55 | b, err := e.AsByte() 56 | if err != nil { 57 | return nil, err 58 | } 59 | err = json.Unmarshal(b, &m) 60 | if err != nil { 61 | return nil, err 62 | } 63 | return m, nil 64 | } 65 | 66 | func (e SelfDescribingPayload) Value() (driver.Value, error) { 67 | b, err := json.Marshal(e) 68 | return string(b), err 69 | } 70 | 71 | func (e SelfDescribingPayload) Scan(input interface{}) error { 72 | return json.Unmarshal(input.([]byte), &e) 73 | } 74 | 75 | type SelfDescribingContext SelfDescribingPayload 76 | -------------------------------------------------------------------------------- /pkg/envelope/selfDescribing_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package envelope 6 | 7 | import ( 8 | "testing" 9 | ) 10 | 11 | func TestSelfDescribingEnvelope(t *testing.T) {} 12 | 13 | func TestSelfDescribingPayload(t *testing.T) {} 14 | 15 | func TestSelfDescribingContext(t *testing.T) {} 16 | -------------------------------------------------------------------------------- /pkg/envelope/validation.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package envelope 6 | 7 | import ( 8 | "database/sql/driver" 9 | "encoding/json" 10 | ) 11 | 12 | type Validation struct { 13 | IsValid *bool `json:"isValid"` 14 | Error *ValidationError `json:"error,omitempty"` 15 | } 16 | 17 | func (e Validation) Value() (driver.Value, error) { 18 | b, err := json.Marshal(e) 19 | return string(b), err 20 | } 21 | 22 | func (e Validation) Scan(input interface{}) error { 23 | return json.Unmarshal(input.([]byte), &e) 24 | } 25 | 26 | type PayloadValidationError struct { 27 | Field string `json:"field,omitempty"` 28 | Description string `json:"description,omitempty"` 29 | ErrorType string `json:"errorType,omitempty"` 30 | } 31 | 32 | type ValidationError struct { 33 | ErrorType *string `json:"errorType,omitempty"` 34 | ErrorResolution *string `json:"errorResolution,omitempty"` 35 | Errors []PayloadValidationError `json:"payloadValidationErrors,omitempty"` 36 | } 37 | 38 | func (e *ValidationError) Value() (driver.Value, error) { 39 | b, err := json.Marshal(e) 40 | return string(b), err 41 | } 42 | 43 | func (e *ValidationError) Scan(input interface{}) error { 44 | return json.Unmarshal(input.([]byte), e) 45 | } 46 | 47 | func (e *ValidationError) AsByte() ([]byte, error) { 48 | return json.Marshal(e) 49 | } 50 | -------------------------------------------------------------------------------- /pkg/handler/buz.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package handler 6 | 7 | import ( 8 | "github.com/gin-gonic/gin" 9 | ) 10 | 11 | func BuzHandler() gin.HandlerFunc { 12 | fn := func(c *gin.Context) { 13 | c.String(200, "🐝") 14 | } 15 | return gin.HandlerFunc(fn) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/handler/buz_test.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | "testing" 7 | 8 | testutil "github.com/silverton-io/buz/pkg/testUtil" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestBuzHandler(t *testing.T) { 13 | srv := testutil.BuildTestServer(BuzHandler()) 14 | 15 | resp, _ := http.Get(srv.URL + testutil.URL) 16 | if resp.StatusCode != http.StatusOK { 17 | t.Fatalf(`got status code %v, want %v`, resp.StatusCode, http.StatusOK) 18 | } 19 | defer resp.Body.Close() 20 | b, _ := io.ReadAll(resp.Body) 21 | assert.Equal(t, http.StatusOK, resp.StatusCode) 22 | assert.Equal(t, "🐝", string(b)) 23 | } 24 | -------------------------------------------------------------------------------- /pkg/handler/health.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package handler 6 | 7 | import ( 8 | "net/http" 9 | 10 | "github.com/gin-gonic/gin" 11 | "github.com/silverton-io/buz/pkg/response" 12 | ) 13 | 14 | func HealthcheckHandler(c *gin.Context) { 15 | c.JSON(http.StatusOK, response.Ok) 16 | } 17 | -------------------------------------------------------------------------------- /pkg/handler/health_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package handler 6 | 7 | import ( 8 | "encoding/json" 9 | "io" 10 | "net/http" 11 | "reflect" 12 | "testing" 13 | 14 | "github.com/silverton-io/buz/pkg/response" 15 | testutil "github.com/silverton-io/buz/pkg/testUtil" 16 | ) 17 | 18 | func TestHealthcheckHandler(t *testing.T) { 19 | rec, c, _ := testutil.BuildRecordedEngine() 20 | 21 | HealthcheckHandler(c) 22 | 23 | resp := rec.Result() 24 | 25 | defer resp.Body.Close() 26 | if resp.StatusCode != http.StatusOK { 27 | t.Fatalf(`HealthcheckHandler returned status code %v, want %v`, resp.StatusCode, http.StatusOK) 28 | } 29 | b, _ := io.ReadAll(resp.Body) 30 | marshaledB, _ := json.Marshal(response.Ok) 31 | equiv := reflect.DeepEqual(b, marshaledB) 32 | if !equiv { 33 | t.Fatalf(`HealthcheckHandler returned body %v, want %v`, b, marshaledB) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /pkg/handler/overview_test.go: -------------------------------------------------------------------------------- 1 | package handler 2 | -------------------------------------------------------------------------------- /pkg/handler/stats.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package handler 6 | 7 | import ( 8 | "github.com/gin-gonic/gin" 9 | "github.com/silverton-io/buz/pkg/meta" 10 | "github.com/silverton-io/buz/pkg/stats" 11 | ) 12 | 13 | type StatsResponse struct { 14 | CollectorMeta *meta.CollectorMeta `json:"collectorMeta"` 15 | Stats *stats.ProtocolStats `json:"stats"` 16 | } 17 | 18 | func StatsHandler(m *meta.CollectorMeta) gin.HandlerFunc { 19 | fn := func(c *gin.Context) { 20 | resp := StatsResponse{ 21 | CollectorMeta: m, 22 | // Stats: s, 23 | } 24 | c.JSON(200, resp) 25 | } 26 | return gin.HandlerFunc(fn) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/handler/stats_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package handler 6 | 7 | import ( 8 | "encoding/json" 9 | "io" 10 | "net/http" 11 | "testing" 12 | "time" 13 | 14 | "github.com/google/uuid" 15 | "github.com/silverton-io/buz/pkg/meta" 16 | testutil "github.com/silverton-io/buz/pkg/testUtil" 17 | "github.com/stretchr/testify/assert" 18 | ) 19 | 20 | func TestStatsHandler(t *testing.T) { 21 | u := uuid.New() 22 | now := time.Now().UTC() 23 | m := meta.CollectorMeta{ 24 | Version: "1.0.x", 25 | InstanceId: u, 26 | StartTime: now, 27 | TrackerDomain: "somewhere.net", 28 | CookieDomain: "somewhere.io", 29 | } 30 | 31 | rec, c, _ := testutil.BuildRecordedEngine() 32 | 33 | handler := StatsHandler(&m) 34 | 35 | handler(c) 36 | 37 | resp := rec.Result() 38 | defer resp.Body.Close() 39 | assert.Equal(t, http.StatusOK, resp.StatusCode) 40 | b, err := io.ReadAll(resp.Body) 41 | if err != nil { 42 | t.Fatalf("Could not read response: %v", err) 43 | } 44 | expectedResponse := StatsResponse{ 45 | CollectorMeta: &m, 46 | // Stats: &s, 47 | } 48 | expected, err := json.Marshal(expectedResponse) 49 | if err != nil { 50 | t.Fatalf(`Could not marshal expected response`) 51 | } 52 | assert.Equal(t, expected, b) 53 | } 54 | -------------------------------------------------------------------------------- /pkg/input/input.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | package input 5 | 6 | import ( 7 | "github.com/gin-gonic/gin" 8 | "github.com/silverton-io/buz/pkg/config" 9 | "github.com/silverton-io/buz/pkg/envelope" 10 | "github.com/silverton-io/buz/pkg/manifold" 11 | "github.com/silverton-io/buz/pkg/meta" 12 | ) 13 | 14 | type Input interface { 15 | Initialize(routerGroup *gin.RouterGroup, manifold *manifold.Manifold, conf *config.Config, metadata *meta.CollectorMeta) error 16 | Handler(m manifold.Manifold, conf config.Config, metadata *meta.CollectorMeta) gin.HandlerFunc 17 | SquawkboxHandler(m manifold.Manifold, conf config.Config, metadata *meta.CollectorMeta) gin.HandlerFunc 18 | EnvelopeBuilder(c *gin.Context, conf *config.Config, metadata *meta.CollectorMeta) []envelope.Envelope 19 | // Routes() []string 20 | // Auth() interface{} 21 | } 22 | -------------------------------------------------------------------------------- /pkg/inputApp/inputApp.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package inputapp 6 | -------------------------------------------------------------------------------- /pkg/manifold/kernelBufferManifold.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package manifold 6 | 7 | type KernelBufferManifold struct { 8 | // IMPLEMENT ME 9 | } 10 | -------------------------------------------------------------------------------- /pkg/manifold/manifold.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package manifold 6 | 7 | import ( 8 | "github.com/silverton-io/buz/pkg/backend/backendutils" 9 | "github.com/silverton-io/buz/pkg/config" 10 | "github.com/silverton-io/buz/pkg/envelope" 11 | "github.com/silverton-io/buz/pkg/meta" 12 | "github.com/silverton-io/buz/pkg/registry" 13 | ) 14 | 15 | type Manifold interface { 16 | Initialize(registry *registry.Registry, sinks *[]backendutils.Sink, conf *config.Config, metadata *meta.CollectorMeta) error 17 | Enqueue(envelopes []envelope.Envelope) error 18 | GetRegistry() *registry.Registry 19 | Shutdown() error 20 | } 21 | -------------------------------------------------------------------------------- /pkg/manifold/simpleManifold.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package manifold 6 | 7 | import ( 8 | "github.com/rs/zerolog/log" 9 | "github.com/silverton-io/buz/pkg/annotator" 10 | "github.com/silverton-io/buz/pkg/backend/backendutils" 11 | "github.com/silverton-io/buz/pkg/config" 12 | "github.com/silverton-io/buz/pkg/envelope" 13 | "github.com/silverton-io/buz/pkg/meta" 14 | "github.com/silverton-io/buz/pkg/registry" 15 | ) 16 | 17 | // A stupid-simple manifold with strict guarantees. 18 | // This manifold requires buffering at the client level for substantial event volumes. 19 | // Otherwise it will probably overload the configured sink(s). 20 | type SimpleManifold struct { 21 | registry *registry.Registry 22 | sinks *[]backendutils.Sink 23 | conf *config.Config 24 | collectorMetdata *meta.CollectorMeta 25 | } 26 | 27 | func (m *SimpleManifold) Initialize(registry *registry.Registry, sinks *[]backendutils.Sink, conf *config.Config, metadata *meta.CollectorMeta) error { 28 | m.registry = registry 29 | m.sinks = sinks 30 | m.conf = conf 31 | m.collectorMetdata = metadata 32 | return nil 33 | } 34 | 35 | func (m *SimpleManifold) Enqueue(envelopes []envelope.Envelope) error { 36 | annotatedEnvelopes := annotator.Annotate(envelopes, m.registry) 37 | for _, sink := range *m.sinks { 38 | meta := sink.Metadata() 39 | log.Debug().Interface("metadata", meta).Msg("🟡 enqueueing envelopes to sink") 40 | err := sink.Enqueue(annotatedEnvelopes) 41 | if err != nil { 42 | log.Error().Err(err).Interface("metadata", sink.Metadata()).Msg("failed to enqueue envelopes to sink") 43 | } 44 | } 45 | return nil 46 | } 47 | 48 | func (m *SimpleManifold) GetRegistry() *registry.Registry { 49 | return m.registry 50 | } 51 | 52 | func (m *SimpleManifold) Shutdown() error { 53 | log.Info().Msg("shutting down simple manifold") 54 | log.Info().Msg("manifold shut down") 55 | return nil 56 | } 57 | -------------------------------------------------------------------------------- /pkg/meta/collector.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package meta 6 | 7 | import ( 8 | "time" 9 | 10 | "github.com/google/uuid" 11 | "github.com/silverton-io/buz/pkg/config" 12 | ) 13 | 14 | type CollectorMeta struct { 15 | Version string `json:"version"` 16 | Name string `json:"name"` 17 | InstanceId uuid.UUID `json:"instanceId"` 18 | StartTime time.Time `json:"startTime"` 19 | TrackerDomain string `json:"trackerDomain"` 20 | CookieDomain string `json:"cookieDomain"` 21 | } 22 | 23 | func (m *CollectorMeta) Elapsed() float64 { 24 | return time.Since(m.StartTime).Seconds() 25 | } 26 | 27 | func BuildCollectorMeta(version string, conf *config.Config) *CollectorMeta { 28 | instanceId := uuid.New() 29 | m := CollectorMeta{ 30 | Version: version, 31 | Name: conf.App.Name, 32 | InstanceId: instanceId, 33 | StartTime: time.Now().UTC(), 34 | TrackerDomain: conf.App.TrackerDomain, 35 | CookieDomain: conf.Cookie.Domain, 36 | } 37 | return &m 38 | } 39 | -------------------------------------------------------------------------------- /pkg/middleware/auth.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package middleware 6 | 7 | import ( 8 | "net/http" 9 | "strings" 10 | 11 | "github.com/gin-gonic/gin" 12 | "github.com/silverton-io/buz/pkg/config" 13 | "github.com/silverton-io/buz/pkg/response" 14 | ) 15 | 16 | const ( 17 | BASIC = "Basic" 18 | BEARER = "Bearer" 19 | ) 20 | 21 | type authHeader struct { 22 | Token string `header:"Authorization"` 23 | } 24 | 25 | func tokenIsValid(token string, conf config.Auth) bool { 26 | for _, validToken := range conf.Tokens { 27 | if token == validToken { 28 | return true 29 | } 30 | } 31 | return false 32 | } 33 | 34 | // The simplest-possible way to lock down routes 35 | func Auth(conf config.Auth) gin.HandlerFunc { 36 | return func(c *gin.Context) { 37 | h := authHeader{} 38 | if err := c.ShouldBindHeader(&h); err != nil { 39 | // Can't bind Authorization header 40 | c.JSON(http.StatusUnauthorized, response.MissingAuthHeader) 41 | c.Abort() 42 | return 43 | } 44 | if h.Token == "" { 45 | // No Authorization header present 46 | c.JSON(http.StatusUnauthorized, response.MissingAuthHeader) 47 | c.Abort() 48 | return 49 | } 50 | tokenParts := strings.Split(h.Token, " ") 51 | if len(tokenParts) < 2 { 52 | // Header present but missing scheme or token 53 | c.JSON(http.StatusUnauthorized, response.MissingAuthSchemeOrToken) 54 | c.Abort() 55 | return 56 | } 57 | scheme := tokenParts[0] 58 | token := tokenParts[1] 59 | if scheme != BEARER && scheme != BASIC { 60 | // Auth scheme isn't supported 61 | c.JSON(http.StatusUnauthorized, response.InvalidAuthScheme) 62 | c.Abort() 63 | return 64 | } 65 | isValid := tokenIsValid(token, conf) 66 | if !isValid { 67 | // Invalid token 68 | c.JSON(http.StatusUnauthorized, response.InvalidAuthToken) 69 | c.Abort() 70 | return 71 | } 72 | c.Next() 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /pkg/middleware/auth_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package middleware 6 | -------------------------------------------------------------------------------- /pkg/middleware/cors.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package middleware 6 | 7 | import ( 8 | "net/http" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/gin-gonic/gin" 13 | "github.com/silverton-io/buz/pkg/config" 14 | ) 15 | 16 | func CORS(conf config.Cors) gin.HandlerFunc { 17 | return func(c *gin.Context) { 18 | origin := c.Request.Header.Get("Origin") 19 | 20 | for _, domain := range conf.AllowOrigin { 21 | if domain == "*" { 22 | c.Header("Access-Control-Allow-Origin", origin) 23 | break 24 | } 25 | if strings.HasSuffix(origin, domain) { 26 | c.Header("Access-Control-Allow-Origin", origin) 27 | break 28 | } 29 | } 30 | 31 | if strconv.FormatBool(conf.AllowCredentials) == "true" { 32 | c.Header("Access-Control-Allow-Credentials", strconv.FormatBool(conf.AllowCredentials)) 33 | } 34 | 35 | c.Header("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With, Set-Cookie, Cookie") 36 | c.Header("Access-Control-Allow-Methods", strings.Join(conf.AllowMethods, ", ")) 37 | c.Header("Access-Control-Max-Age", strconv.Itoa(conf.MaxAge)) 38 | 39 | if c.Request.Method == "OPTIONS" { 40 | c.AbortWithStatus(http.StatusNoContent) 41 | return 42 | } 43 | 44 | c.Next() 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /pkg/middleware/cors_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package middleware 6 | 7 | import ( 8 | "net/http" 9 | "net/http/httptest" 10 | "testing" 11 | 12 | "github.com/gin-gonic/gin" 13 | "github.com/silverton-io/buz/pkg/config" 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | func TestCors(t *testing.T) { 18 | u := "/test" 19 | conf := config.Cors{ 20 | Enabled: true, 21 | AllowOrigin: []string{"http://allowed-origin.com"}, 22 | AllowCredentials: true, 23 | AllowMethods: []string{"GET", "OPTIONS"}, 24 | MaxAge: 86400, 25 | } 26 | r := gin.New() 27 | r.Use(CORS(conf)) 28 | r.GET(u, testHandler) 29 | ts := httptest.NewServer(r) 30 | defer ts.Close() 31 | 32 | t.Run("preflight success", func(t *testing.T) { 33 | var client = &http.Client{} 34 | req, _ := http.NewRequest(http.MethodOptions, ts.URL+u, nil) 35 | req.Header.Set("Origin", "http://allowed-origin.com") 36 | resp, _ := client.Do(req) 37 | 38 | assert.Equal(t, []string{"true"}, resp.Header["Access-Control-Allow-Credentials"]) 39 | assert.Equal(t, []string{"GET, OPTIONS"}, resp.Header["Access-Control-Allow-Methods"]) 40 | assert.Equal(t, []string{"http://allowed-origin.com"}, resp.Header["Access-Control-Allow-Origin"]) 41 | assert.Equal(t, []string{"86400"}, resp.Header["Access-Control-Max-Age"]) 42 | assert.Equal(t, http.StatusNoContent, resp.StatusCode) 43 | }) 44 | 45 | t.Run("preflight fail", func(t *testing.T) { 46 | var client = &http.Client{} 47 | req, _ := http.NewRequest(http.MethodOptions, ts.URL+u, nil) 48 | req.Header.Set("Origin", "http://not-allowed-origin.com") 49 | resp, _ := client.Do(req) 50 | 51 | assert.Equal(t, []string{"true"}, resp.Header["Access-Control-Allow-Credentials"]) 52 | assert.Equal(t, []string{"GET, OPTIONS"}, resp.Header["Access-Control-Allow-Methods"]) 53 | assert.Equal(t, []string([]string(nil)), resp.Header["Access-Control-Allow-Origin"]) 54 | assert.Equal(t, []string{"86400"}, resp.Header["Access-Control-Max-Age"]) 55 | assert.Equal(t, http.StatusNoContent, resp.StatusCode) 56 | }) 57 | 58 | t.Run("get", func(t *testing.T) { 59 | resp, _ := http.Get(ts.URL + u) 60 | assert.Equal(t, http.StatusOK, resp.StatusCode) 61 | }) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/middleware/identity.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package middleware 6 | 7 | import ( 8 | "net/http" 9 | 10 | "github.com/gin-gonic/gin" 11 | "github.com/google/uuid" 12 | "github.com/silverton-io/buz/pkg/config" 13 | "github.com/silverton-io/buz/pkg/constants" 14 | ) 15 | 16 | func Identity(conf config.Identity) gin.HandlerFunc { 17 | return func(c *gin.Context) { 18 | identityCookieValue, _ := c.Cookie(conf.Cookie.Name) 19 | switch conf.Cookie.SameSite { 20 | case "None": 21 | c.SetSameSite(http.SameSiteNoneMode) 22 | case "Lax": 23 | c.SetSameSite(http.SameSiteLaxMode) 24 | case "Strict": 25 | c.SetSameSite(http.SameSiteStrictMode) 26 | } 27 | if identityCookieValue != "" { 28 | c.SetCookie( 29 | conf.Cookie.Name, 30 | identityCookieValue, 31 | 60*60*24*conf.Cookie.TtlDays, 32 | conf.Cookie.Path, 33 | conf.Cookie.Domain, 34 | conf.Cookie.Secure, 35 | false, 36 | ) 37 | } else { 38 | identityCookieValue = uuid.New().String() 39 | c.SetCookie( 40 | conf.Cookie.Name, 41 | identityCookieValue, 42 | 60*60*24*conf.Cookie.TtlDays, 43 | conf.Cookie.Path, 44 | conf.Cookie.Domain, 45 | conf.Cookie.Secure, 46 | false, 47 | ) 48 | } 49 | c.Set(constants.IDENTITY, identityCookieValue) 50 | c.Next() 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /pkg/middleware/jsonLogger.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package middleware 6 | 7 | import ( 8 | "bytes" 9 | "encoding/json" 10 | "io" 11 | "strings" 12 | "time" 13 | 14 | "github.com/gin-gonic/gin" 15 | "github.com/rs/zerolog/log" 16 | "github.com/silverton-io/buz/pkg/util" 17 | ) 18 | 19 | type request struct { 20 | ResponseCode int `json:"responseCode"` 21 | RequestDuration time.Duration `json:"requestDuration"` 22 | RequestDurationForHumans string `json:"requestDurationForHumans"` 23 | ClientIp string `json:"clientIp"` 24 | RequestMethod string `json:"requestMethod"` 25 | RequestUri string `json:"requestUri"` 26 | Body interface{} `json:"body"` 27 | } 28 | 29 | func getIp(c *gin.Context) string { 30 | ip := c.Request.Header.Get("X-Forwarded-For") 31 | if len(ip) == 0 { 32 | ip = c.Request.Header.Get("X-Real-IP") 33 | } 34 | if len(ip) == 0 { 35 | ip = c.Request.RemoteAddr 36 | } 37 | if strings.Contains(ip, ",") { 38 | ip = strings.Split(ip, ",")[0] 39 | } 40 | return ip 41 | } 42 | 43 | func RequestLogger() gin.HandlerFunc { 44 | return func(c *gin.Context) { 45 | start := time.Now().UTC() 46 | end := time.Now().UTC() 47 | duration := util.GetDuration(start, end) 48 | buf, _ := io.ReadAll(c.Request.Body) 49 | r1 := io.NopCloser(bytes.NewBuffer(buf)) 50 | r2 := io.NopCloser(bytes.NewBuffer(buf)) 51 | reqBody, err := io.ReadAll(r1) 52 | c.Request.Body = r2 53 | c.Next() 54 | if err != nil { 55 | log.Error().Err(err).Msg("could not read request body") 56 | } 57 | 58 | var b interface{} 59 | if string(reqBody) != "" { 60 | err = json.Unmarshal(reqBody, &b) 61 | 62 | if err != nil { 63 | log.Debug().Err(err).Interface("body", reqBody).Msg("could not unmarshal request body") 64 | } 65 | } 66 | 67 | r := request{ 68 | ResponseCode: c.Writer.Status(), 69 | RequestDuration: duration, 70 | RequestDurationForHumans: duration.String(), 71 | ClientIp: getIp(c), 72 | RequestMethod: c.Request.Method, 73 | RequestUri: c.Request.RequestURI, 74 | Body: b, 75 | } 76 | log.Info().Interface("request", r).Msg("🟢") 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /pkg/middleware/jsonLogger_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package middleware 6 | -------------------------------------------------------------------------------- /pkg/middleware/rateLimiter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package middleware 6 | 7 | import ( 8 | "net/http" 9 | "time" 10 | 11 | "github.com/gin-gonic/gin" 12 | "github.com/rs/zerolog/log" 13 | "github.com/silverton-io/buz/pkg/config" 14 | "github.com/silverton-io/buz/pkg/response" 15 | limiter "github.com/ulule/limiter/v3" 16 | ginMiddleware "github.com/ulule/limiter/v3/drivers/middleware/gin" 17 | "github.com/ulule/limiter/v3/drivers/store/memory" 18 | ) 19 | 20 | func getDurationFromString(period string) time.Duration { 21 | switch period { 22 | case "MS": 23 | return 1 * time.Millisecond 24 | case "S": 25 | return 1 * time.Second 26 | case "M": 27 | return 1 * time.Minute 28 | case "H": 29 | return 1 * time.Hour 30 | case "D": 31 | return 24 * time.Hour 32 | default: 33 | return 1 * time.Second 34 | } 35 | } 36 | 37 | func onLimitReachedHandler(c *gin.Context) { 38 | log.Trace().Stack().Msg("limit reached - throttled request") 39 | c.JSON(http.StatusTooManyRequests, response.RateLimitExceeded) 40 | } 41 | 42 | func BuildRateLimiter(conf config.RateLimiter) *limiter.Limiter { 43 | period := getDurationFromString(conf.Period) 44 | rate := limiter.Rate{ 45 | Period: period, 46 | Limit: conf.Limit, 47 | } 48 | store := memory.NewStore() 49 | l := limiter.New(store, rate) 50 | return l 51 | } 52 | 53 | func BuildRateLimiterMiddleware(l *limiter.Limiter) gin.HandlerFunc { 54 | middleware := ginMiddleware.NewMiddleware(l, ginMiddleware.WithLimitReachedHandler(onLimitReachedHandler)) 55 | return middleware 56 | } 57 | -------------------------------------------------------------------------------- /pkg/middleware/rateLimiter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package middleware 6 | 7 | import ( 8 | "encoding/json" 9 | "io" 10 | "net/http" 11 | "net/http/httptest" 12 | "testing" 13 | "time" 14 | 15 | "github.com/gin-gonic/gin" 16 | "github.com/silverton-io/buz/pkg/config" 17 | "github.com/silverton-io/buz/pkg/response" 18 | "github.com/stretchr/testify/assert" 19 | ) 20 | 21 | func TestGetDurationFromString(t *testing.T) { 22 | var testCases = []struct { 23 | period string 24 | wantDuration time.Duration 25 | }{ 26 | {"MS", 1 * time.Millisecond}, 27 | {"S", 1 * time.Second}, 28 | {"M", 1 * time.Minute}, 29 | {"H", 1 * time.Hour}, 30 | {"D", 24 * time.Hour}, 31 | {"default", 1 * time.Second}, 32 | } 33 | 34 | for _, tc := range testCases { 35 | t.Run(tc.period, func(t *testing.T) { 36 | got := getDurationFromString(tc.period) 37 | assert.Equal(t, tc.wantDuration, got) 38 | }) 39 | } 40 | } 41 | 42 | func TestOnLimitReachedHandler(t *testing.T) { 43 | rec := httptest.NewRecorder() 44 | c, _ := gin.CreateTestContext(rec) 45 | 46 | onLimitReachedHandler(c) 47 | 48 | body, _ := io.ReadAll(rec.Body) 49 | wantBody, _ := json.Marshal(response.RateLimitExceeded) 50 | assert.Equal(t, http.StatusTooManyRequests, rec.Result().StatusCode) 51 | assert.Equal(t, wantBody, body) 52 | } 53 | 54 | func TestBuildRateLimiter(t *testing.T) { 55 | c := config.RateLimiter{ 56 | Enabled: true, 57 | Period: "H", 58 | Limit: int64(1), 59 | } 60 | wantDuration := getDurationFromString(c.Period) 61 | limiter := BuildRateLimiter(c) 62 | assert.Equal(t, limiter.Rate.Period, wantDuration) 63 | assert.Equal(t, limiter.Rate.Limit, c.Limit) 64 | } 65 | 66 | func TestBuildRateLimiterMiddleware(t *testing.T) { 67 | c := config.RateLimiter{ 68 | Enabled: true, 69 | Period: "H", 70 | Limit: int64(1), 71 | } 72 | limiter := BuildRateLimiter(c) 73 | BuildRateLimiterMiddleware(limiter) 74 | } 75 | -------------------------------------------------------------------------------- /pkg/middleware/timeout.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package middleware 6 | 7 | import ( 8 | "net/http" 9 | "time" 10 | 11 | "github.com/gin-contrib/timeout" 12 | "github.com/gin-gonic/gin" 13 | "github.com/rs/zerolog/log" 14 | "github.com/silverton-io/buz/pkg/config" 15 | "github.com/silverton-io/buz/pkg/response" 16 | ) 17 | 18 | func timeoutHandler(c *gin.Context) { 19 | log.Trace().Stack().Msg("request timed out") 20 | c.JSON(http.StatusRequestTimeout, response.Timeout) 21 | } 22 | 23 | func Timeout(conf config.Timeout) gin.HandlerFunc { 24 | // TODO: pass context down the line so events aren't passed to invalid if the request times out. 25 | return timeout.New( 26 | timeout.WithTimeout(time.Duration(conf.Ms)*time.Millisecond), 27 | timeout.WithHandler(func(c *gin.Context) { 28 | c.Next() 29 | }), 30 | timeout.WithResponse(timeoutHandler), 31 | ) 32 | } 33 | -------------------------------------------------------------------------------- /pkg/middleware/timeout_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package middleware 6 | 7 | import ( 8 | "encoding/json" 9 | "io" 10 | "net/http" 11 | "net/http/httptest" 12 | "reflect" 13 | "testing" 14 | 15 | "github.com/gin-gonic/gin" 16 | "github.com/silverton-io/buz/pkg/config" 17 | "github.com/silverton-io/buz/pkg/response" 18 | ) 19 | 20 | func TestTimeout(t *testing.T) { 21 | u := "/somepath" 22 | fastTimeout := config.Timeout{ 23 | Enabled: true, 24 | Ms: 1, 25 | } 26 | slowTimeout := config.Timeout{ 27 | Enabled: true, 28 | Ms: 30, 29 | } 30 | okResponse, _ := json.Marshal(response.Ok) 31 | timeoutResponse, _ := json.Marshal(response.Timeout) 32 | 33 | var testCases = []struct { 34 | config config.Timeout 35 | wantCode int 36 | wantResponse []byte 37 | }{ 38 | {config: fastTimeout, wantCode: 408, wantResponse: timeoutResponse}, 39 | {config: slowTimeout, wantCode: 200, wantResponse: okResponse}, 40 | } 41 | 42 | for _, tc := range testCases { 43 | gin.SetMode(gin.TestMode) 44 | r := gin.New() 45 | r.Use(Timeout(tc.config)) 46 | r.GET(u, testHandler) 47 | ts := httptest.NewServer(r) 48 | 49 | resp, _ := http.Get(ts.URL + u) 50 | if resp.StatusCode != tc.wantCode { 51 | t.Fatalf(`got status code %v, want %v`, resp.StatusCode, tc.wantCode) 52 | } 53 | defer resp.Body.Close() 54 | body, _ := io.ReadAll(resp.Body) 55 | bodyEquiv := reflect.DeepEqual(body, tc.wantResponse) 56 | if !bodyEquiv { 57 | t.Fatalf(`got response %v, want %v`, body, tc.wantResponse) 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /pkg/params/handler.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package params 6 | 7 | import ( 8 | "github.com/silverton-io/buz/pkg/config" 9 | "github.com/silverton-io/buz/pkg/meta" 10 | ) 11 | 12 | type Handler struct { 13 | Config *config.Config 14 | CollectorMeta *meta.CollectorMeta 15 | } 16 | -------------------------------------------------------------------------------- /pkg/protocol/cloudevents/envelopeBuilder.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package cloudevents 6 | 7 | import ( 8 | "io" 9 | 10 | "github.com/gin-gonic/gin" 11 | "github.com/rs/zerolog/log" 12 | "github.com/silverton-io/buz/pkg/config" 13 | "github.com/silverton-io/buz/pkg/envelope" 14 | "github.com/silverton-io/buz/pkg/meta" 15 | "github.com/silverton-io/buz/pkg/protocol" 16 | "github.com/tidwall/gjson" 17 | ) 18 | 19 | func buildEnvelopesFromRequest(c *gin.Context, conf *config.Config, m *meta.CollectorMeta) []envelope.Envelope { 20 | var envelopes []envelope.Envelope 21 | reqBody, err := io.ReadAll(c.Request.Body) 22 | contexts := envelope.BuildContextsFromRequest(c) 23 | if err != nil { 24 | log.Error().Err(err).Msg("🔴 could not read request body") 25 | return envelopes 26 | } 27 | for _, ce := range gjson.ParseBytes(reqBody).Array() { 28 | evnt, err := buildEvent(ce) 29 | if err != nil { 30 | log.Error().Err(err).Msg("🔴 could not build Cloudevent") 31 | } 32 | n := envelope.NewEnvelope(conf.App) 33 | n.Protocol = protocol.CLOUDEVENTS 34 | if evnt.DataSchema != "" { 35 | n.Schema = evnt.DataSchema 36 | } 37 | if evnt.Time != nil { 38 | n.Timestamp = *evnt.Time 39 | } 40 | n.Contexts = &contexts 41 | n.Payload = evnt.Data 42 | envelopes = append(envelopes, n) 43 | } 44 | return envelopes 45 | } 46 | -------------------------------------------------------------------------------- /pkg/protocol/cloudevents/event.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package cloudevents 6 | 7 | import ( 8 | "database/sql/driver" 9 | "encoding/json" 10 | "time" 11 | ) 12 | 13 | type CloudEvent struct { // https://github.com/cloudevents/spec/blob/v1.0.2/cloudevents/formats/cloudevents.json 14 | Id string `json:"id"` 15 | Source string `json:"source"` 16 | SpecVersion string `json:"specversion"` 17 | Type string `json:"type"` 18 | DataContentType string `json:"datacontenttype"` 19 | DataSchema string `json:"dataschema"` 20 | Subject *string `json:"subject"` 21 | Time *time.Time `json:"time"` 22 | Data map[string]interface{} `json:"data"` 23 | Datab64 string `json:"datab64"` 24 | } 25 | 26 | func (e CloudEvent) SchemaName() *string { 27 | return &e.DataSchema 28 | } 29 | 30 | func (e CloudEvent) PayloadAsByte() ([]byte, error) { 31 | eBytes, err := json.Marshal(e.Data) 32 | if err != nil { 33 | return nil, err 34 | } 35 | return eBytes, nil 36 | } 37 | 38 | func (e CloudEvent) AsByte() ([]byte, error) { 39 | eBytes, err := json.Marshal(e) 40 | if err != nil { 41 | return nil, err 42 | } 43 | return eBytes, nil 44 | } 45 | 46 | func (e CloudEvent) AsMap() (map[string]interface{}, error) { 47 | var event map[string]interface{} 48 | cByte, err := e.AsByte() 49 | if err != nil { 50 | return nil, err 51 | } 52 | err = json.Unmarshal(cByte, &event) 53 | if err != nil { 54 | return nil, err 55 | } 56 | return event, nil 57 | } 58 | 59 | func (e CloudEvent) Value() (driver.Value, error) { 60 | b, err := json.Marshal(e) 61 | return string(b), err 62 | } 63 | 64 | func (e CloudEvent) Scan(input interface{}) error { 65 | return json.Unmarshal(input.([]byte), &e) 66 | } 67 | -------------------------------------------------------------------------------- /pkg/protocol/cloudevents/eventBuilder.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package cloudevents 6 | 7 | import ( 8 | "encoding/json" 9 | 10 | "github.com/rs/zerolog/log" 11 | "github.com/tidwall/gjson" 12 | ) 13 | 14 | func buildEvent(payload gjson.Result) (CloudEvent, error) { 15 | event := CloudEvent{} 16 | event.DataContentType = "application/json" // Always 17 | 18 | pBytes, err := json.Marshal(payload.Value().(map[string]interface{})) 19 | if err != nil { 20 | log.Error().Err(err).Msg("🔴 could not marshal cloudevent payload") 21 | return CloudEvent{}, nil 22 | } 23 | err = json.Unmarshal(pBytes, &event) 24 | if err != nil { 25 | log.Error().Err(err).Msg("🔴 could not unmarshal payload to CloudEvent") 26 | return CloudEvent{}, err 27 | } 28 | return event, nil 29 | } 30 | -------------------------------------------------------------------------------- /pkg/protocol/cloudevents/input.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package cloudevents 6 | 7 | import ( 8 | "net/http" 9 | 10 | "github.com/gin-gonic/gin" 11 | "github.com/rs/zerolog/log" 12 | "github.com/silverton-io/buz/pkg/config" 13 | "github.com/silverton-io/buz/pkg/envelope" 14 | "github.com/silverton-io/buz/pkg/manifold" 15 | "github.com/silverton-io/buz/pkg/meta" 16 | "github.com/silverton-io/buz/pkg/response" 17 | ) 18 | 19 | type CloudeventsInput struct{} 20 | 21 | func (i *CloudeventsInput) Initialize(routerGroup *gin.RouterGroup, manifold *manifold.Manifold, conf *config.Config, metadata *meta.CollectorMeta) error { 22 | if conf.Inputs.Cloudevents.Enabled { 23 | log.Info().Msg("🟢 initializing cloudevents input") 24 | routerGroup.POST(conf.Inputs.Cloudevents.Path, i.Handler(*manifold, *conf, metadata)) 25 | } 26 | if conf.Squawkbox.Enabled { 27 | log.Info().Msg("🟢 initializing cloudevents input squawkbox") 28 | routerGroup.POST("/squawkbox/cloudevents", i.SquawkboxHandler(*manifold, *conf, metadata)) 29 | } 30 | return nil 31 | } 32 | 33 | func (i *CloudeventsInput) Handler(m manifold.Manifold, conf config.Config, metadata *meta.CollectorMeta) gin.HandlerFunc { 34 | fn := func(c *gin.Context) { 35 | if c.ContentType() == "application/cloudevents+json" || c.ContentType() == "application/cloudevents-batch+json" { 36 | envelopes := i.EnvelopeBuilder(c, &conf, metadata) 37 | err := m.Enqueue(envelopes) 38 | if err != nil { 39 | c.Header("Retry-After", response.RETRY_AFTER_60) 40 | c.JSON(http.StatusServiceUnavailable, response.ManifoldDistributionError) 41 | } else { 42 | c.JSON(http.StatusOK, response.Ok) 43 | } 44 | } else { 45 | c.JSON(http.StatusBadRequest, response.InvalidContentType) 46 | } 47 | } 48 | return gin.HandlerFunc(fn) 49 | 50 | } 51 | 52 | func (i *CloudeventsInput) SquawkboxHandler(m manifold.Manifold, conf config.Config, metadata *meta.CollectorMeta) gin.HandlerFunc { 53 | fn := func(c *gin.Context) { 54 | envelopes := i.EnvelopeBuilder(c, &conf, metadata) 55 | c.JSON(http.StatusOK, envelopes) 56 | } 57 | return gin.HandlerFunc(fn) 58 | } 59 | 60 | func (i *CloudeventsInput) EnvelopeBuilder(c *gin.Context, conf *config.Config, metadata *meta.CollectorMeta) []envelope.Envelope { 61 | return buildEnvelopesFromRequest(c, conf, metadata) 62 | } 63 | -------------------------------------------------------------------------------- /pkg/protocol/pixel/envelopeBuilder.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package pixel 6 | 7 | import ( 8 | "github.com/gin-gonic/gin" 9 | "github.com/rs/zerolog/log" 10 | "github.com/silverton-io/buz/pkg/config" 11 | "github.com/silverton-io/buz/pkg/envelope" 12 | "github.com/silverton-io/buz/pkg/meta" 13 | "github.com/silverton-io/buz/pkg/protocol" 14 | ) 15 | 16 | // NOTE - one envelope per request 17 | func buildEnvelopesFromRequest(c *gin.Context, conf *config.Config, m *meta.CollectorMeta) []envelope.Envelope { 18 | var envelopes []envelope.Envelope 19 | contexts := envelope.BuildContextsFromRequest(c) 20 | n := envelope.NewEnvelope(conf.App) 21 | evnt, err := buildEvent(c) 22 | if err != nil { 23 | log.Error().Err(err).Msg("🔴 could not build pixel event") 24 | } 25 | n.Protocol = protocol.PIXEL 26 | if evnt.Schema != "" { 27 | n.Schema = evnt.Schema 28 | } 29 | n.Contexts = &contexts 30 | n.Payload = evnt.Data 31 | envelopes = append(envelopes, n) 32 | return envelopes 33 | } 34 | -------------------------------------------------------------------------------- /pkg/protocol/pixel/eventBuilder.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package pixel 6 | 7 | import ( 8 | b64 "encoding/base64" 9 | "encoding/json" 10 | 11 | "github.com/gin-gonic/gin" 12 | "github.com/silverton-io/buz/pkg/envelope" 13 | "github.com/silverton-io/buz/pkg/util" 14 | ) 15 | 16 | const ( 17 | B64_ENCODED_PAYLOAD_PARAM string = "hbp" 18 | ARBITRARY_PIXEL_SCHEMA string = "io.silverton/buz/pixel/arbitrary/v1.0.json" 19 | ) 20 | 21 | func buildEvent(c *gin.Context) (envelope.SelfDescribingPayload, error) { 22 | params := util.MapUrlParams(c) 23 | schemaName := util.GetSchemaNameFromRequest(c, ARBITRARY_PIXEL_SCHEMA) 24 | base64EncodedPayload := params[B64_ENCODED_PAYLOAD_PARAM] 25 | if base64EncodedPayload != nil { 26 | p, err := b64.RawStdEncoding.DecodeString(base64EncodedPayload.(string)) 27 | if err != nil { 28 | return envelope.SelfDescribingPayload{}, err 29 | } 30 | var payload map[string]interface{} 31 | err = json.Unmarshal(p, &payload) 32 | if err != nil { 33 | return envelope.SelfDescribingPayload{}, err 34 | } 35 | sdp := envelope.SelfDescribingPayload{ 36 | Schema: schemaName, 37 | Data: payload, 38 | } 39 | return sdp, nil 40 | } 41 | 42 | sdp := envelope.SelfDescribingPayload{ 43 | Schema: schemaName, 44 | Data: params, 45 | } 46 | return sdp, nil 47 | } 48 | -------------------------------------------------------------------------------- /pkg/protocol/protocol.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package protocol 6 | 7 | const ( 8 | SNOWPLOW string = "snowplow" 9 | SELF_DESCRIBING string = "selfDescribing" 10 | CLOUDEVENTS string = "cloudevents" 11 | WEBHOOK string = "webhook" 12 | PIXEL string = "pixel" 13 | ) 14 | 15 | func GetInputProtocols() []string { 16 | return []string{SNOWPLOW, SELF_DESCRIBING, CLOUDEVENTS, WEBHOOK, PIXEL} 17 | } 18 | -------------------------------------------------------------------------------- /pkg/protocol/protocol_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package protocol 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestInputConst(t *testing.T) { 14 | assert.Equal(t, "snowplow", SNOWPLOW) 15 | assert.Equal(t, "selfDescribing", SELF_DESCRIBING) 16 | assert.Equal(t, "cloudevents", CLOUDEVENTS) 17 | assert.Equal(t, "webhook", WEBHOOK) 18 | assert.Equal(t, "pixel", PIXEL) 19 | } 20 | -------------------------------------------------------------------------------- /pkg/protocol/selfdescribing/envelopeBuilder.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package selfdescribing 6 | 7 | import ( 8 | "bytes" 9 | "compress/gzip" 10 | "io" 11 | 12 | "github.com/gin-gonic/gin" 13 | "github.com/rs/zerolog/log" 14 | "github.com/silverton-io/buz/pkg/config" 15 | "github.com/silverton-io/buz/pkg/envelope" 16 | "github.com/silverton-io/buz/pkg/meta" 17 | "github.com/silverton-io/buz/pkg/protocol" 18 | "github.com/tidwall/gjson" 19 | ) 20 | 21 | func buildEnvelopesFromRequest(c *gin.Context, conf *config.Config, m *meta.CollectorMeta) []envelope.Envelope { 22 | var envelopes []envelope.Envelope 23 | reqBody, err := io.ReadAll(c.Request.Body) 24 | if err != nil { 25 | log.Error().Err(err).Msg("🔴 could not read request body") 26 | return envelopes 27 | } 28 | contexts := envelope.BuildContextsFromRequest(c) 29 | // If the request body is gzipped, decompress it 30 | if c.GetHeader("Content-Encoding") == "gzip" { 31 | reader, err := gzip.NewReader(bytes.NewReader(reqBody)) 32 | if err != nil { 33 | log.Error().Err(err).Msg("🔴 could not decompress gzipped request body") 34 | return envelopes 35 | } 36 | defer reader.Close() 37 | reqBody, err = io.ReadAll(reader) 38 | if err != nil { 39 | log.Error().Err(err).Msg("🔴 could not read decompressed gzipped request body") 40 | return envelopes 41 | } 42 | } 43 | 44 | for _, e := range gjson.ParseBytes(reqBody).Array() { 45 | n := envelope.NewEnvelope(conf.App) 46 | evnt, err := buildEvent(e, conf.SelfDescribing) 47 | if err != nil { 48 | log.Error().Err(err).Msg("🔴 could not build generic event") 49 | } 50 | 51 | n.Protocol = protocol.SELF_DESCRIBING 52 | if evnt.Payload.Schema != "" { 53 | n.Schema = evnt.Payload.Schema 54 | } 55 | n.Contexts = &contexts 56 | n.Payload = evnt.Payload.Data 57 | envelopes = append(envelopes, n) 58 | } 59 | return envelopes 60 | } 61 | -------------------------------------------------------------------------------- /pkg/protocol/selfdescribing/eventBuilder.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package selfdescribing 6 | 7 | import ( 8 | "github.com/rs/zerolog/log" 9 | "github.com/silverton-io/buz/pkg/config" 10 | "github.com/silverton-io/buz/pkg/envelope" 11 | "github.com/tidwall/gjson" 12 | ) 13 | 14 | type GenericEvent envelope.SelfDescribingEvent 15 | 16 | func buildEvent(e gjson.Result, conf config.SelfDescribing) (GenericEvent, error) { 17 | var sdPayload envelope.SelfDescribingPayload 18 | c := e.Get(conf.Contexts.RootKey).Value() 19 | var contexts map[string]interface{} 20 | if c != nil { 21 | contexts = c.(map[string]interface{}) 22 | } 23 | payload := e.Get(conf.Payload.RootKey) 24 | payloadSchema := payload.Get(conf.Payload.SchemaKey).String() 25 | payloadData := payload.Get(conf.Payload.DataKey).Value() 26 | if payloadData == nil { 27 | log.Error().Stack().Msg("🔴 no data payload found in generic event for key: " + conf.Payload.RootKey + "." + conf.Payload.DataKey) 28 | log.Debug().Interface("event", e.Value()).Interface("config", conf).Msg("🟡 event format does not match config format") 29 | } else { 30 | sdPayload = envelope.SelfDescribingPayload{ 31 | Schema: payloadSchema, 32 | Data: payloadData.(map[string]interface{}), 33 | } 34 | } 35 | genEvent := GenericEvent{ 36 | Contexts: contexts, // FIXME - validate these contexts? 37 | Payload: sdPayload, 38 | } 39 | return genEvent, nil 40 | } 41 | -------------------------------------------------------------------------------- /pkg/protocol/snowplow/envelopeBuilder.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package snowplow 6 | 7 | import ( 8 | "io" 9 | "time" 10 | 11 | "github.com/gin-gonic/gin" 12 | "github.com/rs/zerolog/log" 13 | "github.com/silverton-io/buz/pkg/config" 14 | "github.com/silverton-io/buz/pkg/envelope" 15 | "github.com/silverton-io/buz/pkg/meta" 16 | "github.com/silverton-io/buz/pkg/protocol" 17 | "github.com/silverton-io/buz/pkg/util" 18 | "github.com/tidwall/gjson" 19 | ) 20 | 21 | func buildSnowplowEnvelope(conf config.Config, e SnowplowEvent) envelope.Envelope { 22 | n := envelope.NewEnvelope(conf.App) 23 | if e.DvceCreatedTstamp != nil { 24 | n.Timestamp = *e.DvceCreatedTstamp 25 | } else { 26 | n.Timestamp = time.Now() 27 | } 28 | n.Protocol = protocol.SNOWPLOW 29 | n.Schema = *e.SelfDescribingEvent.SchemaName() 30 | n.Payload = e.Map() 31 | return n 32 | } 33 | 34 | func buildEnvelopesFromRequest(c *gin.Context, conf *config.Config, m *meta.CollectorMeta) []envelope.Envelope { 35 | var envelopes []envelope.Envelope 36 | if c.Request.Method == "POST" { 37 | body, err := io.ReadAll(c.Request.Body) 38 | if err != nil { 39 | log.Error().Err(err).Msg("🔴 could not read request body") 40 | return envelopes 41 | } 42 | payloadData := gjson.GetBytes(body, "data") 43 | for _, event := range payloadData.Array() { 44 | spEvent := buildEventFromMappedParams(c, event.Value().(map[string]interface{}), *conf) 45 | e := buildSnowplowEnvelope(*conf, spEvent) 46 | envelopes = append(envelopes, e) 47 | } 48 | } else { 49 | params := util.MapUrlParams(c) 50 | spEvent := buildEventFromMappedParams(c, params, *conf) 51 | e := buildSnowplowEnvelope(*conf, spEvent) 52 | envelopes = append(envelopes, e) 53 | } 54 | return envelopes 55 | } 56 | -------------------------------------------------------------------------------- /pkg/protocol/snowplow/event_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package snowplow 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestConstants(t *testing.T) { 14 | assert.Equal(t, PAGE_PING, "page_ping") 15 | assert.Equal(t, PAGE_VIEW, "page_view") 16 | assert.Equal(t, STRUCT_EVENT, "struct_event") 17 | assert.Equal(t, SELF_DESCRIBING_EVENT, "self_describing") 18 | assert.Equal(t, TRANSACTION, "transaction") 19 | assert.Equal(t, TRANSACTION_ITEM, "transaction_item") 20 | assert.Equal(t, AD_IMPRESSION, "ad_impression") 21 | assert.Equal(t, UNKNOWN_EVENT, "unknown_event") 22 | assert.Equal(t, UNKNOWN_SCHEMA, "unknown_schema") 23 | } 24 | 25 | // FIXME 26 | func TestSnowplowEvent(t *testing.T) {} 27 | 28 | func TestGetEventType(t *testing.T) { 29 | assert.Equal(t, PAGE_PING, getEventType("pp")) 30 | assert.Equal(t, PAGE_VIEW, getEventType("pv")) 31 | assert.Equal(t, STRUCT_EVENT, getEventType("se")) 32 | assert.Equal(t, SELF_DESCRIBING_EVENT, getEventType("ue")) 33 | assert.Equal(t, TRANSACTION, getEventType("tr")) 34 | assert.Equal(t, TRANSACTION_ITEM, getEventType("ti")) 35 | assert.Equal(t, AD_IMPRESSION, getEventType("ad")) 36 | assert.Equal(t, UNKNOWN_EVENT, getEventType("yikes")) 37 | } 38 | -------------------------------------------------------------------------------- /pkg/protocol/webhook/envelopeBuilder.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package webhook 6 | 7 | import ( 8 | "bytes" 9 | "compress/gzip" 10 | "io" 11 | 12 | "github.com/gin-gonic/gin" 13 | "github.com/rs/zerolog/log" 14 | "github.com/silverton-io/buz/pkg/config" 15 | "github.com/silverton-io/buz/pkg/envelope" 16 | "github.com/silverton-io/buz/pkg/meta" 17 | "github.com/silverton-io/buz/pkg/protocol" 18 | "github.com/tidwall/gjson" 19 | ) 20 | 21 | func buildEnvelopesFromRequest(c *gin.Context, conf *config.Config, m *meta.CollectorMeta) []envelope.Envelope { 22 | var envelopes []envelope.Envelope 23 | reqBody, err := io.ReadAll(c.Request.Body) 24 | if err != nil { 25 | log.Error().Err(err).Msg("🔴 could not read request body") 26 | return envelopes 27 | } 28 | // If the request body is gzipped, decompress it 29 | if c.GetHeader("Content-Encoding") == "gzip" { 30 | log.Debug().Msg("🟢 Request body is gzip encoded - attempting to decompress.") 31 | reader, err := gzip.NewReader(bytes.NewReader(reqBody)) 32 | if err != nil { 33 | log.Error().Err(err).Msg("🔴 could not decompress gzipped request body") 34 | return envelopes 35 | } 36 | defer reader.Close() 37 | reqBody, err = io.ReadAll(reader) 38 | if err != nil { 39 | log.Error().Err(err).Msg("🔴 could not read decompressed gzipped request body") 40 | return envelopes 41 | } 42 | } 43 | for _, e := range gjson.ParseBytes(reqBody).Array() { 44 | n := envelope.NewEnvelope(conf.App) 45 | contexts := envelope.BuildContextsFromRequest(c) 46 | evnt, err := buildEvent(c, e) 47 | if err != nil { 48 | log.Error().Err(err).Msg("🔴 could not build webhook event") 49 | } 50 | n.Protocol = protocol.WEBHOOK 51 | if evnt.Schema != "" { 52 | n.Schema = evnt.Schema 53 | } 54 | n.Contexts = &contexts 55 | n.Payload = evnt.Data 56 | envelopes = append(envelopes, n) 57 | } 58 | return envelopes 59 | } 60 | -------------------------------------------------------------------------------- /pkg/protocol/webhook/eventBuilder.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package webhook 6 | 7 | import ( 8 | "github.com/gin-gonic/gin" 9 | "github.com/silverton-io/buz/pkg/envelope" 10 | "github.com/silverton-io/buz/pkg/util" 11 | "github.com/tidwall/gjson" 12 | ) 13 | 14 | const ARBITRARY_WEBHOOK_SCHEMA = "io.silverton/buz/hook/arbitrary/v1.0.json" 15 | 16 | func buildEvent(c *gin.Context, payload gjson.Result) (envelope.SelfDescribingPayload, error) { 17 | schemaName := util.GetSchemaNameFromRequest(c, ARBITRARY_WEBHOOK_SCHEMA) 18 | sdp := envelope.SelfDescribingPayload{ 19 | Schema: schemaName, 20 | Data: payload.Value().(map[string]interface{}), 21 | } 22 | return sdp, nil 23 | } 24 | -------------------------------------------------------------------------------- /pkg/registry/handler.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package registry 6 | 7 | import ( 8 | "github.com/gin-gonic/gin" 9 | "github.com/rs/zerolog/log" 10 | "github.com/silverton-io/buz/pkg/response" 11 | "github.com/tidwall/gjson" 12 | ) 13 | 14 | func PurgeCacheHandler(r *Registry) gin.HandlerFunc { 15 | fn := func(c *gin.Context) { 16 | log.Debug().Msg("🟡 schema cache purged") 17 | r.Cache.Clear() 18 | c.JSON(200, response.CachePurged) 19 | } 20 | return gin.HandlerFunc(fn) 21 | } 22 | 23 | func GetSchemaHandler(r *Registry) gin.HandlerFunc { 24 | fn := func(c *gin.Context) { 25 | schemaName := c.Param(SCHEMA_PARAM)[1:] 26 | exists, schemaContents := r.Get(schemaName) 27 | if !exists { 28 | c.JSON(404, response.SchemaNotAvailable) 29 | } else { 30 | schema := gjson.ParseBytes(schemaContents) 31 | c.JSON(200, schema.Value()) 32 | } 33 | 34 | } 35 | return gin.HandlerFunc(fn) 36 | } 37 | -------------------------------------------------------------------------------- /pkg/registry/handler_test.go: -------------------------------------------------------------------------------- 1 | package registry 2 | -------------------------------------------------------------------------------- /pkg/registry/param.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package registry 6 | 7 | const ( 8 | SCHEMAS_ROUTE = "/s/" 9 | CACHE_PURGE_ROUTE = "/c/purge" 10 | SCHEMA_PARAM = "schema" 11 | ) 12 | -------------------------------------------------------------------------------- /pkg/request/request.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package request 6 | 7 | import ( 8 | "bytes" 9 | "encoding/json" 10 | "io" 11 | "net/http" 12 | "net/url" 13 | 14 | "github.com/rs/zerolog/log" 15 | "github.com/silverton-io/buz/pkg/envelope" 16 | ) 17 | 18 | const ( 19 | JSON_CONTENT_TYPE string = "application/json" 20 | ) 21 | 22 | func PostPayload(url url.URL, payload interface{}, header http.Header) (resp *http.Response, err error) { 23 | data, err := json.Marshal(payload) 24 | if err != nil { 25 | log.Error().Err(err).Msg("🔴 could not marshal payload") 26 | return nil, err 27 | } 28 | buf := bytes.NewBuffer(data) 29 | // Set up a client, add appropriate headers, and make the request 30 | client := http.Client{} 31 | req, err := http.NewRequest(http.MethodPost, url.String(), buf) 32 | if err != nil { 33 | log.Error().Err(err).Msg("🔴 could not build request") 34 | } 35 | header.Add("Content-Type", JSON_CONTENT_TYPE) 36 | req.Header = header 37 | resp, err = client.Do(req) 38 | if resp == nil { 39 | return resp, nil 40 | } 41 | if resp.StatusCode != http.StatusOK { 42 | _, err := io.ReadAll(resp.Body) 43 | if err != nil { 44 | log.Error().Err(err).Msg("🔴 could not read response body") 45 | return nil, err 46 | } 47 | } 48 | if err != nil { 49 | log.Error().Err(err).Msg("🔴 could not post payload") 50 | return nil, err 51 | } 52 | return resp, nil 53 | } 54 | 55 | func PostEvent(url url.URL, payload envelope.SelfDescribingEvent) (resp *http.Response, err error) { 56 | header := http.Header{} // No headers by default 57 | resp, err = PostPayload(url, payload, header) 58 | return resp, err 59 | } 60 | 61 | func PostEnvelopes(url url.URL, envelopes []envelope.Envelope, header http.Header) (resp *http.Response, err error) { 62 | resp, err = PostPayload(url, envelopes, header) 63 | return resp, err 64 | } 65 | 66 | func Get(url url.URL) (body []byte, err error) { 67 | resp, err := http.Get(url.String()) 68 | if err != nil { 69 | log.Trace().Err(err).Msg("could not get url " + url.String()) 70 | return nil, err 71 | } 72 | defer resp.Body.Close() 73 | body, ioerr := io.ReadAll(resp.Body) 74 | if ioerr != nil { 75 | log.Trace().Err(ioerr).Msg("could not read response body") 76 | return nil, ioerr 77 | } 78 | return body, nil 79 | } 80 | -------------------------------------------------------------------------------- /pkg/request/request_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package request 6 | 7 | import ( 8 | "encoding/json" 9 | "io" 10 | "net/http" 11 | "net/http/httptest" 12 | "net/url" 13 | "reflect" 14 | "testing" 15 | 16 | "github.com/silverton-io/buz/pkg/envelope" 17 | ) 18 | 19 | func TestConst(t *testing.T) { 20 | want_js_ct := "application/json" 21 | 22 | if JSON_CONTENT_TYPE != want_js_ct { 23 | t.Fatalf(`got %v, want %v`, JSON_CONTENT_TYPE, want_js_ct) 24 | } 25 | } 26 | 27 | func TestPostEvent(t *testing.T) { 28 | u := "/somewhere" 29 | payload := envelope.SelfDescribingEvent{} 30 | marshaledPayload, _ := json.Marshal(payload) 31 | 32 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 33 | w.Write([]byte("made it")) 34 | d := r.URL.EscapedPath() 35 | if d != u { 36 | t.Fatalf(`posted payload to url %v, want %v`, d, u) 37 | } 38 | sentPayload, _ := io.ReadAll(r.Body) 39 | payloadsEquiv := reflect.DeepEqual(sentPayload, marshaledPayload) 40 | if !payloadsEquiv { 41 | t.Fatalf(`posted body %v, want %v`, sentPayload, marshaledPayload) 42 | } 43 | })) 44 | defer ts.Close() 45 | 46 | dest, _ := url.Parse(ts.URL + u) 47 | 48 | PostEvent(*dest, payload) 49 | 50 | } 51 | 52 | func TestGet(t *testing.T) { 53 | u := "/somewhere" 54 | wantResp := []byte("something important of course") 55 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 56 | d := r.URL.EscapedPath() 57 | t.Run("proper url", func(t *testing.T) { 58 | if d != u { 59 | t.Fatalf(`got url %v, want %v`, d, u) 60 | } 61 | }) 62 | w.Write(wantResp) 63 | })) 64 | 65 | dest, _ := url.Parse(ts.URL + u) 66 | 67 | respBody, _ := Get(*dest) 68 | t.Run("proper response", func(t *testing.T) { 69 | equiv := reflect.DeepEqual(respBody, wantResp) 70 | if !equiv { 71 | t.Fatalf(`got %v, want %v`, respBody, wantResp) 72 | } 73 | }) 74 | } 75 | -------------------------------------------------------------------------------- /pkg/response/response.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package response 6 | 7 | type Response struct { 8 | Message string `json:"message"` 9 | } 10 | 11 | var Ok = Response{ 12 | Message: "ok", 13 | } 14 | 15 | var InvalidContentType = Response{ 16 | Message: "invalid content type", 17 | } 18 | 19 | var BadRequest = Response{ 20 | Message: "bad request", 21 | } 22 | 23 | var SchemaNotCached = Response{ 24 | Message: "schema not cached", 25 | } 26 | 27 | var SchemaNotAvailable = Response{ 28 | Message: "schema not available", 29 | } 30 | 31 | var CachePurged = Response{ 32 | Message: "cache purged", 33 | } 34 | 35 | var Timeout = Response{ 36 | Message: "request timed out", 37 | } 38 | 39 | var RateLimitExceeded = Response{ 40 | Message: "rate limit exceeded", 41 | } 42 | 43 | var ManifoldDistributionError = Response{ 44 | Message: "distribution error", 45 | } 46 | 47 | var MissingAuthHeader = Response{ 48 | Message: "missing authorization header", 49 | } 50 | 51 | var MissingAuthSchemeOrToken = Response{ 52 | Message: "missing auth scheme or token", 53 | } 54 | 55 | var InvalidAuthScheme = Response{ 56 | Message: "invalid scheme", 57 | } 58 | 59 | var InvalidAuthToken = Response{ 60 | Message: "invalid token", 61 | } 62 | -------------------------------------------------------------------------------- /pkg/response/response_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package response 6 | 7 | import "testing" 8 | 9 | func TestResponse(t *testing.T) { 10 | var testCases = []struct { 11 | provided Response 12 | want Response 13 | }{ 14 | {Ok, Response{Message: "ok"}}, 15 | {InvalidContentType, Response{Message: "invalid content type"}}, 16 | {BadRequest, Response{Message: "bad request"}}, 17 | {SchemaNotCached, Response{Message: "schema not cached"}}, 18 | {Timeout, Response{Message: "request timed out"}}, 19 | {RateLimitExceeded, Response{Message: "rate limit exceeded"}}, 20 | } 21 | 22 | for _, tc := range testCases { 23 | t.Run(tc.want.Message, func(t *testing.T) { 24 | if tc.provided != tc.want { 25 | t.Fatalf(`got %v, want %v`, tc.provided, tc.want) 26 | } 27 | }) 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /pkg/response/retry.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package response 6 | 7 | const ( 8 | RETRY_AFTER_3 string = "3" 9 | RETRY_AFTER_30 string = "30" 10 | RETRY_AFTER_60 string = "60" 11 | RETRY_AFTER_90 string = "90" 12 | RETRY_AFTER_120 string = "120" 13 | ) 14 | -------------------------------------------------------------------------------- /pkg/sink/sink_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package sink 6 | 7 | import ( 8 | "github.com/google/uuid" 9 | "github.com/silverton-io/buz/pkg/backend/backendutils" 10 | "github.com/silverton-io/buz/pkg/config" 11 | "github.com/stretchr/testify/mock" 12 | ) 13 | 14 | type MockSink struct { 15 | mock.Mock 16 | } 17 | 18 | func (ms *MockSink) Metadata() backendutils.SinkMetadata { 19 | id := uuid.New() 20 | sinkName := "test" 21 | return backendutils.SinkMetadata{ 22 | Id: id, 23 | Name: sinkName, 24 | DeliveryRequired: false, 25 | DefaultOutput: "somewhere", 26 | DeadletterOutput: "else", 27 | } 28 | } 29 | 30 | func (ms *MockSink) Initialize(conf config.Sink) error { 31 | ms.Called(conf) 32 | return nil 33 | } 34 | 35 | func (ms *MockSink) Shutdown() error { 36 | ms.Called() 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /pkg/stats/protocolStats.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package stats 6 | 7 | type ProtocolStats struct { 8 | // vmu sync.Mutex 9 | // imu sync.Mutex 10 | Invalid map[string]map[string]int64 `json:"invalid"` 11 | Valid map[string]map[string]int64 `json:"valid"` 12 | } 13 | 14 | // func (ps *ProtocolStats) Build() { 15 | // var vProtoStat = make(map[string]map[string]int64) 16 | // var invProtoStat = make(map[string]map[string]int64) 17 | // ps.Valid = vProtoStat 18 | // ps.Invalid = invProtoStat 19 | // for _, protocol := range protocol.GetInputProtocols() { 20 | // var vEventStat = make(map[string]int64) 21 | // var invEventStat = make(map[string]int64) 22 | // ps.Valid[protocol] = vEventStat 23 | // ps.Invalid[protocol] = invEventStat 24 | // } 25 | // } 26 | 27 | // func (ps *ProtocolStats) IncrementValid(event *envelope.EventMeta, count int64) { 28 | // i := ps.Valid[event.Protocol][event.Namespace] 29 | // ps.vmu.Lock() 30 | // defer ps.vmu.Unlock() 31 | // ps.Valid[event.Protocol][event.Namespace] = i + count 32 | // } 33 | 34 | // func (ps *ProtocolStats) IncrementInvalid(event *envelope.EventMeta, count int64) { 35 | // i := ps.Invalid[event.Protocol][event.Namespace] 36 | // ps.imu.Lock() 37 | // defer ps.imu.Unlock() 38 | // ps.Invalid[event.Protocol][event.Namespace] = i + count 39 | // } 40 | 41 | // func BuildProtocolStats() *ProtocolStats { 42 | // ps := ProtocolStats{} 43 | // ps.Build() 44 | // return &ps 45 | // } 46 | -------------------------------------------------------------------------------- /pkg/stats/routes.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package stats 6 | 7 | const STATS_PATH string = "/stats" 8 | -------------------------------------------------------------------------------- /pkg/stats/routes_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package stats 6 | 7 | import "testing" 8 | 9 | func TestStatsRoutes(t *testing.T) { 10 | want := "/stats" 11 | if STATS_PATH != want { 12 | t.Fatalf(`Stats path is %v, want %v`, STATS_PATH, want) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /pkg/tele/tele_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package tele 6 | 7 | import "testing" 8 | 9 | func TestElapsed(t *testing.T) {} 10 | 11 | func TestHeartbeat(t *testing.T) {} 12 | 13 | func TestSis(t *testing.T) {} 14 | 15 | func TestMetry(t *testing.T) {} 16 | -------------------------------------------------------------------------------- /pkg/testUtil/gin.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package testutil 6 | 7 | import ( 8 | "net/http/httptest" 9 | "time" 10 | 11 | "github.com/gin-gonic/gin" 12 | ) 13 | 14 | const URL = "/test" 15 | 16 | func TestHandler(c *gin.Context) { 17 | time.Sleep(3 * time.Millisecond) 18 | } 19 | 20 | func BuildTestServer(handlerFuncs ...gin.HandlerFunc) *httptest.Server { 21 | // Set up gin, router, middleware 22 | gin.SetMode(gin.TestMode) 23 | r := gin.New() 24 | // Use provided middleware/handlerfuncs 25 | for _, hf := range handlerFuncs { 26 | r.Use(hf) 27 | } 28 | r.GET(URL, TestHandler) 29 | return httptest.NewServer(r) 30 | } 31 | 32 | func BuildRecordedEngine() (*httptest.ResponseRecorder, *gin.Context, *gin.Engine) { 33 | rec := httptest.NewRecorder() 34 | context, engine := gin.CreateTestContext(rec) 35 | return rec, context, engine 36 | } 37 | -------------------------------------------------------------------------------- /pkg/util/hash.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "crypto/md5" 9 | "encoding/hex" 10 | ) 11 | 12 | func Md5(s string) string { 13 | m := md5.Sum([]byte(s)) 14 | return hex.EncodeToString(m[:]) 15 | } 16 | -------------------------------------------------------------------------------- /pkg/util/hash_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | ) 11 | 12 | // TestMd5 calls Md5 with a string, 13 | // checking to ensure it returns the appropriate hash. 14 | func TestMd5(t *testing.T) { 15 | var tests = []struct { 16 | in string 17 | want string 18 | }{ 19 | {"giggitygiggitygoo", "c4f081a6f2bcd2d2a40441c161f46dca"}, 20 | {"", "d41d8cd98f00b204e9800998ecf8427e"}, 21 | } 22 | for _, tt := range tests { 23 | tName := fmt.Sprintf("%v,%v", tt.in, tt.want) 24 | t.Run(tName, func(t *testing.T) { 25 | out := Md5(tt.in) 26 | if out != tt.want { 27 | t.Fatalf(`Md5(%v) = %v, want %v`, tt.in, out, tt.want) 28 | } 29 | }) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /pkg/util/http.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "github.com/gin-gonic/gin" 9 | ) 10 | 11 | // HttpHeadersToMap returns a map of http headers, 12 | // but treats single-element headers as strings instead 13 | // of slices of length 1 14 | func HttpHeadersToMap(c *gin.Context) map[string]interface{} { 15 | headers := make(map[string]interface{}) 16 | for k, v := range c.Request.Header { 17 | if len(v) == 1 { 18 | headers[k] = v[0] 19 | } else { 20 | headers[k] = v 21 | } 22 | } 23 | return headers 24 | } 25 | -------------------------------------------------------------------------------- /pkg/util/http_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "net/http" 9 | "net/http/httptest" 10 | "testing" 11 | 12 | "github.com/gin-gonic/gin" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | // TestHttpHeadersToMap ensures 17 | // only the first value of a specific header 18 | // shows up. 19 | func TestHttpHeadersToMap(t *testing.T) { 20 | rec := httptest.NewRecorder() 21 | c, _ := gin.CreateTestContext(rec) 22 | c.Request = &http.Request{ 23 | Header: make(http.Header), 24 | } 25 | 26 | want := map[string]interface{}{ 27 | "Header1": "val1", 28 | "Header2": []string{"val2", "val3"}, 29 | } 30 | c.Request.Header.Add("header1", "val1") 31 | c.Request.Header.Add("Header2", "val2") 32 | c.Request.Header.Add("Header2", "val3") 33 | 34 | got := HttpHeadersToMap(c) 35 | 36 | assert.Equal(t, want, got) 37 | } 38 | -------------------------------------------------------------------------------- /pkg/util/identity.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "github.com/gin-gonic/gin" 9 | "github.com/silverton-io/buz/pkg/config" 10 | "github.com/silverton-io/buz/pkg/constants" 11 | ) 12 | 13 | func GetIdentityOrFallback(c *gin.Context, conf config.Identity) string { 14 | identity := c.GetString(constants.IDENTITY) 15 | if identity == "" { 16 | return conf.Fallback 17 | } 18 | return identity 19 | } 20 | -------------------------------------------------------------------------------- /pkg/util/identity_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "net/http/httptest" 9 | "testing" 10 | 11 | "github.com/gin-gonic/gin" 12 | "github.com/silverton-io/buz/pkg/config" 13 | "github.com/silverton-io/buz/pkg/constants" 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | var FAKE_IDENTITY_CONF = config.Identity{Cookie: config.IdentityCookie{ 18 | Enabled: true, 19 | Name: "nuid", 20 | Secure: true, 21 | TtlDays: 365, 22 | Domain: "new.domain.dev", 23 | Path: "/", 24 | SameSite: "false", 25 | }, 26 | Fallback: "36acc892-3850-4991-bcb6-f8ca7ef23543", 27 | } 28 | 29 | var FAKE_RECORDER = httptest.NewRecorder() 30 | var FAKE_CONTEXT, _ = gin.CreateTestContext(FAKE_RECORDER) 31 | 32 | func TestGetIdentityOrFallbackNoIdentity(t *testing.T) { 33 | want := FAKE_IDENTITY_CONF.Fallback 34 | got := GetIdentityOrFallback(FAKE_CONTEXT, FAKE_IDENTITY_CONF) 35 | 36 | assert.Equal(t, want, got) 37 | } 38 | 39 | func TestGetIdentityOrFallbackWithIdentity(t *testing.T) { 40 | want := "9c96bc72-7913-4058-91bb-3970714a55c2" 41 | FAKE_CONTEXT.Set(constants.IDENTITY, want) 42 | got := GetIdentityOrFallback(FAKE_CONTEXT, FAKE_IDENTITY_CONF) 43 | 44 | assert.Equal(t, want, got) 45 | } 46 | -------------------------------------------------------------------------------- /pkg/util/mapUrlParams.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "net/url" 9 | 10 | "github.com/gin-gonic/gin" 11 | ) 12 | 13 | // Coerce query params to a map[string]interface{}. 14 | // Only use the first val of each key. 15 | func MapUrlParams(c *gin.Context, excludeParams ...string) map[string]interface{} { 16 | mappedParams := make(map[string]interface{}) 17 | params := c.Request.URL.Query() 18 | for k, v := range params { 19 | if excludeParams != nil { 20 | for _, excludeParam := range excludeParams { 21 | if k != excludeParam { 22 | mappedParams[k] = v[0] 23 | } 24 | } 25 | } else { 26 | mappedParams[k] = v[0] 27 | } 28 | } 29 | return mappedParams 30 | } 31 | 32 | func QueryToMap(v url.Values) map[string]interface{} { 33 | mappedParams := make(map[string]interface{}) 34 | for k, v := range v { 35 | mappedParams[k] = v[0] 36 | } 37 | return mappedParams 38 | } 39 | -------------------------------------------------------------------------------- /pkg/util/mapUrlParams_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "net/http" 9 | "net/http/httptest" 10 | "testing" 11 | 12 | "github.com/gin-gonic/gin" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestMapParams(t *testing.T) { 17 | url := "something/else?p1=v1&p2=v2&p2=v3" 18 | want := map[string]interface{}{ 19 | "p1": "v1", 20 | "p2": "v2", 21 | } 22 | req, _ := http.NewRequest("GET", url, nil) 23 | w := httptest.NewRecorder() 24 | c, _ := gin.CreateTestContext(w) 25 | 26 | c.Request = req 27 | 28 | params := MapUrlParams(c) 29 | assert.Equal(t, params, want) 30 | } 31 | 32 | func TestQueryToMap(t *testing.T) { 33 | params := map[string][]string{ 34 | "p1": {"v1", "v2"}, 35 | "p2": {"v2"}, 36 | } 37 | expected := map[string]interface{}{ 38 | "p1": "v1", 39 | "p2": "v2", 40 | } 41 | mapped := QueryToMap(params) 42 | assert.Equal(t, expected, mapped) 43 | } 44 | -------------------------------------------------------------------------------- /pkg/util/pprint.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "encoding/json" 9 | "fmt" 10 | ) 11 | 12 | func Pprint(i interface{}) string { 13 | payload, _ := json.MarshalIndent(i, "", "\t") 14 | stringified := string(payload) 15 | fmt.Println(stringified) 16 | return stringified 17 | } 18 | 19 | func Stringify(i interface{}) string { 20 | payload, _ := json.Marshal(i) 21 | return string(payload) 22 | } 23 | -------------------------------------------------------------------------------- /pkg/util/pprint_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "encoding/json" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestPprint(t *testing.T) { 15 | something := map[string]interface{}{ 16 | "this": "that", 17 | } 18 | marshaled, _ := json.MarshalIndent(something, "", "\t") 19 | want := string(marshaled) 20 | 21 | got := Pprint(something) 22 | 23 | assert.Equal(t, want, got) 24 | } 25 | 26 | func TestStringify(t *testing.T) { 27 | somethingToStringify := map[string]interface{}{ 28 | "something": "here", 29 | } 30 | marshaled, err := json.Marshal(somethingToStringify) 31 | want := string(marshaled) 32 | got := Stringify(somethingToStringify) 33 | if got != want || err != nil { 34 | t.Fatalf(`Stringify(%v) = %v, want %v, err %v`, somethingToStringify, got, want, err) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /pkg/util/schema.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "github.com/gin-gonic/gin" 9 | "github.com/silverton-io/buz/pkg/constants" 10 | ) 11 | 12 | const JSON_EXTENSION = ".json" 13 | 14 | func GetSchemaNameFromRequest(c *gin.Context, fallback string) string { 15 | schemaName := c.Param(constants.BUZ_SCHEMA_PARAM) 16 | if schemaName == "" || schemaName == "/" { // Handle no param, or trailing slash 17 | return fallback 18 | } 19 | schemaName = schemaName[1:] + JSON_EXTENSION 20 | return schemaName 21 | } 22 | -------------------------------------------------------------------------------- /pkg/util/schema_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "net/http/httptest" 9 | "testing" 10 | 11 | "github.com/gin-gonic/gin" 12 | "github.com/silverton-io/buz/pkg/constants" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestGetSchemaNameFromRequest(t *testing.T) { 17 | someSchema := "some/schema/v1.0" 18 | fallback := "some/fallback/v1.0.json" 19 | rec := httptest.NewRecorder() 20 | c, _ := gin.CreateTestContext(rec) 21 | 22 | schema := GetSchemaNameFromRequest(c, fallback) 23 | 24 | assert.Equal(t, fallback, schema) 25 | 26 | c.Params = append(c.Params, gin.Param{Key: constants.BUZ_SCHEMA_PARAM, Value: "/" + someSchema}) 27 | 28 | schema = GetSchemaNameFromRequest(c, fallback) 29 | 30 | assert.Equal(t, someSchema+JSON_EXTENSION, schema) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/util/structToMap.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "encoding/json" 9 | ) 10 | 11 | func StructToMap(v interface{}) (map[string]interface{}, error) { 12 | var m map[string]interface{} 13 | i, _ := json.Marshal(v) 14 | if err := json.Unmarshal(i, &m); err != nil { 15 | return map[string]interface{}{}, err 16 | } 17 | return m, nil 18 | } 19 | -------------------------------------------------------------------------------- /pkg/util/structToMap_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "reflect" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | type testStruct struct { 15 | A string `json:"a"` 16 | } 17 | 18 | // TestStructToMap ensures the proper map[string]interface{} is 19 | // generated after calling it with an arbitrary struct. 20 | func TestStructToMapGood(t *testing.T) { 21 | var s string = "something" 22 | ts := testStruct{ 23 | A: s, 24 | } 25 | want := map[string]interface{}{ 26 | "a": s, 27 | } 28 | got, _ := StructToMap(ts) 29 | equivalent := reflect.DeepEqual(got, want) 30 | if !equivalent { 31 | t.Fatalf(`StructToMap(%v) = %v, want %v`, ts, got, want) 32 | } 33 | 34 | } 35 | 36 | func TestStructToMapBad(t *testing.T) { 37 | something := "this" 38 | want := map[string]interface{}{} 39 | 40 | got, err := StructToMap(something) 41 | 42 | assert.Equal(t, want, got) 43 | assert.Error(t, err) 44 | } 45 | -------------------------------------------------------------------------------- /pkg/util/time.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import "time" 8 | 9 | // GetDuration returns the associated duration between two times 10 | func GetDuration(start time.Time, end time.Time) time.Duration { 11 | return end.Sub(start) 12 | } 13 | -------------------------------------------------------------------------------- /pkg/util/time_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package util 6 | 7 | import ( 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func TestGetDuration(t *testing.T) { 13 | want := 2 * time.Second 14 | start := time.Now().UTC() 15 | end := start.Add(want) 16 | duration := GetDuration(start, end) 17 | if duration != want { 18 | t.Fatalf(`GetDuration(%v, %v) = %v, want %v`, start, end, duration, want) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /pkg/validator/invalidMessages.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2022 Silverton Data, Inc. 2 | // You may use, distribute, and modify this code under the terms of the Apache-2.0 license, a copy of 3 | // which may be found at https://github.com/silverton-io/buz/blob/main/LICENSE 4 | 5 | package validator 6 | 7 | type InvalidMessage struct { 8 | Type string `json:"type"` 9 | Resolution string `json:"resolution"` 10 | } 11 | 12 | var InvalidSchema = InvalidMessage{ 13 | Type: "invalid schema", 14 | Resolution: "ensure schema is properly formatted", 15 | } 16 | 17 | var InvalidPayload = InvalidMessage{ 18 | Type: "invalid payload", 19 | Resolution: "publish a valid payload", 20 | } 21 | 22 | var PayloadNotPresent = InvalidMessage{ 23 | Type: "payload not present", 24 | Resolution: "publish the event with a payload", 25 | } 26 | 27 | var UnknownSnowplowEventType = InvalidMessage{ 28 | Type: "unknown snowplow event type", 29 | Resolution: "event type should adhere to the snowplow tracker protocol", 30 | } 31 | 32 | var NoSchemaAssociated = InvalidMessage{ 33 | Type: "no schema associated", 34 | Resolution: "associate a schema with the event", 35 | } 36 | 37 | var NoSchemaInBackend = InvalidMessage{ 38 | Type: "schema not published to cache backend", 39 | Resolution: "publish schema to the cache backend", 40 | } 41 | -------------------------------------------------------------------------------- /schemas/com.github/hook/repository/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "com.github/hook/repository/v1.0.json", 4 | "title": "com.github/hook/repository/v1.0.json", 5 | "description": "Wrapper for repository-level Github hooks", 6 | "owner": { 7 | "org": "silverton", 8 | "team": "buz", 9 | "individual": "jakthom" 10 | }, 11 | "self": { 12 | "vendor": "com.github", 13 | "namespace": "hook.repository", 14 | "version": "1.0" 15 | }, 16 | "type": "object", 17 | "properties": {}, 18 | "additionalProperties": true 19 | } -------------------------------------------------------------------------------- /schemas/com.stripe/hook/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "com.stripe/hook/v1.0.json", 4 | "title": "com.stripe/hook/v1.0.json", 5 | "description": "Wrapper for Stripe hooks", 6 | "owner": { 7 | "org": "silverton", 8 | "team": "buz", 9 | "individual": "jakthom" 10 | }, 11 | "self": { 12 | "vendor": "com.stripe", 13 | "namespace": "hook", 14 | "version": "1.0" 15 | }, 16 | "type": "object", 17 | "properties": {}, 18 | "additionalProperties": true 19 | } -------------------------------------------------------------------------------- /schemas/io.silverton/buz/example/generic/sample/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/buz/example/generic/sample/v1.0.json", 4 | "title": "io.silverton/buz/example/generic/sample/v1.0.json", 5 | "description": "Schema for testing generic events", 6 | "owner": { 7 | "org": "silverton", 8 | "team": "buz", 9 | "individual": "jakthom" 10 | }, 11 | "self": { 12 | "vendor": "io.silverton", 13 | "namespace": "buz.example.generic.sample", 14 | "version": "1.0" 15 | }, 16 | "type": "object", 17 | "properties": { 18 | "id": { 19 | "description": "The id", 20 | "type": "string", 21 | "minLength": 0, 22 | "maxLength": 4096 23 | } 24 | }, 25 | "required": [ 26 | "id" 27 | ], 28 | "additionalProperties": false 29 | } -------------------------------------------------------------------------------- /schemas/io.silverton/buz/example/gettingStarted/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/buz/example/gettingStarted/v1.0.json", 4 | "title": "io.silverton/buz/example/gettingStarted/v1.0.json", 5 | "description": "A getting started event used to bootstrap and demonstrate validation", 6 | "owner": { 7 | "org": "silverton", 8 | "team": "buz", 9 | "individual": "jakthom" 10 | }, 11 | "self": { 12 | "vendor": "io.silverton", 13 | "namespace": "buz.example.quickstart.click", 14 | "version": "1.0" 15 | }, 16 | "type": "object", 17 | "properties": { 18 | "userId": { 19 | "type": "integer", 20 | "description": "The id of the user" 21 | }, 22 | "name": { 23 | "type": "string", 24 | "description": "The name of the user" 25 | }, 26 | "action": { 27 | "type": "string", 28 | "description": "The associated user action" 29 | } 30 | }, 31 | "additionalProperties": false, 32 | "required": [ 33 | "userId", 34 | "name", 35 | "action" 36 | ] 37 | } -------------------------------------------------------------------------------- /schemas/io.silverton/buz/example/productView/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/buz/example/productView/v1.0.json", 4 | "title": "io.silverton/buz/example/productView/v1.0.json", 5 | "description": "Schema for product views", 6 | "owner": { 7 | "org": "silverton", 8 | "team": "buz", 9 | "individual": "jakthom" 10 | }, 11 | "self": { 12 | "vendor": "io.silverton", 13 | "namespace": "buz.example.productView", 14 | "version": "1.0" 15 | }, 16 | "type": "object", 17 | "properties": { 18 | "productId": { 19 | "description": "The product id", 20 | "type": "string", 21 | "minLength": 0, 22 | "maxLength": 4096 23 | }, 24 | "category": { 25 | "description": "The category", 26 | "type": "string" 27 | }, 28 | "brand": { 29 | "description": "The brand", 30 | "type": "string" 31 | }, 32 | "returning": { 33 | "description": "Whether or not the customer is a return visit", 34 | "type": "boolean" 35 | }, 36 | "price": { 37 | "description": "The price of the product viewed", 38 | "type": "number" 39 | }, 40 | "sizes": { 41 | "description": "The available sizes", 42 | "type": "array", 43 | "items": { 44 | "type": "string" 45 | } 46 | }, 47 | "availableSince": { 48 | "description": "The availability date of the product", 49 | "type": "string", 50 | "format": "date-time" 51 | } 52 | }, 53 | "required": [ 54 | "productId", 55 | "category", 56 | "brand", 57 | "returning", 58 | "price", 59 | "sizes", 60 | "availableSince" 61 | ], 62 | "additionalProperties": false 63 | } -------------------------------------------------------------------------------- /schemas/io.silverton/buz/hook/arbitrary/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/buz/hook/arbitrary/v1.0.json", 4 | "title": "io.silverton/buz/hook/arbitrary/v1.0.json", 5 | "description": "A catchall schema for unnamed events consisting of arbitrary webhook payloads.", 6 | "owner": { 7 | "org": "silverton", 8 | "team": "buz", 9 | "individual": "jakthom" 10 | }, 11 | "self": { 12 | "vendor": "io.silverton", 13 | "namespace": "buz.hook.arbitrary", 14 | "version": "1.0" 15 | }, 16 | "type": "object", 17 | "properties": {}, 18 | "additionalProperties": true, 19 | "required": [] 20 | } -------------------------------------------------------------------------------- /schemas/io.silverton/buz/internal/contexts/httpHeaders/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/buz/internal/contexts/httpHeaders/v1.0.json", 4 | "title": "io.silverton/buz/internal/contexts/httpHeaders/v1.0.json", 5 | "description": "HTTP headers context", 6 | "owner": { 7 | "org": "silverton", 8 | "team": "buz", 9 | "individual": "jakthom" 10 | }, 11 | "self": { 12 | "vendor": "io.silverton", 13 | "namespace": "buz.internal.contexts.httpHeaders", 14 | "version": "1.0" 15 | }, 16 | "type": "object", 17 | "properties": {}, 18 | "additionalProperties": true 19 | } -------------------------------------------------------------------------------- /schemas/io.silverton/buz/internal/tele/beat/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/buz/tele/beat/v1.0.json", 4 | "title": "io.silverton/buz/tele/beat/v1.0.json", 5 | "description": "Buz heartbeat event", 6 | "owner": { 7 | "org": "silverton", 8 | "team": "buz", 9 | "individual": "jakthom" 10 | }, 11 | "self": { 12 | "vendor": "io.silverton", 13 | "namespace": "buz.internal.tele.beat", 14 | "version": "1.0" 15 | }, 16 | "type": "object", 17 | "properties": { 18 | "meta": { 19 | "type": "object", 20 | "description": "Metadata about a beat" 21 | }, 22 | "time": { 23 | "type": "string", 24 | "format": "date-time", 25 | "description": "TS of the beat" 26 | }, 27 | "elapsedSeconds": { 28 | "type": "number", 29 | "description": "The number of elapsed seconds since startup" 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /schemas/io.silverton/buz/internal/tele/shutdown/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/buz/tele/shutdown/v1.0.json", 4 | "title": "io.silverton/buz/tele/shutdown/v1.0.json", 5 | "description": "Buz shutdown event", 6 | "owner": { 7 | "org": "silverton", 8 | "team": "buz", 9 | "individual": "jakthom" 10 | }, 11 | "self": { 12 | "vendor": "io.silverton", 13 | "namespace": "buz.internal.tele.shutdown", 14 | "version": "1.0" 15 | }, 16 | "type": "object", 17 | "properties": { 18 | "meta": { 19 | "type": "object", 20 | "description": "Metadata about the instance being shut down" 21 | }, 22 | "time": { 23 | "type": "string", 24 | "format": "date-time", 25 | "description": "TS of the shutdown" 26 | }, 27 | "elapsedSeconds": { 28 | "type": "number", 29 | "description": "The number of elapsed seconds since startup" 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /schemas/io.silverton/buz/internal/tele/startup/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/buz/internal/tele/startup/v1.0.json", 4 | "title": "io.silverton/buz/internal/tele/startup/v1.0.json", 5 | "description": "Buz startup event", 6 | "owner": { 7 | "org": "silverton", 8 | "team": "buz", 9 | "individual": "jakthom" 10 | }, 11 | "self": { 12 | "vendor": "io.silverton", 13 | "namespace": "buz.internal.tele.startup", 14 | "version": "1.0" 15 | }, 16 | "type": "object", 17 | "properties": { 18 | "config": { 19 | "type": "object", 20 | "description": "Configuration at time of startup" 21 | }, 22 | "meta": { 23 | "type": "object", 24 | "description": "Startup metadata" 25 | }, 26 | "time": { 27 | "type": "string", 28 | "format": "date-time", 29 | "description": "TS of the beat" 30 | } 31 | }, 32 | "additionalProperties": false, 33 | "required": [ 34 | "meta", 35 | "time", 36 | "config" 37 | ] 38 | } -------------------------------------------------------------------------------- /schemas/io.silverton/buz/pixel/arbitrary/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/buz/pixel/arbitrary/v1.0.json", 4 | "title": "io.silverton/buz/pixel/arbitrary/v1.0.json", 5 | "description": "A catchall schema for unnamed events consisting of arbitrary pixel payloads.", 6 | "owner": { 7 | "org": "silverton", 8 | "team": "buz", 9 | "individual": "jakthom" 10 | }, 11 | "self": { 12 | "vendor": "io.silverton", 13 | "namespace": "buz.pixel.arbitrary", 14 | "version": "1.0" 15 | }, 16 | "type": "object", 17 | "properties": {}, 18 | "additionalProperties": true, 19 | "required": [] 20 | } -------------------------------------------------------------------------------- /schemas/io.silverton/buz/pixel/linkClick/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/buz/pixel/linkClick/v1.0.json", 4 | "title": "io.silverton/buz/pixel/linkClick/v1.0.json", 5 | "description": "A generalized schema for link clicks tracked via pixel", 6 | "owner": { 7 | "org": "silverton", 8 | "team": "buz", 9 | "individual": "jakthom" 10 | }, 11 | "self": { 12 | "vendor": "io.silverton", 13 | "namespace": "buz.pixel.linkClick", 14 | "version": "1.0" 15 | }, 16 | "type": "object", 17 | "properties": { 18 | "rto": { 19 | "type": "string", 20 | "description": "The link that was redirected to" 21 | } 22 | }, 23 | "additionalProperties": true, 24 | "required": [ 25 | "link" 26 | ] 27 | } -------------------------------------------------------------------------------- /schemas/io.silverton/buz/pixel/pageView/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/buz/pixel/pageView/v1.0.json", 4 | "title": "io.silverton/buz/pixel/pageView/v1.0.json", 5 | "description": "A generalized schema for page views tracked via pixel", 6 | "owner": { 7 | "org": "silverton", 8 | "team": "buz", 9 | "individual": "jakthom" 10 | }, 11 | "self": { 12 | "vendor": "io.silverton", 13 | "namespace": "buz.pixel.pageView", 14 | "version": "1.0" 15 | }, 16 | "type": "object", 17 | "properties": { 18 | "page": { 19 | "type": "string", 20 | "description": "The page" 21 | } 22 | }, 23 | "additionalProperties": true, 24 | "required": [ 25 | "page" 26 | ] 27 | } -------------------------------------------------------------------------------- /schemas/io.silverton/snowplow/page_ping/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/snowplow/page_ping/v1.0.json", 4 | "title": "io.silverton/snowplow/page_ping/v1.0.json", 5 | "description": "Snowplow Page Ping Event", 6 | "self": { 7 | "vendor": "io.silverton", 8 | "namespace": "snowplow.pagePing", 9 | "version": "1.0" 10 | }, 11 | "type": "object", 12 | "properties": { 13 | "pp_xoffset_min": { 14 | "type": [ 15 | "number", 16 | "null" 17 | ], 18 | "description": "The minimum X offset" 19 | }, 20 | "pp_xoffset_max": { 21 | "type": [ 22 | "number", 23 | "null" 24 | ], 25 | "description": "The maximum X offset" 26 | }, 27 | "pp_yoffset_min": { 28 | "type": [ 29 | "number", 30 | "null" 31 | ], 32 | "description": "The minimum Y offset" 33 | }, 34 | "pp_yoffset_max": { 35 | "type": [ 36 | "number", 37 | "null" 38 | ], 39 | "description": "The maximum Y offset" 40 | } 41 | }, 42 | "required": [ 43 | "pp_xoffset_min", 44 | "pp_xoffset_max", 45 | "pp_yoffset_min", 46 | "pp_yoffset_max" 47 | ], 48 | "additionalProperties": false 49 | } -------------------------------------------------------------------------------- /schemas/io.silverton/snowplow/page_view/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/snowplow/page_view/v1.0.json", 4 | "title": "io.silverton/snowplow/page_view/v1.0.json", 5 | "description": "Snowplow Page View Event", 6 | "self": { 7 | "vendor": "io.silverton", 8 | "namespace": "snowplow.pageView", 9 | "version": "1.0" 10 | }, 11 | "type": "object", 12 | "properties": {}, 13 | "additionalProperties": false 14 | } -------------------------------------------------------------------------------- /schemas/io.silverton/snowplow/struct/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/snowplow/struct/v1.0.json", 4 | "title": "io.silverton/snowplow/struct/v1.0.json", 5 | "description": "Snowplow Struct Event", 6 | "self": { 7 | "vendor": "io.silverton", 8 | "namespace": "snowplow.structEvent", 9 | "version": "1.0" 10 | }, 11 | "type": "object", 12 | "properties": { 13 | "se_category": { 14 | "type": "string", 15 | "description": "The struct event category" 16 | }, 17 | "se_action": { 18 | "type": "string", 19 | "description": "The struct event action" 20 | }, 21 | "se_label": { 22 | "type": [ 23 | "string", 24 | "null" 25 | ], 26 | "description": "The struct event label" 27 | }, 28 | "se_property": { 29 | "type": [ 30 | "string", 31 | "null" 32 | ], 33 | "description": "The struct event property" 34 | }, 35 | "se_value": { 36 | "type": [ 37 | "number", 38 | "null" 39 | ], 40 | "description": "The struct event value" 41 | } 42 | }, 43 | "required": [ 44 | "se_category", 45 | "se_action" 46 | ], 47 | "additionalProperties": false 48 | } -------------------------------------------------------------------------------- /schemas/io.silverton/snowplow/transaction/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/snowplow/transaction/v1.0.json", 4 | "title": "io.silverton/snowplow/transaction/v1.0.json", 5 | "description": "Snowplow Transaction Event", 6 | "self": { 7 | "vendor": "io.silverton", 8 | "namespace": "snowplow.transaction", 9 | "version": "1.0" 10 | }, 11 | "type": "object", 12 | "properties": { 13 | "tr_orderid": { 14 | "type": "string", 15 | "description": "The transaction order id" 16 | }, 17 | "tr_affiliation": { 18 | "type": [ 19 | "string", 20 | "null" 21 | ], 22 | "description": "The transaction affiliation" 23 | }, 24 | "tr_total": { 25 | "type": "number", 26 | "description": "The transaction total" 27 | }, 28 | "tr_tax": { 29 | "type": [ 30 | "number", 31 | "null" 32 | ], 33 | "description": "The transaction tax" 34 | }, 35 | "tr_shipping": { 36 | "type": [ 37 | "number", 38 | "null" 39 | ], 40 | "description": "The transaction shipping" 41 | }, 42 | "tr_city": { 43 | "type": [ 44 | "number", 45 | "null" 46 | ], 47 | "description": "The transaction city" 48 | }, 49 | "tr_state": { 50 | "type": [ 51 | "number", 52 | "null" 53 | ], 54 | "description": "The transaction state" 55 | }, 56 | "tr_country": { 57 | "type": [ 58 | "number", 59 | "null" 60 | ], 61 | "description": "The transaction country" 62 | }, 63 | "tr_currency": { 64 | "type": [ 65 | "number", 66 | "null" 67 | ], 68 | "description": "The transaction currency" 69 | } 70 | }, 71 | "required": [ 72 | "tr_orderid", 73 | "tr_total" 74 | ], 75 | "additionalProperties": false 76 | } -------------------------------------------------------------------------------- /schemas/io.silverton/snowplow/transaction_item/v1.0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/silverton-io/buz/main/schemas/io.silverton/buz/internal/meta/v1.0.json", 3 | "$id": "io.silverton/snowplow/transaction_item/v1.0.json", 4 | "title": "io.silverton/snowplow/transaction_item/v1.0.json", 5 | "description": "Snowplow Transaction Item Event", 6 | "self": { 7 | "vendor": "io.silverton", 8 | "namespace": "snowplow.transactionItem", 9 | "version": "1.0" 10 | }, 11 | "type": "object", 12 | "properties": { 13 | "ti_orderid": { 14 | "type": "string", 15 | "description": "The transaction item order id" 16 | }, 17 | "ti_sku": { 18 | "type": "string", 19 | "description": "The transaction item sku" 20 | }, 21 | "ti_name": { 22 | "type": [ 23 | "string", 24 | "null" 25 | ], 26 | "description": "The transaction item name" 27 | }, 28 | "ti_category": { 29 | "type": [ 30 | "string", 31 | "null" 32 | ], 33 | "description": "The transaction item category" 34 | }, 35 | "ti_price": { 36 | "type": "number", 37 | "description": "The transaction item price" 38 | }, 39 | "ti_quantity": { 40 | "type": "number", 41 | "description": "The transaction item quantity" 42 | }, 43 | "ti_currency": { 44 | "type": [ 45 | "string", 46 | "null" 47 | ], 48 | "description": "The transaction item currency" 49 | } 50 | }, 51 | "required": [], 52 | "additionalProperties": false 53 | } --------------------------------------------------------------------------------