├── docs └── dash.png ├── pkg ├── models │ ├── zstd_dictionary │ └── models.go ├── client │ ├── metrics.go │ ├── schedulers │ │ ├── metrics.go │ │ ├── sequential │ │ │ └── sequential.go │ │ └── parallel │ │ │ └── parallel.go │ └── client.go ├── server │ ├── metrics.go │ ├── subscriber_test.go │ ├── subscriber.go │ └── server.go ├── monotonic │ └── clock.go └── consumer │ ├── metrics.go │ ├── persist.go │ └── consumer.go ├── train.sh ├── docker-compose.yaml ├── .gitignore ├── Makefile ├── Dockerfile ├── LICENSE ├── cmd ├── client │ └── main.go └── jetstream │ └── main.go ├── .github └── workflows │ └── docker-image.yml ├── go.mod ├── README.md ├── go.sum └── grafana-dashboard.json /docs/dash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bluesky-social/jetstream/HEAD/docs/dash.png -------------------------------------------------------------------------------- /pkg/models/zstd_dictionary: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bluesky-social/jetstream/HEAD/pkg/models/zstd_dictionary -------------------------------------------------------------------------------- /train.sh: -------------------------------------------------------------------------------- 1 | i=0; websocat "ws://localhost:6008/subscribe?cursor=0" | while IFS= read -r line; do 2 | echo "$line" > "training/$i.json" 3 | i=$((i+1)) 4 | if [ "$i" -ge 100000 ]; then 5 | break 6 | fi 7 | done 8 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | jetstream: 3 | image: ghcr.io/bluesky-social/jetstream:${JETSTREAM_VERSION} 4 | build: 5 | context: . 6 | dockerfile: Dockerfile 7 | restart: always 8 | container_name: jetstream 9 | network_mode: host 10 | volumes: 11 | - ./data:/data 12 | environment: 13 | - JETSTREAM_DATA_DIR=/data 14 | # livness check interval to restart when no events are received (default: 15sec) 15 | - JETSTREAM_LIVENESS_TTL=15s 16 | -------------------------------------------------------------------------------- /pkg/client/metrics.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | var bytesRead = promauto.NewCounterVec(prometheus.CounterOpts{ 9 | Name: "jetstream_client_bytes_read", 10 | Help: "The total number of bytes read from the server", 11 | }, []string{"client"}) 12 | 13 | var eventsRead = promauto.NewCounterVec(prometheus.CounterOpts{ 14 | Name: "jetstream_client_events_read", 15 | Help: "The total number of events read from the server", 16 | }, []string{"client"}) 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # VSCode 2 | .vscode 3 | 4 | # If you prefer the allow list template instead of the deny list, see community template: 5 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 6 | # 7 | # Binaries for programs and plugins 8 | *.exe 9 | *.exe~ 10 | *.dll 11 | *.so 12 | *.dylib 13 | 14 | # Test binary, built with `go test -c` 15 | *.test 16 | 17 | # Output of the go coverage tool, specifically when used with LiteIDE 18 | *.out 19 | 20 | # Dependency directories (remove the comment below to include it) 21 | # vendor/ 22 | 23 | # Go workspace file 24 | go.work 25 | 26 | 27 | .env 28 | data/ 29 | training/ 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GO_CMD_W_CGO = CGO_ENABLED=1 GOOS=linux go 2 | GO_CMD = CGO_ENABLED=0 GOOS=linux go 3 | JETSTREAM_VERSION = sha-$(shell git rev-parse HEAD) 4 | 5 | # Build Jetstream 6 | .PHONY: build 7 | build: 8 | @echo "Building Jetstream Go binary..." 9 | $(GO_CMD_W_CGO) build -o jetstream cmd/jetstream/*.go 10 | 11 | # Run Jetstream 12 | .PHONY: run 13 | run: .env 14 | @echo "Running Jetstream..." 15 | $(GO_CMD_W_CGO) run cmd/jetstream/*.go 16 | 17 | .PHONY: up 18 | up: 19 | @echo "Starting Jetstream..." 20 | JETSTREAM_VERSION=${JETSTREAM_VERSION} docker compose up -d 21 | 22 | .PHONY: rebuild 23 | rebuild: 24 | @echo "Starting Jetstream..." 25 | JETSTREAM_VERSION=${JETSTREAM_VERSION} docker compose up -d --build 26 | 27 | .PHONY: down 28 | down: 29 | @echo "Stopping Jetstream..." 30 | JETSTREAM_VERSION=${JETSTREAM_VERSION} docker compose down 31 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Stage 1: Build the Go binary 2 | FROM golang:1.24.6 AS builder 3 | 4 | # Create a directory for the application 5 | WORKDIR /app 6 | 7 | # Fetch dependencies 8 | COPY go.mod go.sum ./ 9 | 10 | RUN go mod download 11 | 12 | COPY pkg ./pkg 13 | 14 | COPY cmd/jetstream ./cmd/jetstream 15 | 16 | COPY Makefile ./ 17 | 18 | # Build the application 19 | RUN make build 20 | 21 | # Stage 2: Import SSL certificates 22 | FROM alpine:latest as certs 23 | 24 | RUN apk --update add ca-certificates 25 | 26 | # Stage 3: Build a minimal Docker image 27 | FROM debian:stable-slim 28 | 29 | # Import the SSL certificates from the first stage. 30 | COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 31 | 32 | # Copy the binary from the first stage. 33 | COPY --from=builder /app/jetstream . 34 | 35 | # Set the startup command to run the binary 36 | CMD ["./jetstream"] 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Jaz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /pkg/client/schedulers/metrics.go: -------------------------------------------------------------------------------- 1 | package schedulers 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | var WorkItemsAdded = promauto.NewCounterVec(prometheus.CounterOpts{ 9 | Name: "jetstream_scheduler_work_items_added_total", 10 | Help: "Total number of work items added to the consumer pool", 11 | }, []string{"pool", "scheduler_type"}) 12 | 13 | var WorkItemsProcessed = promauto.NewCounterVec(prometheus.CounterOpts{ 14 | Name: "jetstream_scheduler_work_items_processed_total", 15 | Help: "Total number of work items processed by the consumer pool", 16 | }, []string{"pool", "scheduler_type"}) 17 | 18 | var WorkItemsActive = promauto.NewCounterVec(prometheus.CounterOpts{ 19 | Name: "jetstream_scheduler_work_items_active_total", 20 | Help: "Total number of work items passed into a worker", 21 | }, []string{"pool", "scheduler_type"}) 22 | 23 | var WorkersActive = promauto.NewGaugeVec(prometheus.GaugeOpts{ 24 | Name: "jetstream_scheduler_workers_active", 25 | Help: "Number of workers currently active", 26 | }, []string{"pool", "scheduler_type"}) 27 | -------------------------------------------------------------------------------- /pkg/server/metrics.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | var subscribersConnected = promauto.NewGaugeVec(prometheus.GaugeOpts{ 9 | Name: "jetstream_subscribers_connected", 10 | Help: "The number of subscribers connected to the Jetstream server", 11 | }, []string{"ip_address"}) 12 | 13 | var eventsEmitted = promauto.NewCounter(prometheus.CounterOpts{ 14 | Name: "jetstream_events_emitted_total", 15 | Help: "The total number of events emitted by the Jetstream server", 16 | }) 17 | 18 | var bytesEmitted = promauto.NewCounter(prometheus.CounterOpts{ 19 | Name: "jetstream_bytes_emitted_total", 20 | Help: "The total number of bytes emitted by the Jetstream server", 21 | }) 22 | 23 | var eventsDelivered = promauto.NewCounterVec(prometheus.CounterOpts{ 24 | Name: "jetstream_events_delivered_total", 25 | Help: "The total number of events delivered by the Jetstream server", 26 | }, []string{"ip_address"}) 27 | 28 | var bytesDelivered = promauto.NewCounterVec(prometheus.CounterOpts{ 29 | Name: "jetstream_bytes_delivered_total", 30 | Help: "The total number of bytes delivered by the Jetstream server", 31 | }, []string{"ip_address"}) 32 | -------------------------------------------------------------------------------- /pkg/models/models.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "github.com/goccy/go-json" 5 | 6 | _ "embed" 7 | 8 | comatproto "github.com/bluesky-social/indigo/api/atproto" 9 | ) 10 | 11 | //go:embed zstd_dictionary 12 | var ZSTDDictionary []byte 13 | 14 | type Event struct { 15 | Did string `json:"did"` 16 | TimeUS int64 `json:"time_us"` 17 | Kind string `json:"kind,omitempty"` 18 | Commit *Commit `json:"commit,omitempty"` 19 | Account *comatproto.SyncSubscribeRepos_Account `json:"account,omitempty"` 20 | Identity *comatproto.SyncSubscribeRepos_Identity `json:"identity,omitempty"` 21 | } 22 | 23 | type Commit struct { 24 | Rev string `json:"rev,omitempty"` 25 | Operation string `json:"operation,omitempty"` 26 | Collection string `json:"collection,omitempty"` 27 | RKey string `json:"rkey,omitempty"` 28 | Record json.RawMessage `json:"record,omitempty"` 29 | CID string `json:"cid,omitempty"` 30 | } 31 | 32 | var ( 33 | EventKindCommit = "commit" 34 | EventKindAccount = "account" 35 | EventKindIdentity = "identity" 36 | 37 | CommitOperationCreate = "create" 38 | CommitOperationUpdate = "update" 39 | CommitOperationDelete = "delete" 40 | ) 41 | -------------------------------------------------------------------------------- /pkg/monotonic/clock.go: -------------------------------------------------------------------------------- 1 | // Package monotonic provides a monotonic clock. 2 | // The clock is safe for concurrent use and can be used to generate 3 | // increasing timestamps as int64s with a given precision. 4 | package monotonic 5 | 6 | import ( 7 | "fmt" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | // Clock is a monotonic clock that generates increasing timestamps. 13 | type Clock struct { 14 | precision time.Duration 15 | lk sync.Mutex 16 | last int64 17 | } 18 | 19 | // NewClock creates a new Clock with the given precision. 20 | func NewClock(precision time.Duration) (*Clock, error) { 21 | switch precision { 22 | case time.Second, time.Millisecond, time.Microsecond, time.Nanosecond: 23 | default: 24 | return nil, fmt.Errorf("invalid precision: %v", precision) 25 | } 26 | 27 | return &Clock{ 28 | precision: precision, 29 | }, nil 30 | } 31 | 32 | // Now returns the current timestamp as an int64 of the Clock's precision. 33 | // Now will always return a monotonically increasing value. 34 | func (c *Clock) Now() int64 { 35 | c.lk.Lock() 36 | defer c.lk.Unlock() 37 | 38 | s := time.Now() 39 | var now int64 40 | switch c.precision { 41 | case time.Second: 42 | now = s.Unix() 43 | case time.Millisecond: 44 | now = s.UnixMilli() 45 | case time.Microsecond: 46 | now = s.UnixMicro() 47 | case time.Nanosecond: 48 | now = s.UnixNano() 49 | } 50 | 51 | if now <= c.last { 52 | now = c.last + 1 53 | } 54 | c.last = now 55 | return now 56 | } 57 | -------------------------------------------------------------------------------- /pkg/client/schedulers/sequential/sequential.go: -------------------------------------------------------------------------------- 1 | package sequential 2 | 3 | import ( 4 | "context" 5 | "log/slog" 6 | 7 | "github.com/bluesky-social/jetstream/pkg/client/schedulers" 8 | "github.com/bluesky-social/jetstream/pkg/models" 9 | "github.com/prometheus/client_golang/prometheus" 10 | ) 11 | 12 | // Scheduler is a sequential scheduler that will run work on a single worker 13 | type Scheduler struct { 14 | handleEvent func(context.Context, *models.Event) error 15 | 16 | ident string 17 | logger *slog.Logger 18 | 19 | // metrics 20 | itemsAdded prometheus.Counter 21 | itemsProcessed prometheus.Counter 22 | itemsActive prometheus.Counter 23 | workersActive prometheus.Gauge 24 | } 25 | 26 | func NewScheduler(ident string, logger *slog.Logger, handleEvent func(context.Context, *models.Event) error) *Scheduler { 27 | logger = logger.With("component", "sequential-scheduler", "ident", ident) 28 | p := &Scheduler{ 29 | handleEvent: handleEvent, 30 | 31 | ident: ident, 32 | logger: logger, 33 | 34 | itemsAdded: schedulers.WorkItemsAdded.WithLabelValues(ident, "sequential"), 35 | itemsProcessed: schedulers.WorkItemsProcessed.WithLabelValues(ident, "sequential"), 36 | itemsActive: schedulers.WorkItemsActive.WithLabelValues(ident, "sequential"), 37 | workersActive: schedulers.WorkersActive.WithLabelValues(ident, "sequential"), 38 | } 39 | 40 | p.workersActive.Set(1) 41 | 42 | return p 43 | } 44 | 45 | func (p *Scheduler) Shutdown() { 46 | p.workersActive.Set(0) 47 | } 48 | 49 | func (s *Scheduler) AddWork(ctx context.Context, repo string, val *models.Event) error { 50 | s.itemsAdded.Inc() 51 | s.itemsActive.Inc() 52 | err := s.handleEvent(ctx, val) 53 | s.itemsProcessed.Inc() 54 | return err 55 | } 56 | -------------------------------------------------------------------------------- /pkg/consumer/metrics.go: -------------------------------------------------------------------------------- 1 | package consumer 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | var eventsProcessedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ 9 | Name: "consumer_events_processed_total", 10 | Help: "The total number of firehose events processed by Consumer", 11 | }, []string{"event_type", "socket_url"}) 12 | 13 | var opsProcessedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ 14 | Name: "consumer_ops_processed_total", 15 | Help: "The total number of repo operations processed by Consumer", 16 | }, []string{"kind", "op_path", "socket_url"}) 17 | 18 | var eventProcessingDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{ 19 | Name: "consumer_event_processing_duration_seconds", 20 | Help: "The amount of time it takes to process a firehose event", 21 | Buckets: prometheus.ExponentialBuckets(0.0001, 2, 18), 22 | }, []string{"socket_url"}) 23 | 24 | var lastSeqGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ 25 | Name: "consumer_last_seq", 26 | Help: "The sequence number of the last event processed", 27 | }, []string{"socket_url"}) 28 | 29 | var lastEvtProcessedAtGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ 30 | Name: "consumer_last_evt_processed_at", 31 | Help: "The timestamp of the last event processed", 32 | }, []string{"socket_url"}) 33 | 34 | var lastEvtCreatedAtGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ 35 | Name: "consumer_last_evt_created_at", 36 | Help: "The timestamp of the last event created", 37 | }, []string{"socket_url"}) 38 | 39 | var lastEvtCreatedEvtProcessedGapGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{ 40 | Name: "consumer_last_evt_created_evt_processed_gap", 41 | Help: "The gap between the last event's event timestamp and when it was processed by consumer", 42 | }, []string{"socket_url"}) 43 | 44 | var eventsSequencedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ 45 | Name: "consumer_events_sequenced_total", 46 | Help: "The total number of events sequenced", 47 | }, []string{"socket_url"}) 48 | 49 | var eventsPersistedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ 50 | Name: "consumer_events_persisted_total", 51 | Help: "The total number of events persisted", 52 | }, []string{"socket_url"}) 53 | 54 | var eventsEmittedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ 55 | Name: "consumer_events_emitted_total", 56 | Help: "The total number of events emitted", 57 | }, []string{"socket_url"}) 58 | -------------------------------------------------------------------------------- /cmd/client/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "log/slog" 9 | "os" 10 | "time" 11 | 12 | apibsky "github.com/bluesky-social/indigo/api/bsky" 13 | "github.com/bluesky-social/jetstream/pkg/client" 14 | "github.com/bluesky-social/jetstream/pkg/client/schedulers/sequential" 15 | "github.com/bluesky-social/jetstream/pkg/models" 16 | ) 17 | 18 | const ( 19 | serverAddr = "wss://jetstream.atproto.tools/subscribe" 20 | ) 21 | 22 | func main() { 23 | ctx := context.Background() 24 | slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ 25 | Level: slog.LevelInfo, 26 | AddSource: true, 27 | }))) 28 | logger := slog.Default() 29 | 30 | config := client.DefaultClientConfig() 31 | config.WebsocketURL = serverAddr 32 | config.Compress = true 33 | 34 | h := &handler{ 35 | seenSeqs: make(map[int64]struct{}), 36 | } 37 | 38 | scheduler := sequential.NewScheduler("jetstream_localdev", logger, h.HandleEvent) 39 | 40 | c, err := client.NewClient(config, logger, scheduler) 41 | if err != nil { 42 | log.Fatalf("failed to create client: %v", err) 43 | } 44 | 45 | cursor := time.Now().Add(5 * -time.Minute).UnixMicro() 46 | 47 | // Every 5 seconds print the events read and bytes read and average event size 48 | go func() { 49 | ticker := time.NewTicker(5 * time.Second) 50 | for { 51 | select { 52 | case <-ticker.C: 53 | eventsRead := c.EventsRead.Load() 54 | bytesRead := c.BytesRead.Load() 55 | avgEventSize := bytesRead / max(1, eventsRead) 56 | logger.Info("stats", "events_read", eventsRead, "bytes_read", bytesRead, "avg_event_size", avgEventSize) 57 | } 58 | } 59 | }() 60 | 61 | if err := c.ConnectAndRead(ctx, &cursor); err != nil { 62 | log.Fatalf("failed to connect: %v", err) 63 | } 64 | 65 | slog.Info("shutdown") 66 | } 67 | 68 | type handler struct { 69 | seenSeqs map[int64]struct{} 70 | highwater int64 71 | } 72 | 73 | func (h *handler) HandleEvent(ctx context.Context, event *models.Event) error { 74 | // Unmarshal the record if there is one 75 | if event.Commit != nil && (event.Commit.Operation == models.CommitOperationCreate || event.Commit.Operation == models.CommitOperationUpdate) { 76 | switch event.Commit.Collection { 77 | case "app.bsky.feed.post": 78 | var post apibsky.FeedPost 79 | if err := json.Unmarshal(event.Commit.Record, &post); err != nil { 80 | return fmt.Errorf("failed to unmarshal post: %w", err) 81 | } 82 | fmt.Printf("%v |(%s)| %s\n", time.UnixMicro(event.TimeUS).Local().Format("15:04:05"), event.Did, post.Text) 83 | } 84 | } 85 | 86 | return nil 87 | } 88 | -------------------------------------------------------------------------------- /.github/workflows/docker-image.yml: -------------------------------------------------------------------------------- 1 | name: Docker image 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | 7 | env: 8 | REGISTRY: ghcr.io 9 | IMAGE_NAME: ${{ github.repository }} 10 | 11 | jobs: 12 | build-and-push-image: 13 | runs-on: ubuntu-latest 14 | # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. 15 | permissions: 16 | contents: read 17 | packages: write 18 | attestations: write 19 | id-token: write 20 | # 21 | steps: 22 | - name: Checkout repository 23 | uses: actions/checkout@v4 24 | # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. 25 | - name: Log in to the Container registry 26 | uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 27 | with: 28 | registry: ${{ env.REGISTRY }} 29 | username: ${{ github.actor }} 30 | password: ${{ secrets.GITHUB_TOKEN }} 31 | # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. 32 | - name: Extract metadata (tags, labels) for Docker 33 | id: meta 34 | uses: docker/metadata-action@v5 35 | with: 36 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 37 | tags: | 38 | type=sha 39 | type=sha,format=long 40 | # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. 41 | # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. 42 | # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. 43 | - name: Build and push Docker image 44 | id: push 45 | uses: docker/build-push-action@v5 46 | with: 47 | context: . 48 | push: true 49 | tags: ${{ steps.meta.outputs.tags }} 50 | labels: ${{ steps.meta.outputs.labels }} 51 | 52 | # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)." 53 | - name: Generate artifact attestation 54 | uses: actions/attest-build-provenance@v1 55 | with: 56 | subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} 57 | subject-digest: ${{ steps.push.outputs.digest }} 58 | push-to-registry: true 59 | -------------------------------------------------------------------------------- /pkg/server/subscriber_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestParseMaxMessageSizeBytes(t *testing.T) { 12 | stringTests := []struct { 13 | name string 14 | maxMessageSizeBytes string 15 | expected uint32 16 | }{ 17 | { 18 | name: "zero", 19 | maxMessageSizeBytes: "0", 20 | expected: 0, 21 | }, 22 | { 23 | name: "empty", 24 | maxMessageSizeBytes: "", 25 | expected: 0, 26 | }, 27 | { 28 | name: "invalid", 29 | maxMessageSizeBytes: "nope", 30 | expected: 0, 31 | }, 32 | { 33 | name: "valid", 34 | maxMessageSizeBytes: "1000000", 35 | expected: 1000000, 36 | }, 37 | { 38 | name: "uint32max", 39 | maxMessageSizeBytes: "4294967295", 40 | expected: 4294967295, 41 | }, 42 | { 43 | name: "uint32max_plus1", 44 | maxMessageSizeBytes: "4294967296", 45 | expected: 0, 46 | }, 47 | } 48 | 49 | for _, tt := range stringTests { 50 | t.Run(fmt.Sprintf("string %s", tt.name), func(t *testing.T) { 51 | got := ParseMaxMessageSizeBytes(tt.maxMessageSizeBytes) 52 | if got != tt.expected { 53 | t.Errorf("expected max size to be %d, got %d", tt.expected, got) 54 | } 55 | }) 56 | } 57 | 58 | intTests := []struct { 59 | name string 60 | maxMessageSizeBytes int 61 | expected uint32 62 | }{ 63 | { 64 | name: "zero", 65 | maxMessageSizeBytes: 0, 66 | expected: 0, 67 | }, 68 | { 69 | name: "uint32max", 70 | maxMessageSizeBytes: 4294967295, 71 | expected: 4294967295, 72 | }, 73 | { 74 | name: "uint32max_plus1", 75 | maxMessageSizeBytes: 4294967296, 76 | expected: 0, 77 | }, 78 | } 79 | 80 | for _, tt := range intTests { 81 | t.Run(fmt.Sprintf("int %s", tt.name), func(t *testing.T) { 82 | got := ParseMaxMessageSizeBytes(tt.maxMessageSizeBytes) 83 | if got != tt.expected { 84 | t.Errorf("expected max size to be %d, got %d", tt.expected, got) 85 | } 86 | }) 87 | } 88 | } 89 | 90 | func TestParseSubscriberOptions(t *testing.T) { 91 | testCases := []struct { 92 | name string 93 | data []byte 94 | expected SubscriberOptionsUpdatePayload 95 | }{ 96 | { 97 | name: "empty", 98 | data: []byte(`{}`), 99 | expected: SubscriberOptionsUpdatePayload{ 100 | WantedCollections: nil, 101 | WantedDIDs: nil, 102 | MaxMessageSizeBytes: 0, 103 | }, 104 | }, 105 | { 106 | name: "collection", 107 | data: []byte(`{"wantedCollections":["foo"]}`), 108 | expected: SubscriberOptionsUpdatePayload{ 109 | WantedCollections: []string{"foo"}, 110 | WantedDIDs: nil, 111 | MaxMessageSizeBytes: 0, 112 | }, 113 | }, 114 | { 115 | name: "small", 116 | data: []byte(`{"maxMessageSizeBytes":1000}`), 117 | expected: SubscriberOptionsUpdatePayload{ 118 | WantedCollections: nil, 119 | WantedDIDs: nil, 120 | MaxMessageSizeBytes: 1000, 121 | }, 122 | }, 123 | } 124 | 125 | for _, testCase := range testCases { 126 | t.Run(testCase.name, func(t *testing.T) { 127 | var subOptsUpdate SubscriberOptionsUpdatePayload 128 | if err := json.Unmarshal(testCase.data, &subOptsUpdate); err != nil { 129 | t.Errorf("failed to unmarshal subscriber options update: %v", err) 130 | } 131 | assert.Equal(t, subOptsUpdate, testCase.expected) 132 | }) 133 | } 134 | 135 | } 136 | -------------------------------------------------------------------------------- /pkg/client/schedulers/parallel/parallel.go: -------------------------------------------------------------------------------- 1 | // Package parallel provides a parallel scheduler that will run work on a fixed number of workers. 2 | // Events for the same repository will be processed sequentially, but events for different repositories can be processed concurrently. 3 | package parallel 4 | 5 | import ( 6 | "context" 7 | "log/slog" 8 | "sync" 9 | 10 | "github.com/bluesky-social/jetstream/pkg/client/schedulers" 11 | "github.com/bluesky-social/jetstream/pkg/models" 12 | "github.com/prometheus/client_golang/prometheus" 13 | ) 14 | 15 | // Scheduler is a parallel scheduler that will run work on a fixed number of workers 16 | type Scheduler struct { 17 | numWorkers int 18 | logger *slog.Logger 19 | handleEvent func(context.Context, *models.Event) error 20 | ident string 21 | 22 | feeder chan *consumerTask 23 | wg sync.WaitGroup 24 | 25 | lk sync.Mutex 26 | active map[string][]*consumerTask 27 | 28 | // metrics 29 | itemsAdded prometheus.Counter 30 | itemsProcessed prometheus.Counter 31 | itemsActive prometheus.Counter 32 | workersActive prometheus.Gauge 33 | } 34 | 35 | // NewScheduler creates a new parallel scheduler with the given number of workers 36 | func NewScheduler(numWorkers int, ident string, logger *slog.Logger, handleEvent func(context.Context, *models.Event) error) *Scheduler { 37 | logger = logger.With("component", "parallel-scheduler", "ident", ident) 38 | p := &Scheduler{ 39 | numWorkers: numWorkers, 40 | 41 | logger: logger, 42 | 43 | handleEvent: handleEvent, 44 | 45 | feeder: make(chan *consumerTask), 46 | active: make(map[string][]*consumerTask), 47 | 48 | ident: ident, 49 | 50 | itemsAdded: schedulers.WorkItemsAdded.WithLabelValues(ident, "parallel"), 51 | itemsActive: schedulers.WorkItemsActive.WithLabelValues(ident, "parallel"), 52 | itemsProcessed: schedulers.WorkItemsProcessed.WithLabelValues(ident, "parallel"), 53 | workersActive: schedulers.WorkersActive.WithLabelValues(ident, "parallel"), 54 | } 55 | 56 | p.wg.Add(numWorkers) 57 | for i := 0; i < numWorkers; i++ { 58 | go p.worker() 59 | } 60 | 61 | p.workersActive.Set(float64(numWorkers)) 62 | 63 | return p 64 | } 65 | 66 | // Shutdown shuts down the scheduler, waiting for all workers to finish their current work 67 | // The existing work queue will be processed, but no new work will be accepted 68 | func (p *Scheduler) Shutdown() { 69 | p.logger.Debug("shutting down parallel scheduler", "ident", p.ident) 70 | 71 | for i := 0; i < p.numWorkers; i++ { 72 | p.feeder <- &consumerTask{ 73 | stop: true, 74 | } 75 | } 76 | 77 | close(p.feeder) 78 | 79 | p.wg.Wait() 80 | 81 | p.logger.Debug("parallel scheduler shutdown complete") 82 | } 83 | 84 | type consumerTask struct { 85 | stop bool 86 | ctx context.Context 87 | repo string 88 | val *models.Event 89 | } 90 | 91 | // AddWork adds work to the scheduler 92 | func (p *Scheduler) AddWork(ctx context.Context, repo string, val *models.Event) error { 93 | p.itemsAdded.Inc() 94 | t := &consumerTask{ 95 | ctx: ctx, 96 | repo: repo, 97 | val: val, 98 | } 99 | 100 | // Append to the active list if there is already work for this repository 101 | p.lk.Lock() 102 | a, ok := p.active[repo] 103 | if ok { 104 | p.active[repo] = append(a, t) 105 | p.lk.Unlock() 106 | return nil 107 | } 108 | 109 | // Otherwise, initialize the active list for this repository to catch future work 110 | // and send this task to a worker 111 | p.active[repo] = []*consumerTask{} 112 | p.lk.Unlock() 113 | 114 | select { 115 | case p.feeder <- t: 116 | return nil 117 | case <-ctx.Done(): 118 | return ctx.Err() 119 | } 120 | } 121 | 122 | func (p *Scheduler) worker() { 123 | defer p.wg.Done() 124 | for work := range p.feeder { 125 | for work != nil { 126 | if work.stop { 127 | return 128 | } 129 | 130 | p.itemsActive.Inc() 131 | if err := p.handleEvent(work.ctx, work.val); err != nil { 132 | p.logger.Error("event handler failed", "error", err) 133 | } 134 | p.itemsProcessed.Inc() 135 | 136 | p.lk.Lock() 137 | rem, ok := p.active[work.repo] 138 | if !ok { 139 | p.logger.Error("worker should always have an 'active' entry if a worker is processing a job") 140 | } 141 | 142 | if len(rem) == 0 { 143 | delete(p.active, work.repo) 144 | work = nil 145 | } else { 146 | work = rem[0] 147 | p.active[work.repo] = rem[1:] 148 | } 149 | p.lk.Unlock() 150 | } 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /pkg/client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log/slog" 8 | "net/http" 9 | "net/url" 10 | 11 | "github.com/bluesky-social/jetstream/pkg/models" 12 | "github.com/goccy/go-json" 13 | "github.com/gorilla/websocket" 14 | "github.com/klauspost/compress/zstd" 15 | "go.uber.org/atomic" 16 | ) 17 | 18 | type ClientConfig struct { 19 | Compress bool 20 | WebsocketURL string 21 | WantedDids []string 22 | WantedCollections []string 23 | MaxSize uint32 24 | ExtraHeaders map[string]string 25 | } 26 | 27 | type Scheduler interface { 28 | AddWork(ctx context.Context, repo string, evt *models.Event) error 29 | Shutdown() 30 | } 31 | 32 | type Client struct { 33 | Scheduler Scheduler 34 | con *websocket.Conn 35 | config *ClientConfig 36 | logger *slog.Logger 37 | decoder *zstd.Decoder 38 | BytesRead atomic.Int64 39 | EventsRead atomic.Int64 40 | shutdown chan chan struct{} 41 | } 42 | 43 | func DefaultClientConfig() *ClientConfig { 44 | return &ClientConfig{ 45 | Compress: true, 46 | WebsocketURL: "ws://localhost:6008/subscribe", 47 | WantedDids: []string{}, 48 | WantedCollections: []string{}, 49 | MaxSize: 0, 50 | ExtraHeaders: map[string]string{ 51 | "User-Agent": "jetstream-client/v0.0.1", 52 | }, 53 | } 54 | } 55 | 56 | func NewClient(config *ClientConfig, logger *slog.Logger, scheduler Scheduler) (*Client, error) { 57 | if config == nil { 58 | config = DefaultClientConfig() 59 | } 60 | 61 | logger = logger.With("component", "jetstream-client") 62 | c := Client{ 63 | config: config, 64 | shutdown: make(chan chan struct{}), 65 | logger: logger, 66 | Scheduler: scheduler, 67 | } 68 | 69 | if config.Compress { 70 | c.config.ExtraHeaders["Socket-Encoding"] = "zstd" 71 | dec, err := zstd.NewReader(nil, zstd.WithDecoderDicts(models.ZSTDDictionary)) 72 | if err != nil { 73 | return nil, fmt.Errorf("failed to create zstd decoder: %w", err) 74 | } 75 | c.decoder = dec 76 | } 77 | 78 | return &c, nil 79 | } 80 | 81 | func (c *Client) ConnectAndRead(ctx context.Context, cursor *int64) error { 82 | header := http.Header{} 83 | for k, v := range c.config.ExtraHeaders { 84 | header.Add(k, v) 85 | } 86 | 87 | fullURL := c.config.WebsocketURL 88 | params := []string{} 89 | if cursor != nil { 90 | params = append(params, fmt.Sprintf("cursor=%d", *cursor)) 91 | } 92 | 93 | for _, did := range c.config.WantedDids { 94 | params = append(params, fmt.Sprintf("wantedDids=%s", did)) 95 | } 96 | 97 | for _, collection := range c.config.WantedCollections { 98 | params = append(params, fmt.Sprintf("wantedCollections=%s", collection)) 99 | } 100 | 101 | if c.config.MaxSize > 0 { 102 | params = append(params, fmt.Sprintf("maxSize=%d", c.config.MaxSize)) 103 | } 104 | 105 | if len(params) > 0 { 106 | fullURL += "?" + params[0] 107 | for _, p := range params[1:] { 108 | fullURL += "&" + p 109 | } 110 | } 111 | 112 | u, err := url.Parse(fullURL) 113 | if err != nil { 114 | return fmt.Errorf("failed to parse connection url %q: %w", c.config.WebsocketURL, err) 115 | } 116 | 117 | c.logger.Info("connecting to websocket", "url", u.String()) 118 | con, _, err := websocket.DefaultDialer.DialContext(ctx, u.String(), header) 119 | if err != nil { 120 | return err 121 | } 122 | 123 | c.con = con 124 | 125 | var readErr error = nil 126 | if err := c.readLoop(ctx); err != nil { 127 | readErr = fmt.Errorf("read loop failed: %w", err) 128 | } 129 | 130 | var closeErr error = nil 131 | if err := c.con.Close(); err != nil { 132 | closeErr = fmt.Errorf("failed to close connection: %w", err) 133 | } 134 | 135 | return errors.Join(readErr, closeErr) 136 | } 137 | 138 | func (c *Client) readLoop(ctx context.Context) error { 139 | c.logger.Info("starting websocket read loop") 140 | 141 | bytesRead := bytesRead.WithLabelValues(c.config.WebsocketURL) 142 | eventsRead := eventsRead.WithLabelValues(c.config.WebsocketURL) 143 | 144 | for { 145 | select { 146 | case <-ctx.Done(): 147 | c.logger.Info("shutting down read loop on context completion") 148 | return nil 149 | case s := <-c.shutdown: 150 | c.logger.Info("shutting down read loop on shutdown signal") 151 | s <- struct{}{} 152 | return nil 153 | default: 154 | _, msg, err := c.con.ReadMessage() 155 | if err != nil { 156 | c.logger.Error("failed to read message from websocket", "error", err) 157 | return fmt.Errorf("failed to read message from websocket: %w", err) 158 | } 159 | 160 | bytesRead.Add(float64(len(msg))) 161 | eventsRead.Inc() 162 | c.BytesRead.Add(int64(len(msg))) 163 | c.EventsRead.Inc() 164 | 165 | // Decompress the message if necessary 166 | if c.decoder != nil && c.config.Compress { 167 | m, err := c.decoder.DecodeAll(msg, nil) 168 | if err != nil { 169 | c.logger.Error("failed to decompress message", "error", err) 170 | return fmt.Errorf("failed to decompress message: %w", err) 171 | } 172 | msg = m 173 | } 174 | 175 | // Unpack the message and pass it to the handler 176 | var event models.Event 177 | if err := json.Unmarshal(msg, &event); err != nil { 178 | c.logger.Error("failed to unmarshal event", "error", err) 179 | return fmt.Errorf("failed to unmarshal event: %w", err) 180 | } 181 | 182 | if err := c.Scheduler.AddWork(ctx, event.Did, &event); err != nil { 183 | c.logger.Error("failed to add work to scheduler", "error", err) 184 | return fmt.Errorf("failed to add work to scheduler: %w", err) 185 | } 186 | } 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/bluesky-social/jetstream 2 | 3 | go 1.24 4 | 5 | toolchain go1.24.1 6 | 7 | require ( 8 | github.com/bluesky-social/indigo v0.0.0-20251009212240-20524de167fe 9 | github.com/cockroachdb/pebble v1.1.2 10 | github.com/goccy/go-json v0.10.2 11 | github.com/gorilla/websocket v1.5.1 12 | github.com/klauspost/compress v1.17.9 13 | github.com/labstack/echo/v4 v4.11.3 14 | github.com/labstack/gommon v0.4.1 15 | github.com/prometheus/client_golang v1.19.1 16 | github.com/stretchr/testify v1.9.0 17 | github.com/urfave/cli/v2 v2.26.0 18 | go.opentelemetry.io/otel v1.21.0 19 | go.uber.org/atomic v1.11.0 20 | golang.org/x/sync v0.7.0 21 | golang.org/x/time v0.5.0 22 | ) 23 | 24 | require ( 25 | github.com/DataDog/zstd v1.5.5 // indirect 26 | github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b // indirect 27 | github.com/beorn7/perks v1.0.1 // indirect 28 | github.com/carlmjohnson/versioninfo v0.22.5 // indirect 29 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 30 | github.com/cockroachdb/errors v1.11.3 // indirect 31 | github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect 32 | github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect 33 | github.com/cockroachdb/redact v1.1.5 // indirect 34 | github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect 35 | github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect 36 | github.com/davecgh/go-spew v1.1.1 // indirect 37 | github.com/felixge/httpsnoop v1.0.4 // indirect 38 | github.com/getsentry/sentry-go v0.28.0 // indirect 39 | github.com/go-logr/logr v1.4.1 // indirect 40 | github.com/go-logr/stdr v1.2.2 // indirect 41 | github.com/gocql/gocql v1.7.0 // indirect 42 | github.com/gogo/protobuf v1.3.2 // indirect 43 | github.com/golang-jwt/jwt v3.2.2+incompatible // indirect 44 | github.com/golang/snappy v0.0.4 // indirect 45 | github.com/google/uuid v1.6.0 // indirect 46 | github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect 47 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 48 | github.com/hashicorp/go-retryablehttp v0.7.5 // indirect 49 | github.com/hashicorp/golang-lru v1.0.2 // indirect 50 | github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect 51 | github.com/ipfs/bbloom v0.0.4 // indirect 52 | github.com/ipfs/go-block-format v0.2.0 // indirect 53 | github.com/ipfs/go-blockservice v0.5.2 // indirect 54 | github.com/ipfs/go-cid v0.4.1 // indirect 55 | github.com/ipfs/go-datastore v0.6.0 // indirect 56 | github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect 57 | github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect 58 | github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect 59 | github.com/ipfs/go-ipfs-util v0.0.3 // indirect 60 | github.com/ipfs/go-ipld-cbor v0.1.0 // indirect 61 | github.com/ipfs/go-ipld-format v0.6.0 // indirect 62 | github.com/ipfs/go-ipld-legacy v0.2.1 // indirect 63 | github.com/ipfs/go-libipfs v0.7.0 // indirect 64 | github.com/ipfs/go-log v1.0.5 // indirect 65 | github.com/ipfs/go-log/v2 v2.5.1 // indirect 66 | github.com/ipfs/go-merkledag v0.11.0 // indirect 67 | github.com/ipfs/go-metrics-interface v0.0.1 // indirect 68 | github.com/ipfs/go-verifcid v0.0.3 // indirect 69 | github.com/ipld/go-car v0.6.2 // indirect 70 | github.com/ipld/go-codec-dagpb v1.6.0 // indirect 71 | github.com/ipld/go-ipld-prime v0.21.0 // indirect 72 | github.com/jackc/pgpassfile v1.0.0 // indirect 73 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect 74 | github.com/jackc/pgx/v5 v5.5.0 // indirect 75 | github.com/jackc/puddle/v2 v2.2.1 // indirect 76 | github.com/jbenet/goprocess v0.1.4 // indirect 77 | github.com/jinzhu/inflection v1.0.0 // indirect 78 | github.com/jinzhu/now v1.1.5 // indirect 79 | github.com/klauspost/cpuid/v2 v2.2.7 // indirect 80 | github.com/kr/pretty v0.3.1 // indirect 81 | github.com/kr/text v0.2.0 // indirect 82 | github.com/mattn/go-colorable v0.1.13 // indirect 83 | github.com/mattn/go-isatty v0.0.20 // indirect 84 | github.com/mattn/go-sqlite3 v1.14.22 // indirect 85 | github.com/minio/sha256-simd v1.0.1 // indirect 86 | github.com/mr-tron/base58 v1.2.0 // indirect 87 | github.com/multiformats/go-base32 v0.1.0 // indirect 88 | github.com/multiformats/go-base36 v0.2.0 // indirect 89 | github.com/multiformats/go-multibase v0.2.0 // indirect 90 | github.com/multiformats/go-multihash v0.2.3 // indirect 91 | github.com/multiformats/go-varint v0.0.7 // indirect 92 | github.com/opentracing/opentracing-go v1.2.0 // indirect 93 | github.com/pkg/errors v0.9.1 // indirect 94 | github.com/pmezard/go-difflib v1.0.0 // indirect 95 | github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect 96 | github.com/prometheus/client_model v0.6.1 // indirect 97 | github.com/prometheus/common v0.54.0 // indirect 98 | github.com/prometheus/procfs v0.15.1 // indirect 99 | github.com/rogpeppe/go-internal v1.12.0 // indirect 100 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 101 | github.com/spaolacci/murmur3 v1.1.0 // indirect 102 | github.com/valyala/bytebufferpool v1.0.0 // indirect 103 | github.com/valyala/fasttemplate v1.2.2 // indirect 104 | github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e // indirect 105 | github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect 106 | gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect 107 | gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect 108 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect 109 | go.opentelemetry.io/otel/metric v1.21.0 // indirect 110 | go.opentelemetry.io/otel/trace v1.21.0 // indirect 111 | go.uber.org/multierr v1.11.0 // indirect 112 | go.uber.org/zap v1.26.0 // indirect 113 | golang.org/x/crypto v0.22.0 // indirect 114 | golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect 115 | golang.org/x/net v0.24.0 // indirect 116 | golang.org/x/sys v0.22.0 // indirect 117 | golang.org/x/text v0.16.0 // indirect 118 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect 119 | google.golang.org/protobuf v1.34.2 // indirect 120 | gopkg.in/inf.v0 v0.9.1 // indirect 121 | gopkg.in/yaml.v3 v3.0.1 // indirect 122 | gorm.io/driver/postgres v1.5.7 // indirect 123 | gorm.io/gorm v1.25.9 // indirect 124 | lukechampine.com/blake3 v1.2.1 // indirect 125 | ) 126 | -------------------------------------------------------------------------------- /pkg/consumer/persist.go: -------------------------------------------------------------------------------- 1 | package consumer 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "github.com/bluesky-social/jetstream/pkg/models" 12 | "github.com/cockroachdb/pebble" 13 | "github.com/goccy/go-json" 14 | "github.com/labstack/gommon/log" 15 | "golang.org/x/time/rate" 16 | ) 17 | 18 | // Progress is the cursor for the consumer 19 | type Progress struct { 20 | LastSeq int64 `json:"last_seq"` 21 | LastSeqProcessedAt time.Time `json:"last_seq_processed_at"` 22 | lk sync.RWMutex 23 | } 24 | 25 | func (p *Progress) Update(seq int64, processedAt time.Time) { 26 | p.lk.Lock() 27 | defer p.lk.Unlock() 28 | p.LastSeq = seq 29 | p.LastSeqProcessedAt = processedAt 30 | } 31 | 32 | func (p *Progress) Get() (int64, time.Time) { 33 | p.lk.RLock() 34 | defer p.lk.RUnlock() 35 | return p.LastSeq, p.LastSeqProcessedAt 36 | } 37 | 38 | var cursorKey = []byte("cursor") 39 | 40 | // WriteCursor writes the cursor to file 41 | func (c *Consumer) WriteCursor(ctx context.Context) error { 42 | ctx, span := tracer.Start(ctx, "WriteCursor") 43 | defer span.End() 44 | 45 | // Marshal the cursor JSON 46 | seq, processedAt := c.Progress.Get() 47 | p := Progress{ 48 | LastSeq: seq, 49 | LastSeqProcessedAt: processedAt, 50 | } 51 | data, err := json.Marshal(&p) 52 | if err != nil { 53 | return fmt.Errorf("failed to marshal cursor JSON: %+v", err) 54 | } 55 | 56 | // Write the cursor JSON to pebble 57 | err = c.UncompressedDB.Set(cursorKey, data, pebble.Sync) 58 | if err != nil { 59 | return fmt.Errorf("failed to write cursor to pebble: %w", err) 60 | } 61 | 62 | return nil 63 | } 64 | 65 | // ReadCursor reads the cursor from file 66 | func (c *Consumer) ReadCursor(ctx context.Context) error { 67 | ctx, span := tracer.Start(ctx, "ReadCursor") 68 | defer span.End() 69 | 70 | // Read the cursor from pebble 71 | data, closer, err := c.UncompressedDB.Get(cursorKey) 72 | if err != nil { 73 | if err == pebble.ErrNotFound { 74 | return nil 75 | } 76 | return fmt.Errorf("failed to read cursor from pebble: %w", err) 77 | } 78 | defer closer.Close() 79 | 80 | // Unmarshal the cursor JSON 81 | err = json.Unmarshal(data, c.Progress) 82 | if err != nil { 83 | return fmt.Errorf("failed to unmarshal cursor JSON: %w", err) 84 | } 85 | 86 | return nil 87 | } 88 | 89 | // PersistEvent persists an event to PebbleDB 90 | func (c *Consumer) PersistEvent(ctx context.Context, evt *models.Event, asJSON, compBytes []byte) error { 91 | ctx, span := tracer.Start(ctx, "PersistEvent") 92 | defer span.End() 93 | 94 | // Key structure for events in PebbleDB 95 | // {{event_time_us}}_{{repo}}_{{collection}} 96 | var key []byte 97 | if evt.Kind == models.EventKindCommit && evt.Commit != nil { 98 | key = []byte(fmt.Sprintf("%d_%s_%s", evt.TimeUS, evt.Did, evt.Commit.Collection)) 99 | } else { 100 | key = []byte(fmt.Sprintf("%d_%s", evt.TimeUS, evt.Did)) 101 | } 102 | 103 | // Write the uncompressed event to the uncompressed DB 104 | err := c.UncompressedDB.Set(key, asJSON, pebble.NoSync) 105 | if err != nil { 106 | log.Error("failed to write event to pebble", "error", err) 107 | return fmt.Errorf("failed to write event to pebble: %w", err) 108 | } 109 | 110 | // Write the compressed event to the compressed DB 111 | err = c.CompressedDB.Set(key, compBytes, pebble.NoSync) 112 | if err != nil { 113 | log.Error("failed to write compressed event to pebble", "error", err) 114 | return fmt.Errorf("failed to write compressed event to pebble: %w", err) 115 | } 116 | 117 | return nil 118 | } 119 | 120 | // TrimEvents deletes old events from PebbleDB 121 | func (c *Consumer) TrimEvents(ctx context.Context) error { 122 | ctx, span := tracer.Start(ctx, "TrimEvents") 123 | defer span.End() 124 | 125 | // Keys are stored as strings of the event time in microseconds 126 | // We can range delete events older than the event TTL 127 | trimUntil := time.Now().Add(-c.EventTTL).UnixMicro() 128 | trimKey := []byte(fmt.Sprintf("%d", trimUntil)) 129 | 130 | // Delete all numeric keys older than the trim key 131 | err := c.UncompressedDB.DeleteRange([]byte("0"), trimKey, pebble.Sync) 132 | if err != nil { 133 | log.Error("failed to delete old events", "error", err) 134 | return fmt.Errorf("failed to delete old events: %w", err) 135 | } 136 | 137 | // Delete all numeric keys older than the trim key in the compressed DB 138 | err = c.CompressedDB.DeleteRange([]byte("0"), trimKey, pebble.Sync) 139 | if err != nil { 140 | log.Error("failed to delete old compressed events", "error", err) 141 | return fmt.Errorf("failed to delete old compressed events: %w", err) 142 | } 143 | 144 | return nil 145 | } 146 | 147 | // Saturday, May 19, 2277 12:26:40 PM 148 | var finalKey = []byte("9700000000000000") 149 | 150 | // ReplayEvents replays events from PebbleDB 151 | func (c *Consumer) ReplayEvents(ctx context.Context, compressed bool, cursor int64, playbackRateLimit float64, emit func(context.Context, int64, string, string, func() []byte) error) (int64, error) { 152 | ctx, span := tracer.Start(ctx, "ReplayEvents") 153 | defer span.End() 154 | 155 | // Limit the playback rate to avoid thrashing the host when replaying events 156 | // with very specific filters 157 | limiter := rate.NewLimiter(rate.Limit(playbackRateLimit), int(playbackRateLimit)) 158 | 159 | // Iterate over all events starting from the cursor 160 | var iter *pebble.Iterator 161 | var err error 162 | 163 | iterOptions := &pebble.IterOptions{ 164 | LowerBound: []byte(fmt.Sprintf("%d", cursor)), 165 | UpperBound: finalKey, 166 | } 167 | 168 | if compressed { 169 | iter, err = c.CompressedDB.NewIterWithContext(ctx, iterOptions) 170 | } else { 171 | iter, err = c.UncompressedDB.NewIterWithContext(ctx, iterOptions) 172 | } 173 | if err != nil { 174 | log.Error("failed to create iterator", "error", err) 175 | return 0, fmt.Errorf("failed to create iterator: %w", err) 176 | } 177 | defer iter.Close() 178 | 179 | // This iterator will return events in ascending order of time, starting from the cursor 180 | // and stopping when it reaches the end of the database 181 | // if you never reach the end of the database, you'll just keep consuming slower than the events are produced 182 | lastSeq := int64(0) 183 | for iter.First(); iter.Valid(); iter.Next() { 184 | // Wait for the rate limiter 185 | err := limiter.Wait(ctx) 186 | if err != nil { 187 | log.Error("failed to wait for rate limiter", "error", err) 188 | return 0, fmt.Errorf("failed to wait for rate limiter: %w", err) 189 | } 190 | 191 | // Unpack the key ({{event_time_us}}_{{repo}}_{{collection}}) 192 | key := string(iter.Key()) 193 | parts := strings.Split(key, "_") 194 | if len(parts) < 2 { 195 | log.Error("invalid key format", "key", key) 196 | return 0, fmt.Errorf("invalid key format: %s", key) 197 | } 198 | 199 | timeUS, err := strconv.ParseInt(parts[0], 10, 64) 200 | if err != nil { 201 | log.Error("failed to parse timeUS from event", "error", err) 202 | return 0, fmt.Errorf("failed to parse timeUS: %w", err) 203 | } 204 | 205 | collection := "" 206 | if len(parts) > 2 { 207 | collection = parts[2] 208 | } 209 | 210 | // Emit the event with the valuer function so the subscriber can decide if it wants to filter it out 211 | // without having to read the entire event from the database 212 | err = emit(ctx, timeUS, parts[1], collection, iter.Value) 213 | if err != nil { 214 | log.Error("failed to emit event", "error", err) 215 | return 0, fmt.Errorf("failed to emit event: %w", err) 216 | } 217 | 218 | lastSeq = timeUS 219 | } 220 | 221 | return lastSeq, nil 222 | } 223 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Jetstream 2 | 3 | Jetstream is a streaming service that consumes an ATProto `com.atproto.sync.subscribeRepos` stream and converts it into lightweight, friendly JSON. 4 | 5 | Jetstream converts the CBOR-encoded MST blocks produced by the ATProto firehose and translates them into JSON objects that are easier to interface with using standard tooling available in programming languages. 6 | 7 | ### Public Instances 8 | 9 | As of writing, there are 4 official public Jetstream instances operated by Bluesky. 10 | 11 | | Hostname | Region | 12 | | --------------------------------- | ------- | 13 | | `jetstream1.us-east.bsky.network` | US-East | 14 | | `jetstream2.us-east.bsky.network` | US-East | 15 | | `jetstream1.us-west.bsky.network` | US-West | 16 | | `jetstream2.us-west.bsky.network` | US-West | 17 | 18 | Connect to these instances over WSS: `wss://jetstream2.us-west.bsky.network/subscribe` 19 | 20 | We will monitor and operate these instances and do our best to keep them available for public use by developers. 21 | 22 | Feel free to have multiple connections to Jetstream instances if needed. We encourage you to make use of Jetstream wherever you may consider using the `com.atproto.sync.subscribeRepos` firehose if you don't need the features of the full sync protocol. 23 | 24 | Because cursors for Jetstream are time-based (unix microseconds), you can use the same cursor for multiple instances to get roughly the same data. 25 | 26 | When switching between instances, it may be prudent to rewind your cursor a few seconds for gapless playback if you process events idempotently. 27 | 28 | ## Running Jetstream 29 | 30 | To run Jetstream, make sure you have docker and docker compose installed and run `make up` in the repo root. 31 | 32 | This will pull the latest built image from GHCR and start a Jetstream instance at `http://localhost:6008` 33 | 34 | - To build Jetstream from source via Docker and start it up, run `make rebuild` 35 | 36 | Once started, you can connect to the event stream at: `ws://localhost:6008/subscribe` 37 | 38 | Prometheus metrics are exposed at `http://localhost:6009/metrics` 39 | 40 | A [Grafana Dashboard](#dashboard-preview) for Jetstream is available at `./grafana-dashboard.json` and should be easy to copy/paste into Grafana's dashboard import prompt. 41 | 42 | - This dashboard has a few device-specific graphs for disk and network usage that require NodeExporter and may need to be tuned to your setup. 43 | 44 | ## Consuming Jetstream 45 | 46 | To consume Jetstream you can use any websocket client 47 | 48 | Connect to `ws://localhost:6008/subscribe` to start the stream 49 | 50 | The following Query Parameters are supported: 51 | 52 | - `wantedCollections` - An array of [Collection NSIDs](https://atproto.com/specs/nsid) to filter which records you receive on your stream (default empty = all collections) 53 | - `wantedCollections` supports NSID path prefixes i.e. `app.bsky.graph.*`, or `app.bsky.*`. The prefix before the `.*` must pass NSID validation and Jetstream **does not** support incomplete prefixes i.e. `app.bsky.graph.fo*`. 54 | - Regardless of desired collections, all subscribers receive Account and Identity events. 55 | - You can specify at most 100 wanted collections/prefixes. 56 | - `wantedDids` - An array of Repo DIDs to filter which records you receive on your stream (Default empty = all repos) 57 | - You can specify at most 10,000 wanted DIDs. 58 | - `maxMessageSizeBytes` - The maximum size of a payload that this client would like to receive. Zero means no limit, negative values are treated as zero. (Default "0" or empty = no maximum size) 59 | - `cursor` - A unix microseconds timestamp cursor to begin playback from 60 | - An absent cursor or a cursor from the future will result in live-tail operation 61 | - When reconnecting, use the `time_us` from your most recently processed event and maybe provide a negative buffer (i.e. subtract a few seconds) to ensure gapless playback 62 | - `compress` - Set to `true` to enable `zstd` [compression](#compression) 63 | - `requireHello` - Set to `true` to pause replay/live-tail until the server receives a [`SubscriberOptionsUpdatePayload`](#options-updates) over the socket in a [Subscriber Sourced Message](#subscriber-sourced-messages) 64 | 65 | ### Examples 66 | 67 | A simple example that hits the public instance looks like: 68 | 69 | ```bash 70 | $ websocat wss://jetstream2.us-east.bsky.network/subscribe\?wantedCollections=app.bsky.feed.post 71 | ``` 72 | 73 | A maximal example using all parameters looks like: 74 | 75 | ```bash 76 | $ websocat "ws://localhost:6008/subscribe?wantedCollections=app.bsky.feed.post&wantedCollections=app.bsky.feed.like&wantedCollections=app.bsky.graph.follow&wantedDids=did:plc:q6gjnaw2blty4crticxkmujt&cursor=1725519626134432" 77 | ``` 78 | 79 | ### Example events: 80 | 81 | Jetstream events have 3 `kinds`s (so far): 82 | 83 | - `commit`: a Commit to a repo which involves either a create, update, or delete of a record 84 | - `identity`: an Identity update for a DID which indicates that you may want to purge an identity cache and revalidate the DID doc and handle 85 | - `account`: an Account event that indicates a change in account status i.e. from `active` to `deactivated`, or to `takendown` if the PDS has taken down the repo. 86 | 87 | Jetstream Commits have 3 `operations`: 88 | 89 | - `create`: Create a new record with the contents provided 90 | - `update`: Update an existing record and replace it with the contents provided 91 | - `delete`: Delete an existing record with the DID, Collection, and RKey provided 92 | 93 | #### A like committed to a repo 94 | 95 | ```json 96 | { 97 | "did": "did:plc:eygmaihciaxprqvxpfvl6flk", 98 | "time_us": 1725911162329308, 99 | "kind": "commit", 100 | "commit": { 101 | "rev": "3l3qo2vutsw2b", 102 | "operation": "create", 103 | "collection": "app.bsky.feed.like", 104 | "rkey": "3l3qo2vuowo2b", 105 | "record": { 106 | "$type": "app.bsky.feed.like", 107 | "createdAt": "2024-09-09T19:46:02.102Z", 108 | "subject": { 109 | "cid": "bafyreidc6sydkkbchcyg62v77wbhzvb2mvytlmsychqgwf2xojjtirmzj4", 110 | "uri": "at://did:plc:wa7b35aakoll7hugkrjtf3xf/app.bsky.feed.post/3l3pte3p2e325" 111 | } 112 | }, 113 | "cid": "bafyreidwaivazkwu67xztlmuobx35hs2lnfh3kolmgfmucldvhd3sgzcqi" 114 | } 115 | } 116 | ``` 117 | 118 | #### A deleted follow record 119 | 120 | ```json 121 | { 122 | "did": "did:plc:rfov6bpyztcnedeyyzgfq42k", 123 | "time_us": 1725516666833633, 124 | "kind": "commit", 125 | "commit": { 126 | "rev": "3l3f6nzl3cv2s", 127 | "operation": "delete", 128 | "collection": "app.bsky.graph.follow", 129 | "rkey": "3l3dn7tku762u" 130 | } 131 | } 132 | ``` 133 | 134 | #### An identity update 135 | 136 | ```json 137 | { 138 | "did": "did:plc:ufbl4k27gp6kzas5glhz7fim", 139 | "time_us": 1725516665234703, 140 | "kind": "identity", 141 | "identity": { 142 | "did": "did:plc:ufbl4k27gp6kzas5glhz7fim", 143 | "handle": "yohenrique.bsky.social", 144 | "seq": 1409752997, 145 | "time": "2024-09-05T06:11:04.870Z" 146 | } 147 | } 148 | ``` 149 | 150 | #### An account becoming active 151 | 152 | ```json 153 | { 154 | "did": "did:plc:ufbl4k27gp6kzas5glhz7fim", 155 | "time_us": 1725516665333808, 156 | "kind": "account", 157 | "account": { 158 | "active": true, 159 | "did": "did:plc:ufbl4k27gp6kzas5glhz7fim", 160 | "seq": 1409753013, 161 | "time": "2024-09-05T06:11:04.870Z" 162 | } 163 | } 164 | ``` 165 | 166 | ### Compression 167 | 168 | Jetstream supports `zstd`-based compression of messages. Jetstream uses a custom dictionary for compression that can be found in `pkg/models/zstd_dictionary` and is required to decode compressed messages from the server. 169 | 170 | `zstd` compressed Jetstream messages are ~56% smaller on average than the raw JSON version of the Jetstream firehose. 171 | 172 | The provided client library uses compression by default, using an embedded copy of the Dictionary from the `models` package. 173 | 174 | To request a compressed stream, pass the `Socket-Encoding: zstd` header through when initiating the websocket _or_ pass `compress=true` in the query string. 175 | 176 | ### Subscriber Sourced messages 177 | 178 | Subscribers can send Text messages to Jetstream over the websocket using the `SubscriberSourcedMessage` framing below: 179 | 180 | ```go 181 | type SubscriberSourcedMessage struct { 182 | Type string `json:"type"` 183 | Payload json.RawMessage `json:"payload"` 184 | } 185 | ``` 186 | 187 | The supported message types are as follows: 188 | 189 | - `options_update` 190 | 191 | #### Options Updates 192 | 193 | A client can update their `wantedCollections` and `wantedDids` after connecting to the socket by sending a Subscriber Sourced Message. 194 | 195 | To send an Options Update, provide the string `options_update` in the `type` field and a `SubscriberOptionsUpdatePayload` in the `payload` field. 196 | 197 | The shape for a `SubscriberOptionsUpdatePayload` is as follows: 198 | 199 | ```go 200 | type SubscriberOptionsUpdateMsg struct { 201 | WantedCollections []string `json:"wantedCollections"` 202 | WantedDIDs []string `json:"wantedDids"` 203 | MaxMessageSizeBytes int `json:"maxMessageSizeBytes"` 204 | } 205 | ``` 206 | 207 | If either array is empty, the relevant filter will be disabled (i.e. sending empty `wantedDids` will mean a client gets messages for all DIDs again). 208 | 209 | Some limitations apply around the size of the message: right now the message can be at most 10MB in size and can contain up to 100 collection filters _and_ up to 10,000 DID filters. 210 | 211 | Additionally, a client can connect with `?requireHello=true` in the query params to pause replay/live-tail until the first Options Update message is sent by the client over the socket. 212 | 213 | Invalid Options Updates in `requireHello` mode or normal operating mode will result in the client being disconnected. 214 | 215 | An example Subscriber Sourced Message with an Options Update payload is as follows: 216 | 217 | ```json 218 | { 219 | "type": "options_update", 220 | "payload": { 221 | "wantedCollections": ["app.bsky.feed.post"], 222 | "wantedDids": ["did:plc:q6gjnaw2blty4crticxkmujt"], 223 | "maxMessageSizeBytes": 1000000 224 | } 225 | } 226 | ``` 227 | 228 | The above payload will filter such that a client receives only posts, and only from a the specified DID. 229 | 230 | ### Dashboard Preview 231 | 232 | ![A screenshot of the Jetstream Grafana Dashboard](./docs/dash.png) 233 | -------------------------------------------------------------------------------- /pkg/server/subscriber.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log/slog" 8 | "slices" 9 | "strconv" 10 | "strings" 11 | "sync" 12 | 13 | "github.com/bluesky-social/indigo/atproto/syntax" 14 | "github.com/gorilla/websocket" 15 | "github.com/prometheus/client_golang/prometheus" 16 | "golang.org/x/time/rate" 17 | ) 18 | 19 | type WantedCollections struct { 20 | Prefixes []string 21 | FullPaths map[string]struct{} 22 | } 23 | 24 | type Subscriber struct { 25 | ws *websocket.Conn 26 | conLk sync.Mutex 27 | realIP string 28 | lk sync.Mutex 29 | seq int64 30 | outbox chan *[]byte 31 | hello chan struct{} 32 | id int64 33 | tearingDown bool 34 | 35 | // Subscriber options 36 | 37 | // wantedCollections is nil if the subscriber wants all collections 38 | wantedCollections *WantedCollections 39 | wantedDids map[string]struct{} 40 | cursor *int64 41 | compress bool 42 | maxMessageSizeBytes uint32 43 | 44 | rl *rate.Limiter 45 | 46 | deliveredCounter prometheus.Counter 47 | bytesCounter prometheus.Counter 48 | } 49 | 50 | // emitToSubscriber sends an event to a subscriber if the subscriber wants the event 51 | // It takes a valuer function to get the event bytes so that the caller can avoid 52 | // unnecessary allocations and/or reading from the playback DB if the subscriber doesn't want the event 53 | func emitToSubscriber(ctx context.Context, log *slog.Logger, sub *Subscriber, timeUS int64, did, collection string, playback bool, getEventBytes func() []byte) error { 54 | if !sub.WantsCollection(collection) { 55 | return nil 56 | } 57 | 58 | if len(sub.wantedDids) > 0 { 59 | if _, ok := sub.wantedDids[did]; !ok { 60 | return nil 61 | } 62 | } 63 | 64 | // Skip events that are older than the subscriber's last seen event 65 | if timeUS <= sub.seq { 66 | return nil 67 | } 68 | 69 | evtBytes := getEventBytes() 70 | if sub.maxMessageSizeBytes > 0 && uint32(len(evtBytes)) > sub.maxMessageSizeBytes { 71 | return nil 72 | } 73 | 74 | if playback { 75 | // Copy the event bytes so the playback iterator can reuse the buffer 76 | evtBytes = slices.Clone(evtBytes) 77 | select { 78 | case <-ctx.Done(): 79 | log.Error("failed to send event to subscriber", "error", ctx.Err(), "subscriber", sub.id) 80 | // If we failed to send to a subscriber, close the connection 81 | sub.Terminate("error sending event") 82 | err := sub.ws.Close() 83 | if err != nil { 84 | log.Error("failed to close subscriber connection", "error", err) 85 | } 86 | return ctx.Err() 87 | case sub.outbox <- &evtBytes: 88 | sub.seq = timeUS 89 | sub.deliveredCounter.Inc() 90 | sub.bytesCounter.Add(float64(len(evtBytes))) 91 | } 92 | } else { 93 | select { 94 | case <-ctx.Done(): 95 | log.Error("failed to send event to subscriber", "error", ctx.Err(), "subscriber", sub.id) 96 | // If we failed to send to a subscriber, close the connection 97 | sub.Terminate("error sending event") 98 | err := sub.ws.Close() 99 | if err != nil { 100 | log.Error("failed to close subscriber connection", "error", err) 101 | } 102 | return ctx.Err() 103 | case sub.outbox <- &evtBytes: 104 | sub.seq = timeUS 105 | sub.deliveredCounter.Inc() 106 | sub.bytesCounter.Add(float64(len(evtBytes))) 107 | default: 108 | // Drop slow subscribers if they're live tailing and fall too far behind 109 | log.Error("failed to send event to subscriber, dropping", "error", "buffer full", "subscriber", sub.id) 110 | 111 | // Tearing down a subscriber can block, so do it in a goroutine 112 | go func() { 113 | sub.tearingDown = true 114 | // Don't send a close message cause they won't get it (the socket is backed up) 115 | err := sub.ws.Close() 116 | if err != nil { 117 | log.Error("failed to close subscriber connection", "error", err) 118 | } 119 | }() 120 | } 121 | } 122 | 123 | return nil 124 | } 125 | 126 | var SubMessageOptionsUpdate = "options_update" 127 | 128 | type SubscriberSourcedMessage struct { 129 | Type string `json:"type"` 130 | Payload json.RawMessage `json:"payload"` 131 | } 132 | 133 | type SubscriberOptionsUpdatePayload struct { 134 | WantedCollections []string `json:"wantedCollections"` 135 | WantedDIDs []string `json:"wantedDids"` 136 | MaxMessageSizeBytes int `json:"maxMessageSizeBytes"` 137 | } 138 | 139 | type SubscriberOptions struct { 140 | WantedCollections *WantedCollections 141 | WantedDIDs map[string]struct{} 142 | MaxMessageSizeBytes uint32 143 | Compress bool 144 | Cursor *int64 145 | } 146 | 147 | // ErrInvalidOptions is returned when the subscriber options are invalid 148 | var ErrInvalidOptions = fmt.Errorf("invalid subscriber options") 149 | 150 | func parseSubscriberOptions(ctx context.Context, wantedCollectionsProvided, wantedDidsProvided []string, compress bool, maxMessageSizeBytes uint32, cursor *int64) (*SubscriberOptions, error) { 151 | ctx, span := tracer.Start(ctx, "parseSubscriberOptions") 152 | defer span.End() 153 | 154 | var wantedCol *WantedCollections 155 | if len(wantedCollectionsProvided) > 0 { 156 | wantedCol = &WantedCollections{ 157 | Prefixes: []string{}, 158 | FullPaths: make(map[string]struct{}), 159 | } 160 | 161 | for _, providedCol := range wantedCollectionsProvided { 162 | if strings.HasSuffix(providedCol, ".*") { 163 | wantedCol.Prefixes = append(wantedCol.Prefixes, strings.TrimSuffix(providedCol, "*")) 164 | continue 165 | } 166 | 167 | col, err := syntax.ParseNSID(providedCol) 168 | if err != nil { 169 | 170 | return nil, fmt.Errorf("%w: invalid collection: %s", ErrInvalidOptions, providedCol) 171 | } 172 | wantedCol.FullPaths[col.String()] = struct{}{} 173 | } 174 | } 175 | 176 | didMap := make(map[string]struct{}) 177 | for _, d := range wantedDidsProvided { 178 | did, err := syntax.ParseDID(d) 179 | if err != nil { 180 | return nil, ErrInvalidOptions 181 | } 182 | didMap[did.String()] = struct{}{} 183 | } 184 | 185 | // Reject requests with too many wanted DIDs 186 | if len(didMap) > 10_000 { 187 | return nil, fmt.Errorf("%w: too many wanted DIDs", ErrInvalidOptions) 188 | } 189 | 190 | // Reject requests with too many wanted collections 191 | if wantedCol != nil && len(wantedCol.Prefixes)+len(wantedCol.FullPaths) > 100 { 192 | return nil, fmt.Errorf("%w: too many wanted collections", ErrInvalidOptions) 193 | } 194 | 195 | return &SubscriberOptions{ 196 | WantedCollections: wantedCol, 197 | WantedDIDs: didMap, 198 | Compress: compress, 199 | MaxMessageSizeBytes: maxMessageSizeBytes, 200 | Cursor: cursor, 201 | }, nil 202 | } 203 | 204 | func (s *Server) AddSubscriber(ws *websocket.Conn, realIP string, opts *SubscriberOptions) (*Subscriber, error) { 205 | s.lk.Lock() 206 | defer s.lk.Unlock() 207 | 208 | lim := s.perIPLimiters[realIP] 209 | if lim == nil { 210 | lim = rate.NewLimiter(rate.Limit(s.maxSubRate), int(s.maxSubRate)) 211 | s.perIPLimiters[realIP] = lim 212 | } 213 | 214 | sub := Subscriber{ 215 | ws: ws, 216 | realIP: realIP, 217 | outbox: make(chan *[]byte, 50_000), 218 | hello: make(chan struct{}), 219 | id: s.nextSub, 220 | wantedCollections: opts.WantedCollections, 221 | wantedDids: opts.WantedDIDs, 222 | cursor: opts.Cursor, 223 | compress: opts.Compress, 224 | maxMessageSizeBytes: opts.MaxMessageSizeBytes, 225 | deliveredCounter: eventsDelivered.WithLabelValues(realIP), 226 | bytesCounter: bytesDelivered.WithLabelValues(realIP), 227 | rl: lim, 228 | } 229 | 230 | s.Subscribers[s.nextSub] = &sub 231 | s.nextSub++ 232 | 233 | subscribersConnected.WithLabelValues(realIP).Inc() 234 | 235 | slog.Info("adding subscriber", 236 | "real_ip", realIP, 237 | "id", sub.id, 238 | "wantedCollections", opts.WantedCollections, 239 | "wantedDids", opts.WantedDIDs, 240 | "cursor", opts.Cursor, 241 | "compress", opts.Compress, 242 | "maxMessageSizeBytes", opts.MaxMessageSizeBytes, 243 | ) 244 | 245 | return &sub, nil 246 | } 247 | 248 | func (s *Server) RemoveSubscriber(num int64) { 249 | s.lk.Lock() 250 | defer s.lk.Unlock() 251 | 252 | slog.Info("removing subscriber", "id", num, "real_ip", s.Subscribers[num].realIP) 253 | 254 | subscribersConnected.WithLabelValues(s.Subscribers[num].realIP).Dec() 255 | 256 | delete(s.Subscribers, num) 257 | } 258 | 259 | // WantsCollection returns true if the subscriber wants the given collection 260 | func (sub *Subscriber) WantsCollection(collection string) bool { 261 | if sub.wantedCollections == nil || collection == "" { 262 | return true 263 | } 264 | 265 | // Start with the full paths for fast lookup 266 | if len(sub.wantedCollections.FullPaths) > 0 { 267 | if _, match := sub.wantedCollections.FullPaths[collection]; match { 268 | return true 269 | } 270 | } 271 | 272 | // Check the prefixes (shortest first) 273 | for _, prefix := range sub.wantedCollections.Prefixes { 274 | if strings.HasPrefix(collection, prefix) { 275 | return true 276 | } 277 | } 278 | 279 | return false 280 | } 281 | 282 | func (s *Subscriber) UpdateOptions(opts *SubscriberOptions) { 283 | s.lk.Lock() 284 | defer s.lk.Unlock() 285 | 286 | s.wantedCollections = opts.WantedCollections 287 | s.wantedDids = opts.WantedDIDs 288 | s.cursor = opts.Cursor 289 | s.compress = opts.Compress 290 | } 291 | 292 | // Terminate sends a close message to the subscriber 293 | func (s *Subscriber) Terminate(reason string) error { 294 | return s.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, reason)) 295 | } 296 | 297 | func (s *Subscriber) WriteMessage(msgType int, data []byte) error { 298 | s.conLk.Lock() 299 | defer s.conLk.Unlock() 300 | 301 | return s.ws.WriteMessage(msgType, data) 302 | } 303 | 304 | func (s *Subscriber) SetCursor(cursor *int64) { 305 | s.lk.Lock() 306 | defer s.lk.Unlock() 307 | 308 | s.cursor = cursor 309 | } 310 | 311 | // ParseMaxMessageSizeBytes parses a max size value string or integer and returns a 312 | // uint32. If the value is less than 0, it returns 0. If the value is not a 313 | // valid integer, it returns 0. 314 | func ParseMaxMessageSizeBytes[V int | string](value V) uint32 { 315 | if intValue, ok := any(value).(int); ok { 316 | if intValue < 0 { 317 | return 0 318 | } 319 | return uint32(intValue) 320 | } 321 | 322 | if strValue, ok := any(value).(string); ok { 323 | if strValue == "" { 324 | return 0 325 | } 326 | intValue, err := strconv.Atoi(strValue) 327 | if err != nil || intValue < 0 { 328 | return 0 329 | } 330 | return uint32(intValue) 331 | } 332 | 333 | return 0 334 | } 335 | -------------------------------------------------------------------------------- /pkg/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log/slog" 8 | "net/http" 9 | "strconv" 10 | "strings" 11 | "sync" 12 | "time" 13 | 14 | "github.com/bluesky-social/jetstream/pkg/consumer" 15 | "github.com/bluesky-social/jetstream/pkg/models" 16 | "github.com/gorilla/websocket" 17 | "github.com/labstack/echo/v4" 18 | "go.opentelemetry.io/otel" 19 | "golang.org/x/sync/semaphore" 20 | "golang.org/x/time/rate" 21 | ) 22 | 23 | type Server struct { 24 | Subscribers map[int64]*Subscriber 25 | lk sync.RWMutex 26 | nextSub int64 27 | Consumer *consumer.Consumer 28 | maxSubRate float64 29 | seq int64 30 | perIPLimiters map[string]*rate.Limiter 31 | } 32 | 33 | var upgrader = websocket.Upgrader{ 34 | // Allow all origins in websocket upgrade requests 35 | CheckOrigin: func(r *http.Request) bool { 36 | return true 37 | }, 38 | } 39 | var maxConcurrentEmits = int64(100) 40 | var cutoverThresholdUS = int64(1_000_000) 41 | var tracer = otel.Tracer("jetstream-server") 42 | 43 | func NewServer(maxSubRate float64) (*Server, error) { 44 | s := Server{ 45 | Subscribers: make(map[int64]*Subscriber), 46 | maxSubRate: maxSubRate, 47 | perIPLimiters: make(map[string]*rate.Limiter), 48 | } 49 | 50 | return &s, nil 51 | } 52 | 53 | func (s *Server) GetSeq() int64 { 54 | s.lk.RLock() 55 | defer s.lk.RUnlock() 56 | return s.seq 57 | } 58 | 59 | func (s *Server) SetSeq(seq int64) { 60 | s.lk.Lock() 61 | defer s.lk.Unlock() 62 | s.seq = seq 63 | } 64 | 65 | func (s *Server) HandleSubscribe(c echo.Context) error { 66 | ctx, cancel := context.WithCancel(c.Request().Context()) 67 | defer cancel() 68 | 69 | qWantedCollections := c.Request().URL.Query()["wantedCollections"] 70 | qWantedDids := c.Request().URL.Query()["wantedDids"] 71 | 72 | qMaxMessageSizeBytes := c.Request().URL.Query().Get("maxMessageSizeBytes") 73 | qMaxMessageSizeBytesValue := ParseMaxMessageSizeBytes(qMaxMessageSizeBytes) 74 | 75 | // Check if the user wants to initialize options over the websocket before subscribing 76 | requireHello := c.Request().URL.Query().Get("requireHello") == "true" 77 | 78 | // Check if the user wants zstd compression, can be set in the header or query param 79 | socketEncoding := c.Request().Header.Get("Socket-Encoding") 80 | compress := strings.Contains(socketEncoding, "zstd") || c.Request().URL.Query().Get("compress") == "true" 81 | 82 | // Parse the cursor 83 | var cursor *int64 84 | var err error 85 | qCursor := c.Request().URL.Query().Get("cursor") 86 | if qCursor != "" { 87 | cursor = new(int64) 88 | *cursor, err = strconv.ParseInt(qCursor, 10, 64) 89 | if err != nil { 90 | c.String(http.StatusBadRequest, fmt.Sprintf("invalid cursor: %s", qCursor)) 91 | return fmt.Errorf("invalid cursor: %s", qCursor) 92 | } 93 | 94 | // If given a future cursor, just live tail 95 | if *cursor > time.Now().UnixMicro() { 96 | cursor = nil 97 | } 98 | } 99 | 100 | // Parse the subscriber options 101 | subscriberOpts, err := parseSubscriberOptions(ctx, qWantedCollections, qWantedDids, compress, qMaxMessageSizeBytesValue, cursor) 102 | if err != nil { 103 | c.String(http.StatusBadRequest, err.Error()) 104 | return err 105 | } 106 | 107 | subIP := c.RealIP() 108 | 109 | ws, err := upgrader.Upgrade(c.Response(), c.Request(), nil) 110 | if err != nil { 111 | return err 112 | } 113 | defer ws.Close() 114 | 115 | writeWait := 10 * time.Second 116 | 117 | log := slog.With("source", "server_handle_subscribe", "socket_addr", ws.RemoteAddr().String(), "real_ip", subIP) 118 | 119 | sub, err := s.AddSubscriber(ws, subIP, subscriberOpts) 120 | if err != nil { 121 | log.Error("failed to add subscriber", "error", err) 122 | return err 123 | } 124 | defer s.RemoveSubscriber(sub.id) 125 | 126 | // Handle incoming messages 127 | go func() { 128 | for { 129 | msgType, msgBytes, err := ws.ReadMessage() 130 | if err != nil { 131 | log.Error("failed to read message from websocket", "error", err) 132 | cancel() 133 | return 134 | } 135 | 136 | if len(msgBytes) > 10_000_000 { 137 | log.Error("message from subscriber too large, ignoring", "size", len(msgBytes)) 138 | continue 139 | } 140 | 141 | switch msgType { 142 | case websocket.PingMessage: 143 | if err := sub.WriteMessage(websocket.PongMessage, nil); err != nil { 144 | log.Error("failed to write pong to websocket", "error", err) 145 | cancel() 146 | return 147 | } 148 | case websocket.PongMessage: 149 | log.Debug("received pong message from client") 150 | case websocket.CloseMessage: 151 | log.Info("received close message from client") 152 | cancel() 153 | return 154 | case websocket.TextMessage: 155 | log.Info("received text message from client, parsing as subscriber options update") 156 | log.Debug("text message from client content", "msg", string(msgBytes)) 157 | 158 | var subMessage SubscriberSourcedMessage 159 | if err := json.Unmarshal(msgBytes, &subMessage); err != nil { 160 | log.Error("failed to unmarshal subscriber sourced message", "error", err, "msg", string(msgBytes)) 161 | sub.Terminate(fmt.Sprintf("failed to unmarshal subscriber sourced message: %v", err)) 162 | cancel() 163 | return 164 | } 165 | 166 | switch subMessage.Type { 167 | case SubMessageOptionsUpdate: 168 | var subOptsUpdate SubscriberOptionsUpdatePayload 169 | if err := json.Unmarshal(subMessage.Payload, &subOptsUpdate); err != nil { 170 | log.Error("failed to unmarshal subscriber options update", "error", err, "msg", string(msgBytes)) 171 | sub.Terminate(fmt.Sprintf("failed to unmarshal subscriber options update: %v", err)) 172 | cancel() 173 | return 174 | } 175 | 176 | maxMessageSizeBytes := ParseMaxMessageSizeBytes(subOptsUpdate.MaxMessageSizeBytes) 177 | // Only WantedCollections and WantedDIDs can be updated after the initial connection 178 | // Cursor and Compression settings are fixed for the lifetime of the stream 179 | subscriberOpts, err = parseSubscriberOptions(ctx, subOptsUpdate.WantedCollections, subOptsUpdate.WantedDIDs, compress, maxMessageSizeBytes, sub.cursor) 180 | if err != nil { 181 | log.Error("failed to parse subscriber options", "error", err, "new_opts", subOptsUpdate) 182 | sub.Terminate(fmt.Sprintf("failed to parse subscriber options: %v", err)) 183 | cancel() 184 | return 185 | } 186 | 187 | sub.UpdateOptions(subscriberOpts) 188 | 189 | if requireHello { 190 | sub.hello <- struct{}{} 191 | requireHello = false 192 | } 193 | default: 194 | log.Warn("received unexpected message type from client, ignoring", "type", subMessage.Type) 195 | } 196 | case websocket.BinaryMessage: 197 | log.Warn("received unexpected binary message from client, ignoring") 198 | } 199 | } 200 | }() 201 | 202 | // If requireHello is set, wait for the client to send a valid options update message before proceeding 203 | if requireHello { 204 | log.Info("waiting for hello message from client") 205 | select { 206 | case <-ctx.Done(): 207 | log.Info("shutting down subscriber") 208 | return nil 209 | case <-sub.hello: 210 | log.Info("received hello message from client, proceeding") 211 | } 212 | } 213 | 214 | // Replay events if a cursor is provided 215 | if sub.cursor != nil { 216 | log.Info("replaying events", "cursor", *sub.cursor) 217 | playbackRateLimit := s.maxSubRate * 10 218 | 219 | go func() { 220 | for { 221 | lastSeq, err := s.Consumer.ReplayEvents(ctx, sub.compress, *sub.cursor, playbackRateLimit, func(ctx context.Context, timeUS int64, did, collection string, getEventBytes func() []byte) error { 222 | return emitToSubscriber(ctx, log, sub, timeUS, did, collection, true, getEventBytes) 223 | }) 224 | if err != nil { 225 | log.Error("failed to replay events", "error", err) 226 | sub.Terminate("failed to replay events") 227 | cancel() 228 | return 229 | } 230 | serverLastSeq := s.GetSeq() 231 | log.Info("finished replaying events", "replay_last_time", time.UnixMicro(lastSeq), "server_last_time", time.UnixMicro(serverLastSeq)) 232 | 233 | // If last event replayed is close enough to the last live event, start live tailing 234 | if lastSeq > serverLastSeq-(cutoverThresholdUS/2) { 235 | break 236 | } 237 | 238 | // Otherwise, update the cursor and replay again 239 | lastSeq++ 240 | sub.SetCursor(&lastSeq) 241 | } 242 | log.Info("finished replaying events, starting live tail") 243 | sub.SetCursor(nil) 244 | }() 245 | } 246 | 247 | // Read events from the outbox and send them to the subscriber 248 | pingPeriod := 30 * time.Second 249 | t := time.NewTicker(pingPeriod) 250 | defer t.Stop() 251 | for { 252 | select { 253 | case <-ctx.Done(): 254 | log.Info("shutting down subscriber") 255 | return nil 256 | case msg := <-sub.outbox: 257 | err := sub.rl.Wait(ctx) 258 | if err != nil { 259 | log.Error("failed to wait for rate limiter", "error", err) 260 | return fmt.Errorf("failed to wait for rate limiter: %w", err) 261 | } 262 | 263 | ws.SetWriteDeadline(time.Now().Add(writeWait)) 264 | 265 | // When compression is enabled, the msg is a zstd compressed message 266 | if compress { 267 | if err := sub.WriteMessage(websocket.BinaryMessage, *msg); err != nil { 268 | log.Error("failed to write message to websocket", "error", err) 269 | return nil 270 | } 271 | continue 272 | } 273 | 274 | // Otherwise, the msg is serialized JSON 275 | if err := sub.WriteMessage(websocket.TextMessage, *msg); err != nil { 276 | log.Error("failed to write message to websocket", "error", err) 277 | return nil 278 | } 279 | case <-t.C: 280 | ws.SetWriteDeadline(time.Now().Add(writeWait)) 281 | if err := sub.WriteMessage(websocket.PingMessage, nil); err != nil { 282 | log.Error("failed to write ping to websocket", "error", err) 283 | return nil 284 | } 285 | } 286 | } 287 | } 288 | 289 | func (s *Server) Emit(ctx context.Context, e *models.Event, asJSON, compBytes []byte) error { 290 | ctx, span := tracer.Start(ctx, "Emit") 291 | defer span.End() 292 | 293 | log := slog.With("source", "server_emit") 294 | 295 | s.lk.RLock() 296 | defer func() { 297 | s.lk.RUnlock() 298 | s.SetSeq(e.TimeUS) 299 | }() 300 | 301 | eventsEmitted.Inc() 302 | evtSize := float64(len(asJSON)) 303 | bytesEmitted.Add(evtSize) 304 | 305 | collection := "" 306 | if e.Kind == models.EventKindCommit && e.Commit != nil { 307 | collection = e.Commit.Collection 308 | } 309 | 310 | // Wrap the valuer functions for more lightweight event filtering 311 | getJSONEvent := func() []byte { return asJSON } 312 | getCompressedEvent := func() []byte { return compBytes } 313 | 314 | // Concurrently emit to all subscribers 315 | // We can't move on until all subscribers have received the event or been dropped for being too slow 316 | sem := semaphore.NewWeighted(maxConcurrentEmits) 317 | for _, sub := range s.Subscribers { 318 | if err := sem.Acquire(ctx, 1); err != nil { 319 | log.Error("failed to acquire semaphore", "error", err) 320 | return fmt.Errorf("failed to acquire semaphore: %w", err) 321 | } 322 | go func(sub *Subscriber) { 323 | defer sem.Release(1) 324 | sub.lk.Lock() 325 | defer sub.lk.Unlock() 326 | 327 | // Don't emit events to subscribers that are replaying and are too far behind 328 | if sub.cursor != nil && sub.seq < e.TimeUS-cutoverThresholdUS || sub.tearingDown { 329 | return 330 | } 331 | 332 | // Pick the event valuer for the subscriber based on their compression preference 333 | getEventBytes := getJSONEvent 334 | if sub.compress { 335 | getEventBytes = getCompressedEvent 336 | } 337 | 338 | emitToSubscriber(ctx, log, sub, e.TimeUS, e.Did, collection, false, getEventBytes) 339 | }(sub) 340 | } 341 | 342 | if err := sem.Acquire(ctx, maxConcurrentEmits); err != nil { 343 | log.Error("failed to acquire semaphore", "error", err) 344 | return fmt.Errorf("failed to acquire semaphore: %w", err) 345 | } 346 | 347 | return nil 348 | } 349 | -------------------------------------------------------------------------------- /cmd/jetstream/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "log/slog" 9 | "net/http" 10 | _ "net/http/pprof" 11 | "net/url" 12 | "os" 13 | "os/signal" 14 | "syscall" 15 | "time" 16 | 17 | "github.com/bluesky-social/indigo/events" 18 | "github.com/bluesky-social/indigo/events/schedulers/parallel" 19 | "github.com/bluesky-social/jetstream/pkg/consumer" 20 | "github.com/bluesky-social/jetstream/pkg/server" 21 | "github.com/gorilla/websocket" 22 | "github.com/labstack/echo/v4" 23 | "github.com/labstack/echo/v4/middleware" 24 | "github.com/prometheus/client_golang/prometheus/promhttp" 25 | 26 | "github.com/urfave/cli/v2" 27 | ) 28 | 29 | func main() { 30 | app := cli.App{ 31 | Name: "jetstream", 32 | Usage: "atproto firehose translation service", 33 | Version: "0.1.0", 34 | } 35 | 36 | app.Flags = []cli.Flag{ 37 | &cli.StringFlag{ 38 | Name: "ws-url", 39 | Usage: "full websocket path to the ATProto SubscribeRepos XRPC endpoint", 40 | Value: "wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos", 41 | EnvVars: []string{"JETSTREAM_WS_URL"}, 42 | }, 43 | &cli.IntFlag{ 44 | Name: "worker-count", 45 | Usage: "number of workers to process events", 46 | Value: 100, 47 | EnvVars: []string{"JETSTREAM_WORKER_COUNT"}, 48 | }, 49 | &cli.IntFlag{ 50 | Name: "max-queue-size", 51 | Usage: "max number of events to queue", 52 | Value: 1000, 53 | EnvVars: []string{"JETSTREAM_MAX_QUEUE_SIZE"}, 54 | }, 55 | &cli.StringFlag{ 56 | Name: "listen-addr", 57 | Usage: "addr to serve echo on", 58 | Value: ":6008", 59 | EnvVars: []string{"JETSTREAM_LISTEN_ADDR"}, 60 | }, 61 | &cli.StringFlag{ 62 | Name: "metrics-listen-addr", 63 | Usage: "addr to serve prometheus metrics on", 64 | Value: ":6009", 65 | EnvVars: []string{"JETSTREAM_METRICS_LISTEN_ADDR"}, 66 | }, 67 | &cli.StringFlag{ 68 | Name: "data-dir", 69 | Usage: "directory to store data (pebbleDB)", 70 | Value: "./data", 71 | EnvVars: []string{"JETSTREAM_DATA_DIR"}, 72 | }, 73 | &cli.StringFlag{ 74 | Name: "zstd-dictionary-path", 75 | Usage: "path to the zstd dictionary file", 76 | EnvVars: []string{"JETSTREAM_ZSTD_DICTIONARY_PATH"}, 77 | Required: false, 78 | }, 79 | &cli.DurationFlag{ 80 | Name: "event-ttl", 81 | Usage: "time to live for events", 82 | Value: 24 * time.Hour, 83 | EnvVars: []string{"JETSTREAM_EVENT_TTL"}, 84 | }, 85 | &cli.Float64Flag{ 86 | Name: "max-sub-rate", 87 | Usage: "max rate of events per second we can send to a subscriber", 88 | Value: 5_000, 89 | EnvVars: []string{"JETSTREAM_MAX_SUB_RATE"}, 90 | }, 91 | &cli.Int64Flag{ 92 | Name: "override-relay-cursor", 93 | Usage: "override cursor to start from, if not set will start from the last cursor in the database, if no cursor in the database will start from live", 94 | Value: -1, 95 | EnvVars: []string{"JETSTREAM_OVERRIDE_RELAY_CURSOR"}, 96 | }, 97 | &cli.DurationFlag{ 98 | Name: "liveness-ttl", 99 | Usage: "time to restart when no event detected", 100 | Value: 15 * time.Second, 101 | EnvVars: []string{"JETSTREAM_LIVENESS_TTL"}, 102 | }, 103 | } 104 | 105 | app.Action = Jetstream 106 | 107 | err := app.Run(os.Args) 108 | if err != nil { 109 | log.Fatal(err) 110 | } 111 | } 112 | 113 | // Jetstream is the main function for jetstream 114 | func Jetstream(cctx *cli.Context) error { 115 | ctx := cctx.Context 116 | 117 | log := slog.New(slog.NewJSONHandler(os.Stdout, nil)) 118 | slog.SetDefault(log) 119 | 120 | log.Info("starting jetstream") 121 | 122 | u, err := url.Parse(cctx.String("ws-url")) 123 | if err != nil { 124 | return fmt.Errorf("failed to parse ws-url: %w", err) 125 | } 126 | 127 | s, err := server.NewServer(cctx.Float64("max-sub-rate")) 128 | if err != nil { 129 | return fmt.Errorf("failed to create server: %w", err) 130 | } 131 | 132 | c, err := consumer.NewConsumer( 133 | ctx, 134 | log, 135 | u.String(), 136 | cctx.String("data-dir"), 137 | cctx.Duration("event-ttl"), 138 | s.Emit, 139 | ) 140 | if err != nil { 141 | return fmt.Errorf("failed to create consumer: %w", err) 142 | } 143 | 144 | s.Consumer = c 145 | 146 | scheduler := parallel.NewScheduler(cctx.Int("worker-count"), cctx.Int("max-queue-size"), "prod-firehose", c.HandleStreamEvent) 147 | 148 | // Start a goroutine to manage the cursor, saving the current cursor every 5 seconds. 149 | shutdownCursorManager := make(chan struct{}) 150 | cursorManagerShutdown := make(chan struct{}) 151 | go func() { 152 | ctx := context.Background() 153 | ticker := time.NewTicker(5 * time.Second) 154 | log := log.With("source", "cursor_manager") 155 | 156 | for { 157 | select { 158 | case <-shutdownCursorManager: 159 | log.Info("shutting down cursor manager") 160 | err := c.WriteCursor(ctx) 161 | if err != nil { 162 | log.Error("failed to write cursor", "error", err) 163 | } 164 | log.Info("cursor manager shut down successfully") 165 | close(cursorManagerShutdown) 166 | return 167 | case <-ticker.C: 168 | err := c.WriteCursor(ctx) 169 | if err != nil { 170 | log.Error("failed to write cursor", "error", err) 171 | } 172 | } 173 | } 174 | }() 175 | 176 | // Create a channel that will be closed when we want to stop the application 177 | // Usually when a critical routine returns an error 178 | livenessKill := make(chan struct{}) 179 | 180 | // Start a goroutine to manage the liveness checker, shutting down if no events are received for liveness-ttl 181 | shutdownLivenessChecker := make(chan struct{}) 182 | livenessCheckerShutdown := make(chan struct{}) 183 | go func() { 184 | ticker := time.NewTicker(cctx.Duration("liveness-ttl")) 185 | lastSeq := int64(0) 186 | log := log.With("source", "liveness_checker") 187 | 188 | for { 189 | select { 190 | case <-shutdownLivenessChecker: 191 | log.Info("shutting down liveness checker") 192 | close(livenessCheckerShutdown) 193 | return 194 | case <-ticker.C: 195 | seq, _ := c.Progress.Get() 196 | if seq == lastSeq && seq != 0 { 197 | log.Error("no new events in last "+cctx.Duration("liveness-ttl").String()+", shutting down for docker to restart me", "seq", seq) 198 | close(livenessKill) 199 | } else { 200 | // Trim the database 201 | err := c.TrimEvents(ctx) 202 | if err != nil { 203 | log.Error("failed to trim events", "error", err) 204 | } 205 | log.Info("successful liveness check and trim", "seq", seq) 206 | lastSeq = seq 207 | } 208 | } 209 | } 210 | }() 211 | 212 | m := echo.New() 213 | m.GET("/metrics", echo.WrapHandler(promhttp.Handler())) 214 | m.GET("/debug/pprof/*", echo.WrapHandler(http.DefaultServeMux)) 215 | 216 | metricsServer := &http.Server{ 217 | Addr: cctx.String("metrics-listen-addr"), 218 | Handler: m, 219 | } 220 | 221 | e := echo.New() 222 | e.Use(middleware.CORSWithConfig(middleware.CORSConfig{ 223 | AllowOrigins: []string{"*"}, 224 | AllowMethods: []string{http.MethodGet, http.MethodHead, http.MethodOptions}, 225 | })) 226 | e.GET("/", func(c echo.Context) error { 227 | return c.String(http.StatusOK, "Welcome to Jetstream") 228 | }) 229 | e.GET("/subscribe", s.HandleSubscribe) 230 | 231 | jetServer := &http.Server{ 232 | Addr: cctx.String("listen-addr"), 233 | Handler: e, 234 | } 235 | 236 | // Startup echo server 237 | shutdownEcho := make(chan struct{}) 238 | echoShutdown := make(chan struct{}) 239 | go func() { 240 | logger := log.With("source", "echo_server") 241 | 242 | logger.Info("echo server listening", "addr", cctx.String("listen-addr")) 243 | 244 | go func() { 245 | if err := jetServer.ListenAndServe(); err != http.ErrServerClosed { 246 | logger.Error("failed to start echo server", "error", err) 247 | } 248 | }() 249 | 250 | <-shutdownEcho 251 | if err := jetServer.Shutdown(ctx); err != nil { 252 | logger.Error("failed to shutdown echo server", "error", err) 253 | } 254 | logger.Info("echo server shut down") 255 | close(echoShutdown) 256 | }() 257 | 258 | // Startup metrics server 259 | shutdownMetrics := make(chan struct{}) 260 | metricsShutdown := make(chan struct{}) 261 | go func() { 262 | logger := log.With("source", "metrics_server") 263 | 264 | logger.Info("metrics server listening", "addr", cctx.String("metrics-listen-addr")) 265 | 266 | go func() { 267 | if err := metricsServer.ListenAndServe(); err != http.ErrServerClosed { 268 | logger.Error("failed to start metrics server", "error", err) 269 | } 270 | }() 271 | 272 | <-shutdownMetrics 273 | if err := metricsServer.Shutdown(ctx); err != nil { 274 | logger.Error("failed to shutdown metrics server", "error", err) 275 | } 276 | logger.Info("metrics server shut down") 277 | close(metricsShutdown) 278 | }() 279 | 280 | var cursor *int64 281 | cursorOverride := cctx.Int64("override-relay-cursor") 282 | 283 | // If the last cursor in the database is set, use that as the cursor 284 | if c.Progress.LastSeq >= 0 { 285 | cursor = &c.Progress.LastSeq 286 | } 287 | 288 | // If the override cursor is set, use that instead of the last cursor in the database 289 | if cursorOverride >= 0 { 290 | log.Info("overriding cursor", "cursor", cursorOverride) 291 | cursor = &cursorOverride 292 | } 293 | 294 | // If the cursor is nil, we are starting from live 295 | if cursor != nil { 296 | u.RawQuery = fmt.Sprintf("cursor=%d", *cursor) 297 | } 298 | 299 | log.Info("connecting to websocket", "url", u.String()) 300 | con, _, err := websocket.DefaultDialer.Dial(u.String(), nil) 301 | if err != nil { 302 | log.Error("failed to connect to websocket", "error", err) 303 | return err 304 | } 305 | defer con.Close() 306 | 307 | // Create a channel that will be closed when we want to stop the application 308 | // Usually when a critical routine returns an error 309 | eventsKill := make(chan struct{}) 310 | 311 | shutdownRepoStream := make(chan struct{}) 312 | repoStreamShutdown := make(chan struct{}) 313 | go func() { 314 | ctx := context.Background() 315 | ctx, cancel := context.WithCancel(ctx) 316 | go func() { 317 | logger := log.With("source", "repo_stream") 318 | err = events.HandleRepoStream(ctx, con, scheduler, logger) 319 | if !errors.Is(err, context.Canceled) { 320 | logger.Info("HandleRepoStream returned unexpectedly, killing jetstream", "error", err) 321 | close(eventsKill) 322 | } else { 323 | logger.Info("HandleRepoStream closed on context cancel") 324 | } 325 | close(repoStreamShutdown) 326 | }() 327 | <-shutdownRepoStream 328 | cancel() 329 | }() 330 | 331 | // Trap SIGINT to trigger a shutdown. 332 | signals := make(chan os.Signal, 1) 333 | signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) 334 | 335 | select { 336 | case <-signals: 337 | log.Info("shutting down on signal") 338 | case <-ctx.Done(): 339 | log.Info("shutting down on context done") 340 | case <-livenessKill: 341 | log.Info("shutting down on liveness kill") 342 | case <-eventsKill: 343 | log.Info("shutting down on events kill") 344 | } 345 | 346 | log.Info("shutting down, waiting for workers to clean up...") 347 | 348 | close(shutdownRepoStream) 349 | close(shutdownLivenessChecker) 350 | close(shutdownCursorManager) 351 | close(shutdownEcho) 352 | close(shutdownMetrics) 353 | 354 | shutdownTimeout := time.After(10 * time.Second) 355 | 356 | select { 357 | case <-repoStreamShutdown: 358 | log.Info("Repo stream shutdown completed") 359 | case <-shutdownTimeout: 360 | log.Warn("Shutdown timeout reached for repo stream") 361 | } 362 | 363 | select { 364 | case <-livenessCheckerShutdown: 365 | log.Info("Liveness checker shutdown completed") 366 | case <-shutdownTimeout: 367 | log.Warn("Shutdown timeout reached for liveness checker") 368 | } 369 | 370 | select { 371 | case <-cursorManagerShutdown: 372 | log.Info("Cursor manager shutdown completed") 373 | case <-shutdownTimeout: 374 | log.Warn("Shutdown timeout reached for cursor manager") 375 | } 376 | 377 | select { 378 | case <-echoShutdown: 379 | log.Info("Echo shutdown completed") 380 | case <-shutdownTimeout: 381 | log.Warn("Shutdown timeout reached for echo server") 382 | } 383 | 384 | select { 385 | case <-metricsShutdown: 386 | log.Info("Metrics shutdown completed") 387 | case <-shutdownTimeout: 388 | log.Warn("Shutdown timeout reached for metrics server") 389 | } 390 | 391 | c.Shutdown() 392 | 393 | err = c.UncompressedDB.Close() 394 | if err != nil { 395 | log.Error("failed to close pebble db", "error", err) 396 | } 397 | 398 | err = c.CompressedDB.Close() 399 | if err != nil { 400 | log.Error("failed to close compressed pebble db", "error", err) 401 | } 402 | 403 | log.Info("shut down successfully") 404 | 405 | return nil 406 | } 407 | -------------------------------------------------------------------------------- /pkg/consumer/consumer.go: -------------------------------------------------------------------------------- 1 | package consumer 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "log/slog" 8 | "strings" 9 | "time" 10 | 11 | comatproto "github.com/bluesky-social/indigo/api/atproto" 12 | "github.com/bluesky-social/indigo/atproto/atdata" 13 | "github.com/bluesky-social/indigo/events" 14 | "github.com/bluesky-social/indigo/repo" 15 | "github.com/bluesky-social/indigo/repomgr" 16 | "github.com/bluesky-social/jetstream/pkg/models" 17 | "github.com/bluesky-social/jetstream/pkg/monotonic" 18 | "github.com/cockroachdb/pebble" 19 | "github.com/goccy/go-json" 20 | "github.com/klauspost/compress/zstd" 21 | "github.com/prometheus/client_golang/prometheus" 22 | "go.opentelemetry.io/otel" 23 | "go.opentelemetry.io/otel/attribute" 24 | ) 25 | 26 | // Consumer is the consumer of the firehose 27 | type Consumer struct { 28 | SocketURL string 29 | Progress *Progress 30 | Emit func(context.Context, *models.Event, []byte, []byte) error 31 | UncompressedDB *pebble.DB 32 | CompressedDB *pebble.DB 33 | encoder *zstd.Encoder 34 | EventTTL time.Duration 35 | logger *slog.Logger 36 | clock *monotonic.Clock 37 | buf chan *models.Event 38 | sequencerShutdown chan chan struct{} 39 | 40 | sequenced prometheus.Counter 41 | persisted prometheus.Counter 42 | emitted prometheus.Counter 43 | } 44 | 45 | var tracer = otel.Tracer("consumer") 46 | 47 | // NewConsumer creates a new consumer 48 | func NewConsumer( 49 | ctx context.Context, 50 | logger *slog.Logger, 51 | socketURL string, 52 | dataDir string, 53 | eventTTL time.Duration, 54 | emit func(context.Context, *models.Event, []byte, []byte) error, 55 | ) (*Consumer, error) { 56 | uDBPath := dataDir + "/jetstream.uncompressed.db" 57 | uDB, err := pebble.Open(uDBPath, &pebble.Options{}) 58 | if err != nil { 59 | return nil, fmt.Errorf("failed to open db: %w", err) 60 | } 61 | 62 | cDBPath := dataDir + "/jetstream.compressed.db" 63 | cDB, err := pebble.Open(cDBPath, &pebble.Options{}) 64 | if err != nil { 65 | return nil, fmt.Errorf("failed to open db: %w", err) 66 | } 67 | 68 | log := logger.With("component", "consumer") 69 | 70 | clock, err := monotonic.NewClock(time.Microsecond) 71 | if err != nil { 72 | return nil, fmt.Errorf("failed to create clock: %w", err) 73 | } 74 | 75 | // Create a zstd encoder using the dictionary and a window size of 128KiB 76 | encoder, err := zstd.NewWriter(nil, zstd.WithEncoderDict(models.ZSTDDictionary), zstd.WithWindowSize(1<<17), zstd.WithEncoderConcurrency(1)) 77 | if err != nil { 78 | return nil, fmt.Errorf("failed to create zstd encoder: %w", err) 79 | } 80 | 81 | c := Consumer{ 82 | SocketURL: socketURL, 83 | Progress: &Progress{ 84 | LastSeq: -1, 85 | }, 86 | EventTTL: eventTTL, 87 | Emit: emit, 88 | UncompressedDB: uDB, 89 | CompressedDB: cDB, 90 | encoder: encoder, 91 | logger: log, 92 | clock: clock, 93 | buf: make(chan *models.Event, 10_000), 94 | sequencerShutdown: make(chan chan struct{}), 95 | 96 | sequenced: eventsSequencedCounter.WithLabelValues(socketURL), 97 | persisted: eventsPersistedCounter.WithLabelValues(socketURL), 98 | emitted: eventsEmittedCounter.WithLabelValues(socketURL), 99 | } 100 | 101 | // Check to see if the cursor exists 102 | err = c.ReadCursor(ctx) 103 | if err != nil { 104 | log.Warn("previous cursor not found, starting from live", "error", err) 105 | } 106 | 107 | // Start the sequencer 108 | if err := c.RunSequencer(ctx); err != nil { 109 | return nil, fmt.Errorf("failed to start sequencer: %w", err) 110 | } 111 | 112 | return &c, nil 113 | } 114 | 115 | // HandleStreamEvent handles a stream event from the firehose 116 | func (c *Consumer) HandleStreamEvent(ctx context.Context, xe *events.XRPCStreamEvent) error { 117 | ctx, span := tracer.Start(ctx, "HandleStreamEvent") 118 | defer span.End() 119 | 120 | select { 121 | case <-ctx.Done(): 122 | return ctx.Err() 123 | default: 124 | } 125 | 126 | switch { 127 | case xe.RepoCommit != nil: 128 | eventsProcessedCounter.WithLabelValues("commit", c.SocketURL).Inc() 129 | if xe.RepoCommit.TooBig { 130 | c.logger.Warn("repo commit too big", "repo", xe.RepoCommit.Repo, "seq", xe.RepoCommit.Seq, "rev", xe.RepoCommit.Rev) 131 | return nil 132 | } 133 | return c.HandleRepoCommit(ctx, xe.RepoCommit) 134 | case xe.RepoIdentity != nil: 135 | eventsProcessedCounter.WithLabelValues("identity", c.SocketURL).Inc() 136 | now := time.Now() 137 | c.Progress.Update(xe.RepoIdentity.Seq, now) 138 | // Parse time from the event time string 139 | t, err := time.Parse(time.RFC3339, xe.RepoIdentity.Time) 140 | if err != nil { 141 | c.logger.Error("error parsing time", "error", err) 142 | return nil 143 | } 144 | 145 | // Emit identity update 146 | e := models.Event{ 147 | Did: xe.RepoIdentity.Did, 148 | Kind: models.EventKindIdentity, 149 | Identity: xe.RepoIdentity, 150 | } 151 | // Send to the sequencer 152 | c.buf <- &e 153 | lastEvtCreatedAtGauge.WithLabelValues(c.SocketURL).Set(float64(t.UnixNano())) 154 | lastEvtProcessedAtGauge.WithLabelValues(c.SocketURL).Set(float64(now.UnixNano())) 155 | lastEvtCreatedEvtProcessedGapGauge.WithLabelValues(c.SocketURL).Set(float64(now.Sub(t).Seconds())) 156 | lastSeqGauge.WithLabelValues(c.SocketURL).Set(float64(xe.RepoIdentity.Seq)) 157 | case xe.RepoAccount != nil: 158 | eventsProcessedCounter.WithLabelValues("account", c.SocketURL).Inc() 159 | now := time.Now() 160 | c.Progress.Update(xe.RepoAccount.Seq, now) 161 | // Parse time from the event time string 162 | t, err := time.Parse(time.RFC3339, xe.RepoAccount.Time) 163 | if err != nil { 164 | c.logger.Error("error parsing time", "error", err) 165 | return nil 166 | } 167 | 168 | // Emit account update 169 | e := models.Event{ 170 | Did: xe.RepoAccount.Did, 171 | Kind: models.EventKindAccount, 172 | Account: xe.RepoAccount, 173 | } 174 | // Send to the sequencer 175 | c.buf <- &e 176 | lastEvtCreatedAtGauge.WithLabelValues(c.SocketURL).Set(float64(t.UnixNano())) 177 | lastEvtProcessedAtGauge.WithLabelValues(c.SocketURL).Set(float64(now.UnixNano())) 178 | lastEvtCreatedEvtProcessedGapGauge.WithLabelValues(c.SocketURL).Set(float64(now.Sub(t).Seconds())) 179 | lastSeqGauge.WithLabelValues(c.SocketURL).Set(float64(xe.RepoAccount.Seq)) 180 | case xe.Error != nil: 181 | eventsProcessedCounter.WithLabelValues("error", c.SocketURL).Inc() 182 | return fmt.Errorf("error from firehose: %s", xe.Error.Message) 183 | } 184 | return nil 185 | } 186 | 187 | // HandleRepoCommit handles a repo commit event from the firehose and processes the records 188 | func (c *Consumer) HandleRepoCommit(ctx context.Context, evt *comatproto.SyncSubscribeRepos_Commit) error { 189 | ctx, span := tracer.Start(ctx, "HandleRepoCommit") 190 | defer span.End() 191 | 192 | processedAt := time.Now() 193 | 194 | c.Progress.Update(evt.Seq, processedAt) 195 | 196 | lastSeqGauge.WithLabelValues(c.SocketURL).Set(float64(evt.Seq)) 197 | 198 | log := c.logger.With("repo", evt.Repo, "seq", evt.Seq, "commit", evt.Commit.String()) 199 | 200 | span.AddEvent("Read Repo From Car") 201 | rr, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(evt.Blocks)) 202 | if err != nil { 203 | log.Error("failed to read repo from car", "error", err) 204 | return nil 205 | } 206 | 207 | // Parse time from the event time string 208 | evtCreatedAt, err := time.Parse(time.RFC3339, evt.Time) 209 | if err != nil { 210 | log.Error("error parsing time", "error", err) 211 | return nil 212 | } 213 | 214 | lastEvtCreatedAtGauge.WithLabelValues(c.SocketURL).Set(float64(evtCreatedAt.UnixNano())) 215 | lastEvtProcessedAtGauge.WithLabelValues(c.SocketURL).Set(float64(processedAt.UnixNano())) 216 | lastEvtCreatedEvtProcessedGapGauge.WithLabelValues(c.SocketURL).Set(float64(processedAt.Sub(evtCreatedAt).Seconds())) 217 | 218 | for _, op := range evt.Ops { 219 | collection := strings.Split(op.Path, "/")[0] 220 | rkey := strings.Split(op.Path, "/")[1] 221 | 222 | ek := repomgr.EventKind(op.Action) 223 | log = log.With("action", op.Action, "collection", collection) 224 | 225 | opsProcessedCounter.WithLabelValues(op.Action, collection, c.SocketURL).Inc() 226 | 227 | // recordURI := "at://" + evt.Repo + "/" + op.Path 228 | span.SetAttributes(attribute.String("repo", evt.Repo)) 229 | span.SetAttributes(attribute.String("collection", collection)) 230 | span.SetAttributes(attribute.String("rkey", rkey)) 231 | span.SetAttributes(attribute.Int64("seq", evt.Seq)) 232 | span.SetAttributes(attribute.String("event_kind", op.Action)) 233 | 234 | e := models.Event{ 235 | Did: evt.Repo, 236 | Kind: models.EventKindCommit, 237 | } 238 | 239 | switch ek { 240 | case repomgr.EvtKindCreateRecord: 241 | if op.Cid == nil { 242 | log.Error("update record op missing cid") 243 | continue 244 | } 245 | 246 | rcid, recB, err := rr.GetRecordBytes(ctx, op.Path) 247 | if err != nil { 248 | log.Error("failed to get record bytes", "error", err) 249 | continue 250 | } 251 | 252 | recCid := rcid.String() 253 | if recCid != op.Cid.String() { 254 | log.Error("record cid mismatch", "expected", *op.Cid, "actual", rcid) 255 | continue 256 | } 257 | 258 | rec, err := atdata.UnmarshalCBOR(*recB) 259 | if err != nil { 260 | log.Error("failed to unmarshal record", "error", err) 261 | continue 262 | } 263 | 264 | recJSON, err := json.Marshal(rec) 265 | if err != nil { 266 | log.Error("failed to marshal record to json", "error", err) 267 | continue 268 | } 269 | 270 | e.Commit = &models.Commit{ 271 | Rev: evt.Rev, 272 | Operation: models.CommitOperationCreate, 273 | Collection: collection, 274 | RKey: rkey, 275 | Record: recJSON, 276 | CID: recCid, 277 | } 278 | case repomgr.EvtKindUpdateRecord: 279 | if op.Cid == nil { 280 | log.Error("update record op missing cid") 281 | continue 282 | } 283 | 284 | rcid, recB, err := rr.GetRecordBytes(ctx, op.Path) 285 | if err != nil { 286 | log.Error("failed to get record bytes", "error", err) 287 | continue 288 | } 289 | 290 | recCid := rcid.String() 291 | if recCid != op.Cid.String() { 292 | log.Error("record cid mismatch", "expected", *op.Cid, "actual", rcid) 293 | continue 294 | } 295 | 296 | rec, err := atdata.UnmarshalCBOR(*recB) 297 | if err != nil { 298 | log.Error("failed to unmarshal record", "error", err) 299 | continue 300 | } 301 | 302 | recJSON, err := json.Marshal(rec) 303 | if err != nil { 304 | log.Error("failed to marshal record to json", "error", err) 305 | continue 306 | } 307 | 308 | e.Commit = &models.Commit{ 309 | Rev: evt.Rev, 310 | Operation: models.CommitOperationUpdate, 311 | Collection: collection, 312 | RKey: rkey, 313 | Record: recJSON, 314 | CID: recCid, 315 | } 316 | case repomgr.EvtKindDeleteRecord: 317 | // Emit the delete 318 | e.Commit = &models.Commit{ 319 | Rev: evt.Rev, 320 | Operation: models.CommitOperationDelete, 321 | Collection: collection, 322 | RKey: rkey, 323 | } 324 | default: 325 | log.Warn("unknown event kind from op action", "kind", op.Action) 326 | continue 327 | } 328 | 329 | // Send to the sequencer 330 | c.buf <- &e 331 | } 332 | 333 | eventProcessingDurationHistogram.WithLabelValues(c.SocketURL).Observe(time.Since(processedAt).Seconds()) 334 | return nil 335 | } 336 | 337 | func (c *Consumer) RunSequencer(ctx context.Context) error { 338 | log := c.logger.With("component", "sequencer") 339 | 340 | go func() { 341 | for { 342 | select { 343 | case <-ctx.Done(): 344 | log.Info("shutting down sequencer on context completion") 345 | return 346 | case s := <-c.sequencerShutdown: 347 | log.Info("shutting down sequencer on shutdown signal") 348 | s <- struct{}{} 349 | return 350 | case e := <-c.buf: 351 | // Assign a time_us to the event 352 | e.TimeUS = c.clock.Now() 353 | c.sequenced.Inc() 354 | 355 | // Serialize the event as JSON 356 | asJSON, err := json.Marshal(e) 357 | if err != nil { 358 | log.Error("failed to marshal event", "error", err) 359 | return 360 | } 361 | 362 | // Compress the serialized JSON using zstd 363 | compBytes := c.encoder.EncodeAll(asJSON, nil) 364 | 365 | // Persist the event to the uncompressed and compressed DBs 366 | if err := c.PersistEvent(ctx, e, asJSON, compBytes); err != nil { 367 | log.Error("failed to persist event", "error", err) 368 | return 369 | } 370 | c.persisted.Inc() 371 | 372 | // Emit the event to subscribers 373 | if err := c.Emit(ctx, e, asJSON, compBytes); err != nil { 374 | log.Error("failed to emit event", "error", err) 375 | } 376 | c.emitted.Inc() 377 | } 378 | } 379 | }() 380 | 381 | return nil 382 | } 383 | 384 | func (c *Consumer) Shutdown() { 385 | shutdownTimeout := time.After(10 * time.Second) 386 | shutdown := make(chan struct{}) 387 | c.sequencerShutdown <- shutdown 388 | 389 | select { 390 | case <-shutdownTimeout: 391 | c.logger.Warn("sequencer shutdown timed out") 392 | case <-shutdown: 393 | c.logger.Info("sequencer shutdown complete") 394 | } 395 | } 396 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= 3 | github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= 4 | github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b h1:5/++qT1/z812ZqBvqQt6ToRswSuPZ/B33m6xVHRzADU= 5 | github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b/go.mod h1:4+EPqMRApwwE/6yo6CxiHoSnBzjRr3jsqer7frxP8y4= 6 | github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM= 7 | github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= 8 | github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= 9 | github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= 10 | github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= 11 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 12 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 13 | github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= 14 | github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= 15 | github.com/bluesky-social/indigo v0.0.0-20251009212240-20524de167fe h1:VBhaqE5ewQgXbY5SfSWFZC/AwHFo7cHxZKFYi2ce9Yo= 16 | github.com/bluesky-social/indigo v0.0.0-20251009212240-20524de167fe/go.mod h1:RuQVrCGm42QNsgumKaR6se+XkFKfCPNwdCiTvqKRUck= 17 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= 18 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= 19 | github.com/carlmjohnson/versioninfo v0.22.5 h1:O00sjOLUAFxYQjlN/bzYTuZiS0y6fWDQjMRvwtKgwwc= 20 | github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8= 21 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 22 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 23 | github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= 24 | github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= 25 | github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= 26 | github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= 27 | github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= 28 | github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= 29 | github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= 30 | github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= 31 | github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= 32 | github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= 33 | github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= 34 | github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= 35 | github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= 36 | github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= 37 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 38 | github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= 39 | github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 40 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 41 | github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= 42 | github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= 43 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 44 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 45 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 46 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= 47 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= 48 | github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= 49 | github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 50 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 51 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 52 | github.com/getsentry/sentry-go v0.28.0 h1:7Rqx9M3ythTKy2J6uZLHmc8Sz9OGgIlseuO1iBX/s0M= 53 | github.com/getsentry/sentry-go v0.28.0/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= 54 | github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= 55 | github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= 56 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 57 | github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= 58 | github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 59 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 60 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 61 | github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= 62 | github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= 63 | github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= 64 | github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= 65 | github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= 66 | github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus= 67 | github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4= 68 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 69 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 70 | github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= 71 | github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= 72 | github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 73 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 74 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 75 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 76 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 77 | github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= 78 | github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= 79 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 80 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 81 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 82 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= 83 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 84 | github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= 85 | github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= 86 | github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= 87 | github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= 88 | github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= 89 | github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= 90 | github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= 91 | github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= 92 | github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= 93 | github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= 94 | github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= 95 | github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 96 | github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= 97 | github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= 98 | github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= 99 | github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= 100 | github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= 101 | github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= 102 | github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= 103 | github.com/ipfs/go-bitswap v0.11.0/go.mod h1:05aE8H3XOU+LXpTedeAS0OZpcO1WFsj5niYQH9a1Tmk= 104 | github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= 105 | github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= 106 | github.com/ipfs/go-blockservice v0.5.2 h1:in9Bc+QcXwd1apOVM7Un9t8tixPKdaHQFdLSUM1Xgk8= 107 | github.com/ipfs/go-blockservice v0.5.2/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk= 108 | github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= 109 | github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= 110 | github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= 111 | github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= 112 | github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= 113 | github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= 114 | github.com/ipfs/go-ds-flatfs v0.5.1 h1:ZCIO/kQOS/PSh3vcF1H6a8fkRGS7pOfwfPdx4n/KJH4= 115 | github.com/ipfs/go-ds-flatfs v0.5.1/go.mod h1:RWTV7oZD/yZYBKdbVIFXTX2fdY2Tbvl94NsWqmoyAX4= 116 | github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= 117 | github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= 118 | github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= 119 | github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= 120 | github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= 121 | github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= 122 | github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw= 123 | github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo= 124 | github.com/ipfs/go-ipfs-exchange-interface v0.2.1 h1:jMzo2VhLKSHbVe+mHNzYgs95n0+t0Q69GQ5WhRDZV/s= 125 | github.com/ipfs/go-ipfs-exchange-interface v0.2.1/go.mod h1:MUsYn6rKbG6CTtsDp+lKJPmVt3ZrCViNyH3rfPGsZ2E= 126 | github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= 127 | github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s= 128 | github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= 129 | github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= 130 | github.com/ipfs/go-ipfs-routing v0.3.0 h1:9W/W3N+g+y4ZDeffSgqhgo7BsBSJwPMcyssET9OWevc= 131 | github.com/ipfs/go-ipfs-routing v0.3.0/go.mod h1:dKqtTFIql7e1zYsEuWLyuOU+E0WJWW8JjbTPLParDWo= 132 | github.com/ipfs/go-ipfs-util v0.0.3 h1:2RFdGez6bu2ZlZdI+rWfIdbQb1KudQp3VGwPtdNCmE0= 133 | github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn17xAKWBvs= 134 | github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs= 135 | github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk= 136 | github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten1F5U= 137 | github.com/ipfs/go-ipld-format v0.6.0/go.mod h1:g4QVMTn3marU3qXchwjpKPKgJv+zF+OlaKMyhJ4LHPg= 138 | github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk= 139 | github.com/ipfs/go-ipld-legacy v0.2.1/go.mod h1:782MOUghNzMO2DER0FlBR94mllfdCJCkTtDtPM51otM= 140 | github.com/ipfs/go-libipfs v0.7.0 h1:Mi54WJTODaOL2/ZSm5loi3SwI3jI2OuFWUrQIkJ5cpM= 141 | github.com/ipfs/go-libipfs v0.7.0/go.mod h1:KsIf/03CqhICzyRGyGo68tooiBE2iFbI/rXW7FhAYr0= 142 | github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= 143 | github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= 144 | github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= 145 | github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= 146 | github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= 147 | github.com/ipfs/go-merkledag v0.11.0 h1:DgzwK5hprESOzS4O1t/wi6JDpyVQdvm9Bs59N/jqfBY= 148 | github.com/ipfs/go-merkledag v0.11.0/go.mod h1:Q4f/1ezvBiJV0YCIXvt51W/9/kqJGH4I1LsA7+djsM4= 149 | github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= 150 | github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= 151 | github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= 152 | github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= 153 | github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0zs= 154 | github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw= 155 | github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= 156 | github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= 157 | github.com/ipld/go-car/v2 v2.13.1 h1:KnlrKvEPEzr5IZHKTXLAEub+tPrzeAFQVRlSQvuxBO4= 158 | github.com/ipld/go-car/v2 v2.13.1/go.mod h1:QkdjjFNGit2GIkpQ953KBwowuoukoM75nP/JI1iDJdo= 159 | github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= 160 | github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= 161 | github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= 162 | github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= 163 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 164 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 165 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= 166 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= 167 | github.com/jackc/pgx/v5 v5.5.0 h1:NxstgwndsTRy7eq9/kqYc/BZh5w2hHJV86wjvO+1xPw= 168 | github.com/jackc/pgx/v5 v5.5.0/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= 169 | github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= 170 | github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= 171 | github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= 172 | github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= 173 | github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= 174 | github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= 175 | github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= 176 | github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= 177 | github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= 178 | github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= 179 | github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= 180 | github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= 181 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 182 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 183 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 184 | github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= 185 | github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= 186 | github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= 187 | github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= 188 | github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= 189 | github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= 190 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 191 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 192 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 193 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 194 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 195 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 196 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 197 | github.com/labstack/echo/v4 v4.11.3 h1:Upyu3olaqSHkCjs1EJJwQ3WId8b8b1hxbogyommKktM= 198 | github.com/labstack/echo/v4 v4.11.3/go.mod h1:UcGuQ8V6ZNRmSweBIJkPvGfwCMIlFmiqrPqiEBfPYws= 199 | github.com/labstack/gommon v0.4.1 h1:gqEff0p/hTENGMABzezPoPSRtIh1Cvw0ueMOe0/dfOk= 200 | github.com/labstack/gommon v0.4.1/go.mod h1:TyTrpPqxR5KMk8LKVtLmfMjeQ5FEkBYdxLYPw/WfrOM= 201 | github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= 202 | github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= 203 | github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= 204 | github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= 205 | github.com/libp2p/go-libp2p v0.25.1 h1:YK+YDCHpYyTvitKWVxa5PfElgIpOONU01X5UcLEwJGA= 206 | github.com/libp2p/go-libp2p v0.25.1/go.mod h1:xnK9/1d9+jeQCVvi/f1g12KqtVi/jP/SijtKV1hML3g= 207 | github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= 208 | github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= 209 | github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= 210 | github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= 211 | github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= 212 | github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= 213 | github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= 214 | github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= 215 | github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= 216 | github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= 217 | github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= 218 | github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= 219 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 220 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 221 | github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 222 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 223 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 224 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 225 | github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= 226 | github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= 227 | github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= 228 | github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= 229 | github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= 230 | github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= 231 | github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= 232 | github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= 233 | github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= 234 | github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= 235 | github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= 236 | github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= 237 | github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= 238 | github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= 239 | github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= 240 | github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= 241 | github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= 242 | github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= 243 | github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= 244 | github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= 245 | github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= 246 | github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= 247 | github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= 248 | github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= 249 | github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= 250 | github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= 251 | github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= 252 | github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= 253 | github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= 254 | github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= 255 | github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= 256 | github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= 257 | github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= 258 | github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= 259 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 260 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 261 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 262 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 263 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 264 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 265 | github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f h1:VXTQfuJj9vKR4TCkEuWIckKvdHFeJH/huIFJ9/cXOB0= 266 | github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= 267 | github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= 268 | github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= 269 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 270 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 271 | github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8= 272 | github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= 273 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 274 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 275 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 276 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 277 | github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= 278 | github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= 279 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 280 | github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= 281 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 282 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 283 | github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= 284 | github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= 285 | github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= 286 | github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= 287 | github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= 288 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 289 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 290 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 291 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 292 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 293 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 294 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 295 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 296 | github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= 297 | github.com/urfave/cli/v2 v2.26.0 h1:3f3AMg3HpThFNT4I++TKOejZO8yU55t3JnnSr4S4QEI= 298 | github.com/urfave/cli/v2 v2.26.0/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= 299 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 300 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 301 | github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= 302 | github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= 303 | github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= 304 | github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= 305 | github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= 306 | github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= 307 | github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= 308 | github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= 309 | github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e h1:28X54ciEwwUxyHn9yrZfl5ojgF4CBNLWX7LR0rvBkf4= 310 | github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= 311 | github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI= 312 | github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= 313 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 314 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 315 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 316 | gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA= 317 | gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8= 318 | gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 h1:qwDnMxjkyLmAFgcfgTnfJrmYKWhHnci3GjDqcZp1M3Q= 319 | gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02/go.mod h1:JTnUj0mpYiAsuZLmKjTx/ex3AtMowcCgnE7YNyCEP0I= 320 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= 321 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= 322 | go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= 323 | go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= 324 | go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= 325 | go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= 326 | go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= 327 | go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= 328 | go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 329 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 330 | go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= 331 | go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= 332 | go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= 333 | go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= 334 | go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= 335 | go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= 336 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 337 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 338 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 339 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= 340 | go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= 341 | go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= 342 | go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= 343 | go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= 344 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 345 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 346 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 347 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 348 | golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= 349 | golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= 350 | golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= 351 | golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= 352 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 353 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 354 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 355 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 356 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 357 | golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= 358 | golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= 359 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 360 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 361 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 362 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 363 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 364 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 365 | golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= 366 | golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= 367 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 368 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 369 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 370 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 371 | golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= 372 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 373 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 374 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 375 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 376 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 377 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 378 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 379 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 380 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 381 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 382 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 383 | golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= 384 | golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 385 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 386 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 387 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 388 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 389 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 390 | golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= 391 | golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 392 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 393 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 394 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 395 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 396 | golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 397 | golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 398 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 399 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 400 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 401 | golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 402 | golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= 403 | golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= 404 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 405 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 406 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 407 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 408 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= 409 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= 410 | google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= 411 | google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= 412 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 413 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 414 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 415 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 416 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 417 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 418 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 419 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 420 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 421 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 422 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 423 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 424 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 425 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 426 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 427 | gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM= 428 | gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA= 429 | gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E= 430 | gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE= 431 | gorm.io/gorm v1.25.9 h1:wct0gxZIELDk8+ZqF/MVnHLkA1rvYlBWUMv2EdsK1g8= 432 | gorm.io/gorm v1.25.9/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= 433 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 434 | lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= 435 | lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= 436 | -------------------------------------------------------------------------------- /grafana-dashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_MIMIR", 5 | "label": "Mimir", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "prometheus", 9 | "pluginName": "Prometheus" 10 | } 11 | ], 12 | "__elements": {}, 13 | "__requires": [ 14 | { 15 | "type": "grafana", 16 | "id": "grafana", 17 | "name": "Grafana", 18 | "version": "10.4.1" 19 | }, 20 | { 21 | "type": "datasource", 22 | "id": "prometheus", 23 | "name": "Prometheus", 24 | "version": "1.0.0" 25 | }, 26 | { 27 | "type": "panel", 28 | "id": "timeseries", 29 | "name": "Time series", 30 | "version": "" 31 | } 32 | ], 33 | "annotations": { 34 | "list": [ 35 | { 36 | "builtIn": 1, 37 | "datasource": { 38 | "type": "grafana", 39 | "uid": "-- Grafana --" 40 | }, 41 | "enable": true, 42 | "hide": true, 43 | "iconColor": "rgba(0, 211, 255, 1)", 44 | "name": "Annotations & Alerts", 45 | "type": "dashboard" 46 | } 47 | ] 48 | }, 49 | "editable": true, 50 | "fiscalYearStartMonth": 0, 51 | "graphTooltip": 1, 52 | "id": null, 53 | "links": [], 54 | "liveNow": false, 55 | "panels": [ 56 | { 57 | "datasource": { 58 | "type": "prometheus", 59 | "uid": "${DS_MIMIR}" 60 | }, 61 | "fieldConfig": { 62 | "defaults": { 63 | "color": { 64 | "mode": "palette-classic" 65 | }, 66 | "custom": { 67 | "axisBorderShow": false, 68 | "axisCenteredZero": false, 69 | "axisColorMode": "text", 70 | "axisLabel": "", 71 | "axisPlacement": "auto", 72 | "barAlignment": 0, 73 | "drawStyle": "line", 74 | "fillOpacity": 75, 75 | "gradientMode": "none", 76 | "hideFrom": { 77 | "legend": false, 78 | "tooltip": false, 79 | "viz": false 80 | }, 81 | "insertNulls": false, 82 | "lineInterpolation": "linear", 83 | "lineWidth": 1, 84 | "pointSize": 5, 85 | "scaleDistribution": { 86 | "type": "linear" 87 | }, 88 | "showPoints": "auto", 89 | "spanNulls": false, 90 | "stacking": { 91 | "group": "A", 92 | "mode": "normal" 93 | }, 94 | "thresholdsStyle": { 95 | "mode": "off" 96 | } 97 | }, 98 | "mappings": [], 99 | "thresholds": { 100 | "mode": "absolute", 101 | "steps": [ 102 | { 103 | "color": "green", 104 | "value": null 105 | }, 106 | { 107 | "color": "red", 108 | "value": 80 109 | } 110 | ] 111 | }, 112 | "unit": "cps" 113 | }, 114 | "overrides": [] 115 | }, 116 | "gridPos": { 117 | "h": 12, 118 | "w": 12, 119 | "x": 0, 120 | "y": 0 121 | }, 122 | "id": 1, 123 | "options": { 124 | "legend": { 125 | "calcs": [ 126 | "mean" 127 | ], 128 | "displayMode": "table", 129 | "placement": "right", 130 | "showLegend": true, 131 | "sortBy": "Mean", 132 | "sortDesc": true 133 | }, 134 | "tooltip": { 135 | "mode": "single", 136 | "sort": "none" 137 | } 138 | }, 139 | "targets": [ 140 | { 141 | "datasource": { 142 | "type": "prometheus", 143 | "uid": "${DS_MIMIR}" 144 | }, 145 | "editorMode": "code", 146 | "expr": "rate(consumer_ops_processed_total{job=~\"${job}\"}[$__rate_interval]) > 0", 147 | "instant": false, 148 | "legendFormat": "{{kind}} {{op_path}}", 149 | "range": true, 150 | "refId": "A" 151 | } 152 | ], 153 | "title": "Consumer Operation Throughput", 154 | "type": "timeseries" 155 | }, 156 | { 157 | "datasource": { 158 | "type": "prometheus", 159 | "uid": "${DS_MIMIR}" 160 | }, 161 | "fieldConfig": { 162 | "defaults": { 163 | "color": { 164 | "mode": "palette-classic" 165 | }, 166 | "custom": { 167 | "axisBorderShow": false, 168 | "axisCenteredZero": false, 169 | "axisColorMode": "series", 170 | "axisLabel": "", 171 | "axisPlacement": "auto", 172 | "barAlignment": 0, 173 | "drawStyle": "line", 174 | "fillOpacity": 0, 175 | "gradientMode": "none", 176 | "hideFrom": { 177 | "legend": false, 178 | "tooltip": false, 179 | "viz": false 180 | }, 181 | "insertNulls": false, 182 | "lineInterpolation": "linear", 183 | "lineWidth": 1, 184 | "pointSize": 5, 185 | "scaleDistribution": { 186 | "type": "linear" 187 | }, 188 | "showPoints": "auto", 189 | "spanNulls": false, 190 | "stacking": { 191 | "group": "A", 192 | "mode": "none" 193 | }, 194 | "thresholdsStyle": { 195 | "mode": "off" 196 | } 197 | }, 198 | "mappings": [], 199 | "thresholds": { 200 | "mode": "absolute", 201 | "steps": [ 202 | { 203 | "color": "green", 204 | "value": null 205 | }, 206 | { 207 | "color": "red", 208 | "value": 80 209 | } 210 | ] 211 | }, 212 | "unit": "cps" 213 | }, 214 | "overrides": [ 215 | { 216 | "matcher": { 217 | "id": "byFrameRefID", 218 | "options": "B" 219 | }, 220 | "properties": [ 221 | { 222 | "id": "custom.axisPlacement", 223 | "value": "right" 224 | }, 225 | { 226 | "id": "unit", 227 | "value": "bps" 228 | } 229 | ] 230 | } 231 | ] 232 | }, 233 | "gridPos": { 234 | "h": 12, 235 | "w": 12, 236 | "x": 12, 237 | "y": 0 238 | }, 239 | "id": 3, 240 | "options": { 241 | "legend": { 242 | "calcs": [ 243 | "mean" 244 | ], 245 | "displayMode": "table", 246 | "placement": "right", 247 | "showLegend": true, 248 | "sortBy": "Name", 249 | "sortDesc": true 250 | }, 251 | "tooltip": { 252 | "mode": "multi", 253 | "sort": "none" 254 | } 255 | }, 256 | "targets": [ 257 | { 258 | "datasource": { 259 | "type": "prometheus", 260 | "uid": "${DS_MIMIR}" 261 | }, 262 | "editorMode": "code", 263 | "expr": "rate(jetstream_events_delivered_total{job=~\"${job}\"}[$__rate_interval])> 0", 264 | "instant": false, 265 | "legendFormat": "{{ip_address}}", 266 | "range": true, 267 | "refId": "A" 268 | }, 269 | { 270 | "datasource": { 271 | "type": "prometheus", 272 | "uid": "${DS_MIMIR}" 273 | }, 274 | "editorMode": "code", 275 | "expr": "rate(jetstream_bytes_delivered_total{job=~\"${job}\"}[$__rate_interval]) * 8 > 0", 276 | "hide": false, 277 | "instant": false, 278 | "legendFormat": "{{ip_address}}", 279 | "range": true, 280 | "refId": "B" 281 | } 282 | ], 283 | "title": "Subscriber Throughput", 284 | "type": "timeseries" 285 | }, 286 | { 287 | "datasource": { 288 | "type": "prometheus", 289 | "uid": "${DS_MIMIR}" 290 | }, 291 | "fieldConfig": { 292 | "defaults": { 293 | "color": { 294 | "mode": "palette-classic" 295 | }, 296 | "custom": { 297 | "axisBorderShow": false, 298 | "axisCenteredZero": false, 299 | "axisColorMode": "text", 300 | "axisLabel": "", 301 | "axisPlacement": "auto", 302 | "barAlignment": 0, 303 | "drawStyle": "line", 304 | "fillOpacity": 0, 305 | "gradientMode": "none", 306 | "hideFrom": { 307 | "legend": false, 308 | "tooltip": false, 309 | "viz": false 310 | }, 311 | "insertNulls": false, 312 | "lineInterpolation": "linear", 313 | "lineWidth": 1, 314 | "pointSize": 5, 315 | "scaleDistribution": { 316 | "type": "linear" 317 | }, 318 | "showPoints": "auto", 319 | "spanNulls": false, 320 | "stacking": { 321 | "group": "A", 322 | "mode": "none" 323 | }, 324 | "thresholdsStyle": { 325 | "mode": "off" 326 | } 327 | }, 328 | "mappings": [], 329 | "thresholds": { 330 | "mode": "absolute", 331 | "steps": [ 332 | { 333 | "color": "green", 334 | "value": null 335 | }, 336 | { 337 | "color": "red", 338 | "value": 80 339 | } 340 | ] 341 | }, 342 | "unit": "bps" 343 | }, 344 | "overrides": [] 345 | }, 346 | "gridPos": { 347 | "h": 5, 348 | "w": 12, 349 | "x": 0, 350 | "y": 12 351 | }, 352 | "id": 6, 353 | "options": { 354 | "legend": { 355 | "calcs": [ 356 | "mean" 357 | ], 358 | "displayMode": "list", 359 | "placement": "bottom", 360 | "showLegend": true, 361 | "sortBy": "Mean", 362 | "sortDesc": true 363 | }, 364 | "tooltip": { 365 | "mode": "multi", 366 | "sort": "none" 367 | } 368 | }, 369 | "targets": [ 370 | { 371 | "datasource": { 372 | "type": "prometheus", 373 | "uid": "${DS_MIMIR}" 374 | }, 375 | "editorMode": "code", 376 | "expr": "rate(indigo_repo_stream_bytes_total{job=~\"${job}\"}[$__rate_interval]) * 8 > 0", 377 | "instant": false, 378 | "legendFormat": "Bytes In", 379 | "range": true, 380 | "refId": "A" 381 | } 382 | ], 383 | "title": "Consumer Firehose Throughput", 384 | "type": "timeseries" 385 | }, 386 | { 387 | "datasource": { 388 | "type": "prometheus", 389 | "uid": "${DS_MIMIR}" 390 | }, 391 | "fieldConfig": { 392 | "defaults": { 393 | "color": { 394 | "mode": "palette-classic" 395 | }, 396 | "custom": { 397 | "axisBorderShow": false, 398 | "axisCenteredZero": false, 399 | "axisColorMode": "text", 400 | "axisLabel": "", 401 | "axisPlacement": "auto", 402 | "barAlignment": 0, 403 | "drawStyle": "line", 404 | "fillOpacity": 0, 405 | "gradientMode": "none", 406 | "hideFrom": { 407 | "legend": false, 408 | "tooltip": false, 409 | "viz": false 410 | }, 411 | "insertNulls": false, 412 | "lineInterpolation": "linear", 413 | "lineWidth": 1, 414 | "pointSize": 5, 415 | "scaleDistribution": { 416 | "type": "linear" 417 | }, 418 | "showPoints": "auto", 419 | "spanNulls": false, 420 | "stacking": { 421 | "group": "A", 422 | "mode": "none" 423 | }, 424 | "thresholdsStyle": { 425 | "mode": "off" 426 | } 427 | }, 428 | "mappings": [], 429 | "thresholds": { 430 | "mode": "absolute", 431 | "steps": [ 432 | { 433 | "color": "green", 434 | "value": null 435 | }, 436 | { 437 | "color": "red", 438 | "value": 80 439 | } 440 | ] 441 | }, 442 | "unit": "s" 443 | }, 444 | "overrides": [] 445 | }, 446 | "gridPos": { 447 | "h": 8, 448 | "w": 5, 449 | "x": 12, 450 | "y": 12 451 | }, 452 | "id": 4, 453 | "options": { 454 | "legend": { 455 | "calcs": [], 456 | "displayMode": "list", 457 | "placement": "bottom", 458 | "showLegend": true 459 | }, 460 | "tooltip": { 461 | "mode": "single", 462 | "sort": "none" 463 | } 464 | }, 465 | "targets": [ 466 | { 467 | "datasource": { 468 | "type": "prometheus", 469 | "uid": "${DS_MIMIR}" 470 | }, 471 | "editorMode": "code", 472 | "expr": "rate(process_cpu_seconds_total{job=~\"${job}\"}[$__rate_interval])", 473 | "legendFormat": "CPU Seconds per Second", 474 | "range": true, 475 | "refId": "A" 476 | } 477 | ], 478 | "title": "CPU Utilization", 479 | "type": "timeseries" 480 | }, 481 | { 482 | "datasource": { 483 | "type": "prometheus", 484 | "uid": "${DS_MIMIR}" 485 | }, 486 | "fieldConfig": { 487 | "defaults": { 488 | "color": { 489 | "mode": "palette-classic" 490 | }, 491 | "custom": { 492 | "axisBorderShow": false, 493 | "axisCenteredZero": false, 494 | "axisColorMode": "text", 495 | "axisLabel": "", 496 | "axisPlacement": "auto", 497 | "barAlignment": 0, 498 | "drawStyle": "line", 499 | "fillOpacity": 0, 500 | "gradientMode": "none", 501 | "hideFrom": { 502 | "legend": false, 503 | "tooltip": false, 504 | "viz": false 505 | }, 506 | "insertNulls": false, 507 | "lineInterpolation": "linear", 508 | "lineWidth": 1, 509 | "pointSize": 5, 510 | "scaleDistribution": { 511 | "type": "linear" 512 | }, 513 | "showPoints": "auto", 514 | "spanNulls": false, 515 | "stacking": { 516 | "group": "A", 517 | "mode": "none" 518 | }, 519 | "thresholdsStyle": { 520 | "mode": "off" 521 | } 522 | }, 523 | "mappings": [], 524 | "thresholds": { 525 | "mode": "absolute", 526 | "steps": [ 527 | { 528 | "color": "green", 529 | "value": null 530 | }, 531 | { 532 | "color": "red", 533 | "value": 80 534 | } 535 | ] 536 | }, 537 | "unit": "bytes" 538 | }, 539 | "overrides": [] 540 | }, 541 | "gridPos": { 542 | "h": 8, 543 | "w": 5, 544 | "x": 17, 545 | "y": 12 546 | }, 547 | "id": 5, 548 | "options": { 549 | "legend": { 550 | "calcs": [], 551 | "displayMode": "list", 552 | "placement": "bottom", 553 | "showLegend": true 554 | }, 555 | "tooltip": { 556 | "mode": "single", 557 | "sort": "none" 558 | } 559 | }, 560 | "targets": [ 561 | { 562 | "datasource": { 563 | "type": "prometheus", 564 | "uid": "${DS_MIMIR}" 565 | }, 566 | "editorMode": "code", 567 | "expr": "process_resident_memory_bytes{job=~\"${job}\"}", 568 | "legendFormat": "Memory Usage", 569 | "range": true, 570 | "refId": "A" 571 | } 572 | ], 573 | "title": "Memory Utilization", 574 | "type": "timeseries" 575 | }, 576 | { 577 | "datasource": { 578 | "type": "prometheus", 579 | "uid": "${DS_MIMIR}" 580 | }, 581 | "fieldConfig": { 582 | "defaults": { 583 | "color": { 584 | "mode": "palette-classic" 585 | }, 586 | "custom": { 587 | "axisBorderShow": false, 588 | "axisCenteredZero": false, 589 | "axisColorMode": "text", 590 | "axisLabel": "", 591 | "axisPlacement": "auto", 592 | "barAlignment": 0, 593 | "drawStyle": "line", 594 | "fillOpacity": 0, 595 | "gradientMode": "none", 596 | "hideFrom": { 597 | "legend": false, 598 | "tooltip": false, 599 | "viz": false 600 | }, 601 | "insertNulls": false, 602 | "lineInterpolation": "linear", 603 | "lineWidth": 1, 604 | "pointSize": 5, 605 | "scaleDistribution": { 606 | "type": "linear" 607 | }, 608 | "showPoints": "auto", 609 | "spanNulls": false, 610 | "stacking": { 611 | "group": "A", 612 | "mode": "none" 613 | }, 614 | "thresholdsStyle": { 615 | "mode": "off" 616 | } 617 | }, 618 | "mappings": [], 619 | "thresholds": { 620 | "mode": "absolute", 621 | "steps": [ 622 | { 623 | "color": "green", 624 | "value": null 625 | }, 626 | { 627 | "color": "red", 628 | "value": 80 629 | } 630 | ] 631 | }, 632 | "unit": "s" 633 | }, 634 | "overrides": [] 635 | }, 636 | "gridPos": { 637 | "h": 8, 638 | "w": 12, 639 | "x": 0, 640 | "y": 17 641 | }, 642 | "id": 2, 643 | "options": { 644 | "legend": { 645 | "calcs": [], 646 | "displayMode": "list", 647 | "placement": "bottom", 648 | "showLegend": true 649 | }, 650 | "tooltip": { 651 | "mode": "single", 652 | "sort": "none" 653 | } 654 | }, 655 | "targets": [ 656 | { 657 | "datasource": { 658 | "type": "prometheus", 659 | "uid": "${DS_MIMIR}" 660 | }, 661 | "editorMode": "code", 662 | "expr": "histogram_quantile(0.5, sum(rate(consumer_event_processing_duration_seconds_bucket{job=~\"${job}\"}[$__rate_interval])) by (le)) > 0", 663 | "hide": false, 664 | "instant": false, 665 | "legendFormat": "P50", 666 | "range": true, 667 | "refId": "B" 668 | }, 669 | { 670 | "datasource": { 671 | "type": "prometheus", 672 | "uid": "${DS_MIMIR}" 673 | }, 674 | "editorMode": "code", 675 | "expr": "histogram_quantile(0.95, sum(rate(consumer_event_processing_duration_seconds_bucket{job=~\"${job}\"}[$__rate_interval])) by (le)) > 0", 676 | "hide": false, 677 | "instant": false, 678 | "legendFormat": "P95", 679 | "range": true, 680 | "refId": "C" 681 | }, 682 | { 683 | "datasource": { 684 | "type": "prometheus", 685 | "uid": "${DS_MIMIR}" 686 | }, 687 | "editorMode": "code", 688 | "expr": "histogram_quantile(0.99, sum(rate(consumer_event_processing_duration_seconds_bucket{job=~\"${job}\"}[$__rate_interval])) by (le)) > 0", 689 | "instant": false, 690 | "legendFormat": "P99", 691 | "range": true, 692 | "refId": "A" 693 | } 694 | ], 695 | "title": "Consumer Event Processing Time", 696 | "type": "timeseries" 697 | }, 698 | { 699 | "datasource": { 700 | "type": "prometheus", 701 | "uid": "${DS_MIMIR}" 702 | }, 703 | "description": "", 704 | "fieldConfig": { 705 | "defaults": { 706 | "color": { 707 | "mode": "palette-classic" 708 | }, 709 | "custom": { 710 | "axisBorderShow": false, 711 | "axisCenteredZero": false, 712 | "axisColorMode": "text", 713 | "axisLabel": "bytes", 714 | "axisPlacement": "auto", 715 | "axisSoftMax": 55999999999, 716 | "barAlignment": 0, 717 | "drawStyle": "line", 718 | "fillOpacity": 40, 719 | "gradientMode": "none", 720 | "hideFrom": { 721 | "legend": false, 722 | "tooltip": false, 723 | "viz": false 724 | }, 725 | "insertNulls": false, 726 | "lineInterpolation": "linear", 727 | "lineWidth": 1, 728 | "pointSize": 5, 729 | "scaleDistribution": { 730 | "type": "linear" 731 | }, 732 | "showPoints": "never", 733 | "spanNulls": false, 734 | "stacking": { 735 | "group": "A", 736 | "mode": "none" 737 | }, 738 | "thresholdsStyle": { 739 | "mode": "dashed" 740 | } 741 | }, 742 | "links": [], 743 | "mappings": [], 744 | "min": 0, 745 | "thresholds": { 746 | "mode": "absolute", 747 | "steps": [ 748 | { 749 | "color": "green", 750 | "value": null 751 | }, 752 | { 753 | "color": "red", 754 | "value": 43000000000 755 | } 756 | ] 757 | }, 758 | "unit": "bytes" 759 | }, 760 | "overrides": [] 761 | }, 762 | "gridPos": { 763 | "h": 5, 764 | "w": 5, 765 | "x": 12, 766 | "y": 20 767 | }, 768 | "id": 7, 769 | "options": { 770 | "legend": { 771 | "calcs": [ 772 | "mean", 773 | "lastNotNull", 774 | "max", 775 | "min" 776 | ], 777 | "displayMode": "table", 778 | "placement": "bottom", 779 | "showLegend": true 780 | }, 781 | "tooltip": { 782 | "mode": "multi", 783 | "sort": "none" 784 | } 785 | }, 786 | "pluginVersion": "9.2.0", 787 | "targets": [ 788 | { 789 | "datasource": { 790 | "type": "prometheus", 791 | "uid": "${DS_MIMIR}" 792 | }, 793 | "editorMode": "code", 794 | "expr": "node_filesystem_size_bytes{job=\"$exporter_job\",device!~'rootfs', mountpoint=\"/\"} - node_filesystem_avail_bytes{job=\"$exporter_job\", device!~'rootfs', mountpoint=\"/\"}", 795 | "format": "time_series", 796 | "intervalFactor": 1, 797 | "legendFormat": "{{mountpoint}}", 798 | "range": true, 799 | "refId": "A", 800 | "step": 240 801 | } 802 | ], 803 | "title": "Disk Space Used", 804 | "type": "timeseries" 805 | }, 806 | { 807 | "datasource": { 808 | "type": "prometheus", 809 | "uid": "${DS_MIMIR}" 810 | }, 811 | "description": "Basic network info per interface", 812 | "fieldConfig": { 813 | "defaults": { 814 | "color": { 815 | "mode": "palette-classic" 816 | }, 817 | "custom": { 818 | "axisBorderShow": false, 819 | "axisCenteredZero": false, 820 | "axisColorMode": "text", 821 | "axisLabel": "", 822 | "axisPlacement": "auto", 823 | "barAlignment": 0, 824 | "drawStyle": "line", 825 | "fillOpacity": 40, 826 | "gradientMode": "none", 827 | "hideFrom": { 828 | "legend": false, 829 | "tooltip": false, 830 | "viz": false 831 | }, 832 | "insertNulls": false, 833 | "lineInterpolation": "linear", 834 | "lineWidth": 1, 835 | "pointSize": 5, 836 | "scaleDistribution": { 837 | "type": "linear" 838 | }, 839 | "showPoints": "never", 840 | "spanNulls": false, 841 | "stacking": { 842 | "group": "A", 843 | "mode": "none" 844 | }, 845 | "thresholdsStyle": { 846 | "mode": "off" 847 | } 848 | }, 849 | "links": [], 850 | "mappings": [], 851 | "thresholds": { 852 | "mode": "absolute", 853 | "steps": [ 854 | { 855 | "color": "green", 856 | "value": null 857 | }, 858 | { 859 | "color": "red", 860 | "value": 80 861 | } 862 | ] 863 | }, 864 | "unit": "bps" 865 | }, 866 | "overrides": [ 867 | { 868 | "matcher": { 869 | "id": "byName", 870 | "options": "Recv_bytes_eth2" 871 | }, 872 | "properties": [ 873 | { 874 | "id": "color", 875 | "value": { 876 | "fixedColor": "#7EB26D", 877 | "mode": "fixed" 878 | } 879 | } 880 | ] 881 | }, 882 | { 883 | "matcher": { 884 | "id": "byName", 885 | "options": "Recv_bytes_lo" 886 | }, 887 | "properties": [ 888 | { 889 | "id": "color", 890 | "value": { 891 | "fixedColor": "#0A50A1", 892 | "mode": "fixed" 893 | } 894 | } 895 | ] 896 | }, 897 | { 898 | "matcher": { 899 | "id": "byName", 900 | "options": "Recv_drop_eth2" 901 | }, 902 | "properties": [ 903 | { 904 | "id": "color", 905 | "value": { 906 | "fixedColor": "#6ED0E0", 907 | "mode": "fixed" 908 | } 909 | } 910 | ] 911 | }, 912 | { 913 | "matcher": { 914 | "id": "byName", 915 | "options": "Recv_drop_lo" 916 | }, 917 | "properties": [ 918 | { 919 | "id": "color", 920 | "value": { 921 | "fixedColor": "#E0F9D7", 922 | "mode": "fixed" 923 | } 924 | } 925 | ] 926 | }, 927 | { 928 | "matcher": { 929 | "id": "byName", 930 | "options": "Recv_errs_eth2" 931 | }, 932 | "properties": [ 933 | { 934 | "id": "color", 935 | "value": { 936 | "fixedColor": "#BF1B00", 937 | "mode": "fixed" 938 | } 939 | } 940 | ] 941 | }, 942 | { 943 | "matcher": { 944 | "id": "byName", 945 | "options": "Recv_errs_lo" 946 | }, 947 | "properties": [ 948 | { 949 | "id": "color", 950 | "value": { 951 | "fixedColor": "#CCA300", 952 | "mode": "fixed" 953 | } 954 | } 955 | ] 956 | }, 957 | { 958 | "matcher": { 959 | "id": "byName", 960 | "options": "Trans_bytes_eth2" 961 | }, 962 | "properties": [ 963 | { 964 | "id": "color", 965 | "value": { 966 | "fixedColor": "#7EB26D", 967 | "mode": "fixed" 968 | } 969 | } 970 | ] 971 | }, 972 | { 973 | "matcher": { 974 | "id": "byName", 975 | "options": "Trans_bytes_lo" 976 | }, 977 | "properties": [ 978 | { 979 | "id": "color", 980 | "value": { 981 | "fixedColor": "#0A50A1", 982 | "mode": "fixed" 983 | } 984 | } 985 | ] 986 | }, 987 | { 988 | "matcher": { 989 | "id": "byName", 990 | "options": "Trans_drop_eth2" 991 | }, 992 | "properties": [ 993 | { 994 | "id": "color", 995 | "value": { 996 | "fixedColor": "#6ED0E0", 997 | "mode": "fixed" 998 | } 999 | } 1000 | ] 1001 | }, 1002 | { 1003 | "matcher": { 1004 | "id": "byName", 1005 | "options": "Trans_drop_lo" 1006 | }, 1007 | "properties": [ 1008 | { 1009 | "id": "color", 1010 | "value": { 1011 | "fixedColor": "#E0F9D7", 1012 | "mode": "fixed" 1013 | } 1014 | } 1015 | ] 1016 | }, 1017 | { 1018 | "matcher": { 1019 | "id": "byName", 1020 | "options": "Trans_errs_eth2" 1021 | }, 1022 | "properties": [ 1023 | { 1024 | "id": "color", 1025 | "value": { 1026 | "fixedColor": "#BF1B00", 1027 | "mode": "fixed" 1028 | } 1029 | } 1030 | ] 1031 | }, 1032 | { 1033 | "matcher": { 1034 | "id": "byName", 1035 | "options": "Trans_errs_lo" 1036 | }, 1037 | "properties": [ 1038 | { 1039 | "id": "color", 1040 | "value": { 1041 | "fixedColor": "#CCA300", 1042 | "mode": "fixed" 1043 | } 1044 | } 1045 | ] 1046 | }, 1047 | { 1048 | "matcher": { 1049 | "id": "byName", 1050 | "options": "recv_bytes_lo" 1051 | }, 1052 | "properties": [ 1053 | { 1054 | "id": "color", 1055 | "value": { 1056 | "fixedColor": "#0A50A1", 1057 | "mode": "fixed" 1058 | } 1059 | } 1060 | ] 1061 | }, 1062 | { 1063 | "matcher": { 1064 | "id": "byName", 1065 | "options": "recv_drop_eth0" 1066 | }, 1067 | "properties": [ 1068 | { 1069 | "id": "color", 1070 | "value": { 1071 | "fixedColor": "#99440A", 1072 | "mode": "fixed" 1073 | } 1074 | } 1075 | ] 1076 | }, 1077 | { 1078 | "matcher": { 1079 | "id": "byName", 1080 | "options": "recv_drop_lo" 1081 | }, 1082 | "properties": [ 1083 | { 1084 | "id": "color", 1085 | "value": { 1086 | "fixedColor": "#967302", 1087 | "mode": "fixed" 1088 | } 1089 | } 1090 | ] 1091 | }, 1092 | { 1093 | "matcher": { 1094 | "id": "byName", 1095 | "options": "recv_errs_eth0" 1096 | }, 1097 | "properties": [ 1098 | { 1099 | "id": "color", 1100 | "value": { 1101 | "fixedColor": "#BF1B00", 1102 | "mode": "fixed" 1103 | } 1104 | } 1105 | ] 1106 | }, 1107 | { 1108 | "matcher": { 1109 | "id": "byName", 1110 | "options": "recv_errs_lo" 1111 | }, 1112 | "properties": [ 1113 | { 1114 | "id": "color", 1115 | "value": { 1116 | "fixedColor": "#890F02", 1117 | "mode": "fixed" 1118 | } 1119 | } 1120 | ] 1121 | }, 1122 | { 1123 | "matcher": { 1124 | "id": "byName", 1125 | "options": "trans_bytes_eth0" 1126 | }, 1127 | "properties": [ 1128 | { 1129 | "id": "color", 1130 | "value": { 1131 | "fixedColor": "#7EB26D", 1132 | "mode": "fixed" 1133 | } 1134 | } 1135 | ] 1136 | }, 1137 | { 1138 | "matcher": { 1139 | "id": "byName", 1140 | "options": "trans_bytes_lo" 1141 | }, 1142 | "properties": [ 1143 | { 1144 | "id": "color", 1145 | "value": { 1146 | "fixedColor": "#0A50A1", 1147 | "mode": "fixed" 1148 | } 1149 | } 1150 | ] 1151 | }, 1152 | { 1153 | "matcher": { 1154 | "id": "byName", 1155 | "options": "trans_drop_eth0" 1156 | }, 1157 | "properties": [ 1158 | { 1159 | "id": "color", 1160 | "value": { 1161 | "fixedColor": "#99440A", 1162 | "mode": "fixed" 1163 | } 1164 | } 1165 | ] 1166 | }, 1167 | { 1168 | "matcher": { 1169 | "id": "byName", 1170 | "options": "trans_drop_lo" 1171 | }, 1172 | "properties": [ 1173 | { 1174 | "id": "color", 1175 | "value": { 1176 | "fixedColor": "#967302", 1177 | "mode": "fixed" 1178 | } 1179 | } 1180 | ] 1181 | }, 1182 | { 1183 | "matcher": { 1184 | "id": "byName", 1185 | "options": "trans_errs_eth0" 1186 | }, 1187 | "properties": [ 1188 | { 1189 | "id": "color", 1190 | "value": { 1191 | "fixedColor": "#BF1B00", 1192 | "mode": "fixed" 1193 | } 1194 | } 1195 | ] 1196 | }, 1197 | { 1198 | "matcher": { 1199 | "id": "byName", 1200 | "options": "trans_errs_lo" 1201 | }, 1202 | "properties": [ 1203 | { 1204 | "id": "color", 1205 | "value": { 1206 | "fixedColor": "#890F02", 1207 | "mode": "fixed" 1208 | } 1209 | } 1210 | ] 1211 | }, 1212 | { 1213 | "matcher": { 1214 | "id": "byRegexp", 1215 | "options": "/.*trans.*/" 1216 | }, 1217 | "properties": [ 1218 | { 1219 | "id": "custom.transform", 1220 | "value": "negative-Y" 1221 | } 1222 | ] 1223 | } 1224 | ] 1225 | }, 1226 | "gridPos": { 1227 | "h": 5, 1228 | "w": 5, 1229 | "x": 17, 1230 | "y": 20 1231 | }, 1232 | "id": 8, 1233 | "options": { 1234 | "legend": { 1235 | "calcs": [], 1236 | "displayMode": "list", 1237 | "placement": "bottom", 1238 | "showLegend": true 1239 | }, 1240 | "tooltip": { 1241 | "mode": "multi", 1242 | "sort": "none" 1243 | } 1244 | }, 1245 | "pluginVersion": "9.2.0", 1246 | "targets": [ 1247 | { 1248 | "datasource": { 1249 | "type": "prometheus", 1250 | "uid": "${DS_MIMIR}" 1251 | }, 1252 | "editorMode": "code", 1253 | "expr": "irate(node_network_receive_bytes_total{job=\"$exporter_job\", device=\"ens3\"}[$__rate_interval])*8", 1254 | "format": "time_series", 1255 | "intervalFactor": 1, 1256 | "legendFormat": "recv {{device}}", 1257 | "range": true, 1258 | "refId": "A", 1259 | "step": 240 1260 | }, 1261 | { 1262 | "datasource": { 1263 | "type": "prometheus", 1264 | "uid": "${DS_MIMIR}" 1265 | }, 1266 | "editorMode": "code", 1267 | "expr": "irate(node_network_transmit_bytes_total{job=\"$exporter_job\", device=\"ens3\"}[$__rate_interval])*8", 1268 | "format": "time_series", 1269 | "intervalFactor": 1, 1270 | "legendFormat": "trans {{device}} ", 1271 | "range": true, 1272 | "refId": "B", 1273 | "step": 240 1274 | } 1275 | ], 1276 | "title": "Network Traffic", 1277 | "type": "timeseries" 1278 | } 1279 | ], 1280 | "refresh": "5s", 1281 | "schemaVersion": 39, 1282 | "tags": [], 1283 | "templating": { 1284 | "list": [ 1285 | { 1286 | "current": {}, 1287 | "datasource": { 1288 | "type": "prometheus", 1289 | "uid": "${DS_MIMIR}" 1290 | }, 1291 | "definition": "label_values(jetstream_events_emitted_total,job)", 1292 | "hide": 0, 1293 | "includeAll": false, 1294 | "label": "Job", 1295 | "multi": false, 1296 | "name": "job", 1297 | "options": [], 1298 | "query": { 1299 | "query": "label_values(jetstream_events_emitted_total,job)", 1300 | "refId": "PrometheusVariableQueryEditor-VariableQuery" 1301 | }, 1302 | "refresh": 1, 1303 | "regex": "", 1304 | "skipUrlSync": false, 1305 | "sort": 0, 1306 | "type": "query" 1307 | }, 1308 | { 1309 | "current": {}, 1310 | "datasource": { 1311 | "type": "prometheus", 1312 | "uid": "${DS_MIMIR}" 1313 | }, 1314 | "definition": "label_values(node_network_receive_bytes_total,job)", 1315 | "description": "Node-Exporter Job Name for Jetstream Node Exporter", 1316 | "hide": 0, 1317 | "includeAll": false, 1318 | "label": "Exporter Job", 1319 | "multi": false, 1320 | "name": "exporter_job", 1321 | "options": [], 1322 | "query": { 1323 | "qryType": 1, 1324 | "query": "label_values(node_network_receive_bytes_total,job)", 1325 | "refId": "PrometheusVariableQueryEditor-VariableQuery" 1326 | }, 1327 | "refresh": 1, 1328 | "regex": "", 1329 | "skipUrlSync": false, 1330 | "sort": 0, 1331 | "type": "query" 1332 | } 1333 | ] 1334 | }, 1335 | "time": { 1336 | "from": "now-5m", 1337 | "to": "now" 1338 | }, 1339 | "timepicker": {}, 1340 | "timezone": "", 1341 | "title": "Jetstream", 1342 | "uid": "b4cc9c77-e3ce-454b-a296-26921ee7aef3", 1343 | "version": 33, 1344 | "weekStart": "" 1345 | } 1346 | --------------------------------------------------------------------------------