├── .env.template
├── .gitignore
├── Dockerfile
├── LICENSE
├── Makefile
├── README.md
├── api
    ├── Makefile
    ├── cmd
    │   └── main.go
    ├── flags
    │   ├── config.go
    │   └── flags.go
    ├── metrics
    │   └── metrics.go
    └── service
    │   ├── api.go
    │   ├── api_test.go
    │   └── service.go
├── archiver
    ├── Makefile
    ├── cmd
    │   └── main.go
    ├── flags
    │   ├── config.go
    │   └── flags.go
    ├── metrics
    │   └── metrics.go
    └── service
    │   ├── api.go
    │   ├── api_test.go
    │   ├── archiver.go
    │   ├── archiver_test.go
    │   └── service.go
├── common
    ├── beacon
    │   ├── beacontest
    │   │   └── stub.go
    │   └── client.go
    ├── blobtest
    │   └── helpers.go
    ├── flags
    │   ├── config.go
    │   └── flags.go
    └── storage
    │   ├── file.go
    │   ├── file_test.go
    │   ├── s3.go
    │   ├── s3_test.go
    │   ├── storage.go
    │   ├── storage_test.go
    │   └── storagetest
    │       └── stub.go
├── docker-compose.yml
├── go.mod
├── go.sum
└── validator
    ├── Makefile
    ├── cmd
        └── main.go
    ├── flags
        ├── config.go
        └── flags.go
    └── service
        ├── client.go
        ├── service.go
        └── service_test.go


/.env.template:
--------------------------------------------------------------------------------
 1 | # To get started, copy this file to .env and set your beacon http endpoint
 2 | 
 3 | BLOB_ARCHIVER_L1_BEACON_HTTP=<unset>
 4 | BLOB_ARCHIVER_DATA_STORE=s3
 5 | BLOB_ARCHIVER_S3_ENDPOINT=172.17.0.1:9000
 6 | BLOB_ARCHIVER_S3_CREDENTIAL_TYPE=static
 7 | BLOB_ARCHIVER_S3_ACCESS_KEY=admin
 8 | BLOB_ARCHIVER_S3_SECRET_ACCESS_KEY=password
 9 | BLOB_ARCHIVER_S3_ENDPOINT_HTTPS=false
10 | BLOB_ARCHIVER_S3_BUCKET=blobs
11 | BLOB_ARCHIVER_METRICS_ENABLED=true
12 | BLOB_ARCHIVER_METRICS_PORT=7300
13 | BLOB_ARCHIVER_ORIGIN_BLOCK=0x0
14 | 
15 | BLOB_API_L1_BEACON_HTTP=<unset>
16 | BLOB_API_DATA_STORE=s3
17 | BLOB_API_S3_ENDPOINT=172.17.0.1:9000
18 | BLOB_API_S3_CREDENTIAL_TYPE=static
19 | BLOB_API_S3_ACCESS_KEY=admin
20 | BLOB_API_S3_SECRET_ACCESS_KEY=password
21 | BLOB_API_S3_ENDPOINT_HTTPS=false
22 | BLOB_API_S3_BUCKET=blobs
23 | BLOB_API_METRICS_ENABLED=true
24 | BLOB_API_METRICS_PORT=7301
25 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .idea/
2 | .DS_Store
3 | .swp
4 | .env
5 | api/bin
6 | archiver/bin
7 | validator/bin
8 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM golang:1.21.6-alpine3.19 as builder
 2 | 
 3 | RUN apk add --no-cache make gcc musl-dev linux-headers jq bash
 4 | 
 5 | WORKDIR /app
 6 | 
 7 | COPY ./go.mod ./go.sum /app/
 8 | 
 9 | RUN go mod download
10 | 
11 | COPY . /app
12 | 
13 | RUN make build
14 | 
15 | FROM alpine:3.19
16 | 
17 | COPY --from=builder /app/archiver/bin/blob-archiver /usr/local/bin/blob-archiver
18 | COPY --from=builder /app/api/bin/blob-api /usr/local/bin/blob-api
19 | COPY --from=builder /app/validator/bin/blob-validator /usr/local/bin/blob-validator
20 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2023-2024 base.org contributors
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | build:
 2 | 	make -C ./archiver blob-archiver
 3 | 	make -C ./api blob-api
 4 | 	make -C ./validator blob-validator
 5 | .PHONY: build
 6 | 
 7 | build-docker:
 8 | 	docker-compose build
 9 | .PHONY: build-docker
10 | 
11 | clean:
12 | 	make -C ./archiver clean
13 | 	make -C ./api clean
14 | 	make -C ./validator clean
15 | .PHONY: clean
16 | 
17 | test:
18 | 	make -C ./archiver test
19 | 	make -C ./api test
20 | 	make -C ./validator test
21 | .PHONY: test
22 | 
23 | integration:
24 | 	docker-compose down
25 | 	docker-compose up -d minio create-buckets
26 | 	RUN_INTEGRATION_TESTS=true go test -v ./...
27 | .PHONY: integration
28 | 
29 | fmt:
30 | 	gofmt -s -w .
31 | .PHONY: fmt
32 | 
33 | check: fmt clean build build-docker lint test integration
34 | .PHONY: check
35 | 
36 | lint:
37 | 	golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint --timeout 5m -e "errors.As" -e "errors.Is" ./...
38 | .PHONY: lint


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Blob Archiver
 2 | The Blob Archiver is a service to archive and allow querying of all historical blobs from the beacon chain. It consists 
 3 | of two components:
 4 | 
 5 | * **Archiver** - Tracks the beacon chain and writes blobs to a storage backend
 6 | * **API** - Implements the blob sidecars [API](https://ethereum.github.io/beacon-APIs/#/Beacon/getBlobSidecars), which 
 7 | allows clients to retrieve blobs from the storage backend
 8 | 
 9 | ### Storage
10 | There are currently two supported storage options:
11 | 
12 | * On-disk storage - Blobs are written to disk in a directory
13 | * S3 storage - Blobs are written to an S3 bucket (or compatible service)
14 | 
15 | You can control which storage backend is used by setting the `BLOB_API_DATA_STORE` and `BLOB_ARCHIVER_DATA_STORE` to 
16 | either `file` or `s3`.
17 | 
18 | The `s3` backend will also work with (for example) Google Cloud Storage buckets (instructions [here](https://medium.com/google-cloud/using-google-cloud-storage-with-minio-object-storage-c994fe4aab6b)). 
19 | 
20 | ### Data Validity
21 | Currently, the archiver and api do not validate the beacon node's data. Therefore, it's important to either trust the 
22 | Beacon node, or validate the data in the client. There is an open [issue](https://github.com/base-org/blob-archiver/issues/4) 
23 | to add data validation to the archiver and api.
24 | 
25 | ### Development
26 | The `Makefile` contains a number of commands for development:
27 | 
28 | ```sh
29 | # Run the tests
30 | make test
31 | # Run the integration tests (will start a local S3 bucket)
32 | make integration 
33 | 
34 | # Lint the project
35 | make lint
36 | 
37 | # Build the project
38 | make build
39 | 
40 | # Check all tests, formatting, building
41 | make check
42 | ```
43 | 
44 | #### Run Locally
45 | To run the project locally, you should first copy `.env.template` to `.env` and then modify the environment variables
46 | to your beacon client and storage backend of choice. Then you can run the project with:
47 | 
48 | ```sh
49 | docker-compose up
50 | ```
51 | 
52 | You can see a full list of configuration options by running:
53 | ```sh
54 | # API
55 | go run api/cmd/main.go
56 | 
57 | # Archiver
58 | go run archiver/cmd/main.go
59 | 
60 | ```
61 | 


--------------------------------------------------------------------------------
/api/Makefile:
--------------------------------------------------------------------------------
 1 | blob-api:
 2 | 	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/blob-api ./cmd/main.go
 3 | 
 4 | clean:
 5 | 	rm -f bin/blob-api
 6 | 
 7 | test:
 8 | 	go test -v -race ./...
 9 | 
10 | .PHONY: \
11 | 	blob-api \
12 | 	clean \
13 | 	test
14 | 


--------------------------------------------------------------------------------
/api/cmd/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"fmt"
 6 | 	"os"
 7 | 
 8 | 	"github.com/base-org/blob-archiver/api/flags"
 9 | 	"github.com/base-org/blob-archiver/api/metrics"
10 | 	"github.com/base-org/blob-archiver/api/service"
11 | 	"github.com/base-org/blob-archiver/common/beacon"
12 | 	"github.com/base-org/blob-archiver/common/storage"
13 | 	opservice "github.com/ethereum-optimism/optimism/op-service"
14 | 	"github.com/ethereum-optimism/optimism/op-service/cliapp"
15 | 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
16 | 	"github.com/ethereum/go-ethereum/log"
17 | 	"github.com/urfave/cli/v2"
18 | )
19 | 
20 | var (
21 | 	Version   = "v0.0.1"
22 | 	GitCommit = ""
23 | 	GitDate   = ""
24 | )
25 | 
26 | func main() {
27 | 	oplog.SetupDefaults()
28 | 
29 | 	app := cli.NewApp()
30 | 	app.Flags = cliapp.ProtectFlags(flags.Flags)
31 | 	app.Version = opservice.FormatVersion(Version, GitCommit, GitDate, "")
32 | 	app.Name = "blob-api"
33 | 	app.Usage = "API service for Ethereum blobs"
34 | 	app.Description = "Service for fetching blob sidecars from a datastore"
35 | 	app.Action = cliapp.LifecycleCmd(Main())
36 | 
37 | 	err := app.Run(os.Args)
38 | 	if err != nil {
39 | 		log.Crit("Application failed", "message", err)
40 | 	}
41 | }
42 | 
43 | // Main is the entrypoint into the API.
44 | // This method returns a cliapp.LifecycleAction, to create an op-service CLI-lifecycle-managed API Server.
45 | func Main() cliapp.LifecycleAction {
46 | 	return func(cliCtx *cli.Context, closeApp context.CancelCauseFunc) (cliapp.Lifecycle, error) {
47 | 		cfg := flags.ReadConfig(cliCtx)
48 | 		if err := cfg.Check(); err != nil {
49 | 			return nil, fmt.Errorf("config check failed: %w", err)
50 | 		}
51 | 
52 | 		l := oplog.NewLogger(oplog.AppOut(cliCtx), cfg.LogConfig)
53 | 		oplog.SetGlobalLogHandler(l.Handler())
54 | 		opservice.ValidateEnvVars(flags.EnvVarPrefix, flags.Flags, l)
55 | 
56 | 		m := metrics.NewMetrics()
57 | 
58 | 		storageClient, err := storage.NewStorage(cfg.StorageConfig, l)
59 | 		if err != nil {
60 | 			return nil, fmt.Errorf("failed to initialize storage: %w", err)
61 | 		}
62 | 
63 | 		beaconClient, err := beacon.NewBeaconClient(context.Background(), cfg.BeaconConfig)
64 | 		if err != nil {
65 | 			return nil, fmt.Errorf("failed to initialize beacon client: %w", err)
66 | 		}
67 | 
68 | 		l.Info("Initializing API Service")
69 | 		api := service.NewAPI(storageClient, beaconClient, m, l)
70 | 		return service.NewService(l, api, cfg, m.Registry()), nil
71 | 	}
72 | }
73 | 


--------------------------------------------------------------------------------
/api/flags/config.go:
--------------------------------------------------------------------------------
 1 | package flags
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 
 6 | 	common "github.com/base-org/blob-archiver/common/flags"
 7 | 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
 8 | 	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
 9 | 	"github.com/urfave/cli/v2"
10 | )
11 | 
12 | type APIConfig struct {
13 | 	LogConfig     oplog.CLIConfig
14 | 	MetricsConfig opmetrics.CLIConfig
15 | 	BeaconConfig  common.BeaconConfig
16 | 	StorageConfig common.StorageConfig
17 | 
18 | 	ListenAddr string
19 | }
20 | 
21 | func (c APIConfig) Check() error {
22 | 	if err := c.StorageConfig.Check(); err != nil {
23 | 		return fmt.Errorf("storage config check failed: %w", err)
24 | 	}
25 | 
26 | 	if err := c.BeaconConfig.Check(); err != nil {
27 | 		return fmt.Errorf("beacon config check failed: %w", err)
28 | 	}
29 | 
30 | 	if c.ListenAddr == "" {
31 | 		return fmt.Errorf("listen address must be set")
32 | 	}
33 | 
34 | 	return nil
35 | }
36 | 
37 | func ReadConfig(cliCtx *cli.Context) APIConfig {
38 | 	return APIConfig{
39 | 		LogConfig:     oplog.ReadCLIConfig(cliCtx),
40 | 		MetricsConfig: opmetrics.ReadCLIConfig(cliCtx),
41 | 		BeaconConfig:  common.NewBeaconConfig(cliCtx),
42 | 		StorageConfig: common.NewStorageConfig(cliCtx),
43 | 		ListenAddr:    cliCtx.String(ListenAddressFlag.Name),
44 | 	}
45 | }
46 | 


--------------------------------------------------------------------------------
/api/flags/flags.go:
--------------------------------------------------------------------------------
 1 | package flags
 2 | 
 3 | import (
 4 | 	common "github.com/base-org/blob-archiver/common/flags"
 5 | 	opservice "github.com/ethereum-optimism/optimism/op-service"
 6 | 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
 7 | 	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
 8 | 	"github.com/urfave/cli/v2"
 9 | )
10 | 
11 | const EnvVarPrefix = "BLOB_API"
12 | 
13 | var (
14 | 	ListenAddressFlag = &cli.StringFlag{
15 | 		Name:    "api-listen-address",
16 | 		Usage:   "The address to list for new requests on",
17 | 		EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "LISTEN_ADDRESS"),
18 | 		Value:   "0.0.0.0:8000",
19 | 	}
20 | )
21 | 
22 | func init() {
23 | 	Flags = append(Flags, common.CLIFlags(EnvVarPrefix)...)
24 | 	Flags = append(Flags, opmetrics.CLIFlags(EnvVarPrefix)...)
25 | 	Flags = append(Flags, oplog.CLIFlags(EnvVarPrefix)...)
26 | 	Flags = append(Flags, ListenAddressFlag)
27 | }
28 | 
29 | // Flags contains the list of configuration options available to the binary.
30 | var Flags []cli.Flag
31 | 


--------------------------------------------------------------------------------
/api/metrics/metrics.go:
--------------------------------------------------------------------------------
 1 | package metrics
 2 | 
 3 | import (
 4 | 	"github.com/ethereum-optimism/optimism/op-service/metrics"
 5 | 	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
 6 | 	"github.com/prometheus/client_golang/prometheus"
 7 | )
 8 | 
 9 | type BlockIdType string
10 | 
11 | var (
12 | 	MetricsNamespace = "blob_api"
13 | 
14 | 	BlockIdTypeHash    BlockIdType = "hash"
15 | 	BlockIdTypeBeacon  BlockIdType = "beacon"
16 | 	BlockIdTypeInvalid BlockIdType = "invalid"
17 | )
18 | 
19 | type Metricer interface {
20 | 	Registry() *prometheus.Registry
21 | 	RecordBlockIdType(t BlockIdType)
22 | }
23 | 
24 | type metricsRecorder struct {
25 | 	// blockIdType records the type of block id used to request a block. This could be a hash (BlockIdTypeHash), or a
26 | 	// beacon block identifier (BlockIdTypeBeacon).
27 | 	blockIdType *prometheus.CounterVec
28 | 	registry    *prometheus.Registry
29 | }
30 | 
31 | func NewMetrics() Metricer {
32 | 	registry := opmetrics.NewRegistry()
33 | 	factory := metrics.With(registry)
34 | 	return &metricsRecorder{
35 | 		registry: registry,
36 | 		blockIdType: factory.NewCounterVec(prometheus.CounterOpts{
37 | 			Namespace: MetricsNamespace,
38 | 			Name:      "block_id_type",
39 | 			Help:      "The type of block id used to request a block",
40 | 		}, []string{"type"}),
41 | 	}
42 | }
43 | 
44 | func (m *metricsRecorder) RecordBlockIdType(t BlockIdType) {
45 | 	m.blockIdType.WithLabelValues(string(t)).Inc()
46 | }
47 | 
48 | func (m *metricsRecorder) Registry() *prometheus.Registry {
49 | 	return m.registry
50 | }
51 | 


--------------------------------------------------------------------------------
/api/service/api.go:
--------------------------------------------------------------------------------
  1 | package service
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"encoding/json"
  6 | 	"errors"
  7 | 	"fmt"
  8 | 	"net/http"
  9 | 	"slices"
 10 | 	"strconv"
 11 | 	"strings"
 12 | 	"time"
 13 | 
 14 | 	client "github.com/attestantio/go-eth2-client"
 15 | 	"github.com/attestantio/go-eth2-client/api"
 16 | 	"github.com/attestantio/go-eth2-client/spec/deneb"
 17 | 	m "github.com/base-org/blob-archiver/api/metrics"
 18 | 	"github.com/base-org/blob-archiver/common/storage"
 19 | 	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
 20 | 	"github.com/ethereum/go-ethereum/common"
 21 | 	"github.com/ethereum/go-ethereum/common/hexutil"
 22 | 	"github.com/ethereum/go-ethereum/log"
 23 | 	"github.com/go-chi/chi/v5"
 24 | 	"github.com/go-chi/chi/v5/middleware"
 25 | )
 26 | 
 27 | type httpError struct {
 28 | 	Code    int    `json:"code"`
 29 | 	Message string `json:"message"`
 30 | }
 31 | 
 32 | func (e httpError) write(w http.ResponseWriter) {
 33 | 	w.WriteHeader(e.Code)
 34 | 	_ = json.NewEncoder(w).Encode(e)
 35 | }
 36 | 
 37 | func (e httpError) Error() string {
 38 | 	return e.Message
 39 | }
 40 | 
 41 | const (
 42 | 	jsonAcceptType = "application/json"
 43 | 	sszAcceptType  = "application/octet-stream"
 44 | 	serverTimeout  = 60 * time.Second
 45 | )
 46 | 
 47 | var (
 48 | 	errUnknownBlock = &httpError{
 49 | 		Code:    http.StatusNotFound,
 50 | 		Message: "Block not found",
 51 | 	}
 52 | 	errServerError = &httpError{
 53 | 		Code:    http.StatusInternalServerError,
 54 | 		Message: "Internal server error",
 55 | 	}
 56 | )
 57 | 
 58 | func newBlockIdError(input string) *httpError {
 59 | 	return &httpError{
 60 | 		Code:    http.StatusBadRequest,
 61 | 		Message: fmt.Sprintf("invalid block id: %s", input),
 62 | 	}
 63 | }
 64 | 
 65 | func newIndicesError(input string) *httpError {
 66 | 	return &httpError{
 67 | 		Code:    http.StatusBadRequest,
 68 | 		Message: fmt.Sprintf("invalid index input: %s", input),
 69 | 	}
 70 | }
 71 | 
 72 | func newOutOfRangeError(input uint64, blobCount int) *httpError {
 73 | 	return &httpError{
 74 | 		Code:    http.StatusBadRequest,
 75 | 		Message: fmt.Sprintf("invalid index: %d block contains %d blobs", input, blobCount),
 76 | 	}
 77 | }
 78 | 
 79 | type API struct {
 80 | 	dataStoreClient storage.DataStoreReader
 81 | 	beaconClient    client.BeaconBlockHeadersProvider
 82 | 	router          *chi.Mux
 83 | 	logger          log.Logger
 84 | 	metrics         m.Metricer
 85 | }
 86 | 
 87 | func NewAPI(dataStoreClient storage.DataStoreReader, beaconClient client.BeaconBlockHeadersProvider, metrics m.Metricer, logger log.Logger) *API {
 88 | 	result := &API{
 89 | 		dataStoreClient: dataStoreClient,
 90 | 		beaconClient:    beaconClient,
 91 | 		router:          chi.NewRouter(),
 92 | 		logger:          logger,
 93 | 		metrics:         metrics,
 94 | 	}
 95 | 
 96 | 	r := result.router
 97 | 	r.Use(middleware.Logger)
 98 | 	r.Use(middleware.Timeout(serverTimeout))
 99 | 	r.Use(middleware.Recoverer)
100 | 	r.Use(middleware.Heartbeat("/healthz"))
101 | 	r.Use(middleware.Compress(5, jsonAcceptType, sszAcceptType))
102 | 
103 | 	recorder := opmetrics.NewPromHTTPRecorder(metrics.Registry(), m.MetricsNamespace)
104 | 	r.Use(func(handler http.Handler) http.Handler {
105 | 		return opmetrics.NewHTTPRecordingMiddleware(recorder, handler)
106 | 	})
107 | 
108 | 	r.Get("/eth/v1/beacon/blob_sidecars/{id}", result.blobSidecarHandler)
109 | 
110 | 	return result
111 | }
112 | 
113 | func isHash(s string) bool {
114 | 	if len(s) != 66 || !strings.HasPrefix(s, "0x") {
115 | 		return false
116 | 	}
117 | 
118 | 	_, err := hexutil.Decode(s)
119 | 	return err == nil
120 | }
121 | 
122 | func isSlot(id string) bool {
123 | 	_, err := strconv.ParseUint(id, 10, 64)
124 | 	return err == nil
125 | }
126 | 
127 | func isKnownIdentifier(id string) bool {
128 | 	return slices.Contains([]string{"genesis", "finalized", "head"}, id)
129 | }
130 | 
131 | // toBeaconBlockHash converts a string that can be a slot, hash or identifier to a beacon block hash.
132 | func (a *API) toBeaconBlockHash(id string) (common.Hash, *httpError) {
133 | 	if isHash(id) {
134 | 		a.metrics.RecordBlockIdType(m.BlockIdTypeHash)
135 | 		return common.HexToHash(id), nil
136 | 	} else if isSlot(id) || isKnownIdentifier(id) {
137 | 		a.metrics.RecordBlockIdType(m.BlockIdTypeBeacon)
138 | 		result, err := a.beaconClient.BeaconBlockHeader(context.Background(), &api.BeaconBlockHeaderOpts{
139 | 			Common: api.CommonOpts{},
140 | 			Block:  id,
141 | 		})
142 | 
143 | 		if err != nil {
144 | 			var apiErr *api.Error
145 | 			if errors.As(err, &apiErr) && apiErr.StatusCode == 404 {
146 | 				return common.Hash{}, errUnknownBlock
147 | 			}
148 | 
149 | 			return common.Hash{}, errServerError
150 | 		}
151 | 
152 | 		return common.Hash(result.Data.Root), nil
153 | 	} else {
154 | 		a.metrics.RecordBlockIdType(m.BlockIdTypeInvalid)
155 | 		return common.Hash{}, newBlockIdError(id)
156 | 	}
157 | }
158 | 
159 | // blobSidecarHandler implements the /eth/v1/beacon/blob_sidecars/{id} endpoint, using the underlying DataStoreReader
160 | // to fetch blobs instead of the beacon node. This allows clients to fetch expired blobs.
161 | func (a *API) blobSidecarHandler(w http.ResponseWriter, r *http.Request) {
162 | 	param := chi.URLParam(r, "id")
163 | 	beaconBlockHash, err := a.toBeaconBlockHash(param)
164 | 	if err != nil {
165 | 		err.write(w)
166 | 		return
167 | 	}
168 | 
169 | 	result, storageErr := a.dataStoreClient.Read(r.Context(), beaconBlockHash)
170 | 	if storageErr != nil {
171 | 		if errors.Is(storageErr, storage.ErrNotFound) {
172 | 			errUnknownBlock.write(w)
173 | 		} else {
174 | 			a.logger.Info("unexpected error fetching blobs", "err", storageErr, "beaconBlockHash", beaconBlockHash.String(), "param", param)
175 | 			errServerError.write(w)
176 | 		}
177 | 		return
178 | 	}
179 | 
180 | 	blobSidecars := result.BlobSidecars
181 | 
182 | 	filteredBlobSidecars, err := filterBlobs(blobSidecars.Data, r.URL.Query()["indices"])
183 | 	if err != nil {
184 | 		err.write(w)
185 | 		return
186 | 	}
187 | 
188 | 	blobSidecars.Data = filteredBlobSidecars
189 | 	responseType := r.Header.Get("Accept")
190 | 
191 | 	if responseType == sszAcceptType {
192 | 		w.Header().Set("Content-Type", sszAcceptType)
193 | 		res, err := blobSidecars.MarshalSSZ()
194 | 		if err != nil {
195 | 			a.logger.Error("unable to marshal blob sidecars to SSZ", "err", err)
196 | 			errServerError.write(w)
197 | 			return
198 | 		}
199 | 
200 | 		_, err = w.Write(res)
201 | 
202 | 		if err != nil {
203 | 			a.logger.Error("unable to write ssz response", "err", err)
204 | 			errServerError.write(w)
205 | 			return
206 | 		}
207 | 	} else {
208 | 		w.Header().Set("Content-Type", jsonAcceptType)
209 | 		err := json.NewEncoder(w).Encode(blobSidecars)
210 | 		if err != nil {
211 | 			a.logger.Error("unable to encode blob sidecars to JSON", "err", err)
212 | 			errServerError.write(w)
213 | 			return
214 | 		}
215 | 	}
216 | }
217 | 
218 | // filterBlobs filters the blobs based on the indices query provided.
219 | // If no indices are provided, all blobs are returned. If invalid indices are provided, an error is returned.
220 | func filterBlobs(blobs []*deneb.BlobSidecar, _indices []string) ([]*deneb.BlobSidecar, *httpError) {
221 | 	var indices []string
222 | 	if len(_indices) == 0 {
223 | 		return blobs, nil
224 | 	} else if len(_indices) == 1 {
225 | 		indices = strings.Split(_indices[0], ",")
226 | 	} else {
227 | 		indices = _indices
228 | 	}
229 | 
230 | 	indicesMap := map[deneb.BlobIndex]struct{}{}
231 | 	for _, index := range indices {
232 | 		parsedInt, err := strconv.ParseUint(index, 10, 64)
233 | 		if err != nil {
234 | 			return nil, newIndicesError(index)
235 | 		}
236 | 
237 | 		if parsedInt >= uint64(len(blobs)) {
238 | 			return nil, newOutOfRangeError(parsedInt, len(blobs))
239 | 		}
240 | 
241 | 		blobIndex := deneb.BlobIndex(parsedInt)
242 | 		indicesMap[blobIndex] = struct{}{}
243 | 	}
244 | 
245 | 	filteredBlobs := make([]*deneb.BlobSidecar, 0)
246 | 	for _, blob := range blobs {
247 | 		if _, ok := indicesMap[blob.Index]; ok {
248 | 			filteredBlobs = append(filteredBlobs, blob)
249 | 		}
250 | 	}
251 | 
252 | 	return filteredBlobs, nil
253 | }
254 | 


--------------------------------------------------------------------------------
/api/service/api_test.go:
--------------------------------------------------------------------------------
  1 | package service
  2 | 
  3 | import (
  4 | 	"compress/gzip"
  5 | 	"context"
  6 | 	"encoding/json"
  7 | 	"fmt"
  8 | 	"io"
  9 | 	"net/http/httptest"
 10 | 	"os"
 11 | 	"testing"
 12 | 
 13 | 	"github.com/attestantio/go-eth2-client/api"
 14 | 	v1 "github.com/attestantio/go-eth2-client/api/v1"
 15 | 	"github.com/attestantio/go-eth2-client/spec/deneb"
 16 | 	"github.com/attestantio/go-eth2-client/spec/phase0"
 17 | 	"github.com/base-org/blob-archiver/api/metrics"
 18 | 	"github.com/base-org/blob-archiver/common/beacon/beacontest"
 19 | 	"github.com/base-org/blob-archiver/common/blobtest"
 20 | 	"github.com/base-org/blob-archiver/common/storage"
 21 | 	"github.com/ethereum-optimism/optimism/op-service/testlog"
 22 | 	"github.com/ethereum/go-ethereum/common"
 23 | 	"github.com/ethereum/go-ethereum/log"
 24 | 	"github.com/stretchr/testify/require"
 25 | )
 26 | 
 27 | func TestIsHash(t *testing.T) {
 28 | 	require.True(t, isHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"))
 29 | 	// Invalid hex character, ending with z
 30 | 	require.False(t, isHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdez"))
 31 | 	// Missing 0x prefix
 32 | 	require.False(t, isHash("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"))
 33 | 
 34 | 	require.False(t, isHash("genesis"))
 35 | 	require.False(t, isHash("finalized"))
 36 | 	require.False(t, isHash("123"))     // slot
 37 | 	require.False(t, isHash("unknown")) // incorrect input
 38 | }
 39 | 
 40 | func TestIsSlot(t *testing.T) {
 41 | 	require.True(t, isSlot("123"))
 42 | 	require.False(t, isSlot("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"))
 43 | 	require.False(t, isSlot("genesis"))
 44 | 	require.False(t, isSlot("finalized"))
 45 | 	require.False(t, isSlot("unknown"))
 46 | }
 47 | 
 48 | func TestIsNamedIdentifier(t *testing.T) {
 49 | 	require.True(t, isKnownIdentifier("genesis"))
 50 | 	require.True(t, isKnownIdentifier("finalized"))
 51 | 	require.False(t, isKnownIdentifier("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"))
 52 | 	require.False(t, isKnownIdentifier("123"))
 53 | 	require.False(t, isKnownIdentifier("unknown"))
 54 | }
 55 | 
 56 | func setup(t *testing.T) (*API, *storage.FileStorage, *beacontest.StubBeaconClient, func()) {
 57 | 	logger := testlog.Logger(t, log.LvlInfo)
 58 | 	tempDir, err := os.MkdirTemp("", "test")
 59 | 	require.NoError(t, err)
 60 | 	fs := storage.NewFileStorage(tempDir, logger)
 61 | 	beacon := beacontest.NewEmptyStubBeaconClient()
 62 | 	m := metrics.NewMetrics()
 63 | 	a := NewAPI(fs, beacon, m, logger)
 64 | 	return a, fs, beacon, func() {
 65 | 		require.NoError(t, os.RemoveAll(tempDir))
 66 | 	}
 67 | }
 68 | 
 69 | func TestAPIService(t *testing.T) {
 70 | 	a, fs, beaconClient, cleanup := setup(t)
 71 | 	defer cleanup()
 72 | 
 73 | 	rootOne := common.HexToHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef")
 74 | 	rootTwo := common.HexToHash("0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890222222")
 75 | 
 76 | 	blockOne := storage.BlobData{
 77 | 		Header: storage.Header{
 78 | 			BeaconBlockHash: rootOne,
 79 | 		},
 80 | 		BlobSidecars: storage.BlobSidecars{
 81 | 			Data: blobtest.NewBlobSidecars(t, 2),
 82 | 		},
 83 | 	}
 84 | 
 85 | 	blockTwo := storage.BlobData{
 86 | 		Header: storage.Header{
 87 | 			BeaconBlockHash: rootTwo,
 88 | 		},
 89 | 		BlobSidecars: storage.BlobSidecars{
 90 | 			Data: blobtest.NewBlobSidecars(t, 2),
 91 | 		},
 92 | 	}
 93 | 
 94 | 	err := fs.Write(context.Background(), blockOne)
 95 | 	require.NoError(t, err)
 96 | 
 97 | 	err = fs.Write(context.Background(), blockTwo)
 98 | 	require.NoError(t, err)
 99 | 
100 | 	beaconClient.Headers["finalized"] = &v1.BeaconBlockHeader{
101 | 		Root: phase0.Root(rootOne),
102 | 	}
103 | 
104 | 	beaconClient.Headers["head"] = &v1.BeaconBlockHeader{
105 | 		Root: phase0.Root(rootTwo),
106 | 	}
107 | 
108 | 	beaconClient.Headers["1234"] = &v1.BeaconBlockHeader{
109 | 		Root: phase0.Root(rootTwo),
110 | 	}
111 | 
112 | 	tests := []struct {
113 | 		name       string
114 | 		path       string
115 | 		status     int
116 | 		expected   *storage.BlobSidecars
117 | 		errMessage string
118 | 	}{
119 | 		{
120 | 			name:     "fetch root one",
121 | 			path:     fmt.Sprintf("/eth/v1/beacon/blob_sidecars/%s", rootOne),
122 | 			status:   200,
123 | 			expected: &blockOne.BlobSidecars,
124 | 		},
125 | 		{
126 | 			name:     "fetch root two",
127 | 			path:     fmt.Sprintf("/eth/v1/beacon/blob_sidecars/%s", rootTwo),
128 | 			status:   200,
129 | 			expected: &blockTwo.BlobSidecars,
130 | 		},
131 | 		{
132 | 			name:       "fetch unknown",
133 | 			path:       "/eth/v1/beacon/blob_sidecars/0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abc111",
134 | 			status:     404,
135 | 			errMessage: "Block not found",
136 | 		},
137 | 		{
138 | 			name:     "fetch head",
139 | 			path:     "/eth/v1/beacon/blob_sidecars/head",
140 | 			status:   200,
141 | 			expected: &blockTwo.BlobSidecars,
142 | 		},
143 | 		{
144 | 			name:     "fetch finalized",
145 | 			path:     "/eth/v1/beacon/blob_sidecars/finalized",
146 | 			status:   200,
147 | 			expected: &blockOne.BlobSidecars,
148 | 		},
149 | 		{
150 | 			name:     "fetch slot 1234",
151 | 			path:     "/eth/v1/beacon/blob_sidecars/1234",
152 | 			status:   200,
153 | 			expected: &blockTwo.BlobSidecars,
154 | 		},
155 | 		{
156 | 			name:   "indices only returns requested indices",
157 | 			path:   "/eth/v1/beacon/blob_sidecars/1234?indices=1",
158 | 			status: 200,
159 | 			expected: &storage.BlobSidecars{
160 | 				Data: []*deneb.BlobSidecar{
161 | 					blockTwo.BlobSidecars.Data[1],
162 | 				},
163 | 			},
164 | 		},
165 | 		{
166 | 			name:   "deduplicates indices",
167 | 			path:   "/eth/v1/beacon/blob_sidecars/1234?indices=1,1,1",
168 | 			status: 200,
169 | 			expected: &storage.BlobSidecars{
170 | 				Data: []*deneb.BlobSidecar{
171 | 					blockTwo.BlobSidecars.Data[1],
172 | 				},
173 | 			},
174 | 		},
175 | 		{
176 | 			name:   "multi indices",
177 | 			path:   "/eth/v1/beacon/blob_sidecars/1234?indices=0&indices=1",
178 | 			status: 200,
179 | 			expected: &storage.BlobSidecars{
180 | 				Data: blockTwo.BlobSidecars.Data,
181 | 			},
182 | 		},
183 | 		{
184 | 			name:   "multi indices comma separated list",
185 | 			path:   "/eth/v1/beacon/blob_sidecars/1234?indices=0,1",
186 | 			status: 200,
187 | 			expected: &storage.BlobSidecars{
188 | 				Data: blockTwo.BlobSidecars.Data,
189 | 			},
190 | 		},
191 | 		{
192 | 			name:       "only index out of bounds returns empty array",
193 | 			path:       "/eth/v1/beacon/blob_sidecars/1234?indices=3",
194 | 			status:     400,
195 | 			errMessage: "invalid index: 3 block contains 2 blobs",
196 | 		},
197 | 		{
198 | 			name:       "any index out of bounds returns error",
199 | 			path:       "/eth/v1/beacon/blob_sidecars/1234?indices=1,10",
200 | 			status:     400,
201 | 			errMessage: "invalid index: 10 block contains 2 blobs",
202 | 		},
203 | 		{
204 | 			name:       "only index out of bounds (boundary condition) returns error",
205 | 			path:       "/eth/v1/beacon/blob_sidecars/1234?indices=2",
206 | 			status:     400,
207 | 			errMessage: "invalid index: 2 block contains 2 blobs",
208 | 		},
209 | 		{
210 | 			name:       "negative index returns error",
211 | 			path:       "/eth/v1/beacon/blob_sidecars/1234?indices=-2",
212 | 			status:     400,
213 | 			errMessage: "invalid index input: -2",
214 | 		},
215 | 		{
216 | 			name:       "no 0x on hash",
217 | 			path:       "/eth/v1/beacon/blob_sidecars/1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef",
218 | 			status:     400,
219 | 			errMessage: "invalid block id: 1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef",
220 | 		},
221 | 		{
222 | 			name:       "invalid hash",
223 | 			path:       "/eth/v1/beacon/blob_sidecars/0x1234567890abcdef123",
224 | 			status:     400,
225 | 			errMessage: "invalid block id: 0x1234567890abcdef123",
226 | 		},
227 | 		{
228 | 			name:       "invalid named identifier",
229 | 			path:       "/eth/v1/beacon/blob_sidecars/foobar",
230 | 			status:     400,
231 | 			errMessage: "invalid block id: foobar",
232 | 		},
233 | 		{
234 | 			name:   "invalid no parameter specified",
235 | 			path:   "/eth/v1/beacon/blob_sidecars/",
236 | 			status: 404,
237 | 		},
238 | 		{
239 | 			name:   "unknown route",
240 | 			path:   "/eth/v1/",
241 | 			status: 404,
242 | 		},
243 | 	}
244 | 
245 | 	responseFormat := []string{"application/json", "application/octet-stream"}
246 | 
247 | 	for _, test := range tests {
248 | 		for _, rf := range responseFormat {
249 | 			for _, compress := range []bool{true, false} {
250 | 				testName := fmt.Sprintf("%s-%s", test.name, rf)
251 | 				if compress {
252 | 					testName = fmt.Sprintf("%s-%s", testName, "gzip")
253 | 				}
254 | 
255 | 				t.Run(testName, func(t *testing.T) {
256 | 					request := httptest.NewRequest("GET", test.path, nil)
257 | 					request.Header.Set("Accept", rf)
258 | 
259 | 					if compress {
260 | 						request.Header.Set("Accept-Encoding", "gzip")
261 | 					}
262 | 
263 | 					response := httptest.NewRecorder()
264 | 
265 | 					a.router.ServeHTTP(response, request)
266 | 
267 | 					require.Equal(t, test.status, response.Code)
268 | 
269 | 					if test.status == 200 && test.expected != nil {
270 | 						var data []byte
271 | 						if compress {
272 | 							reader, err := gzip.NewReader(response.Body)
273 | 							require.NoError(t, err)
274 | 
275 | 							data, err = io.ReadAll(reader)
276 | 							require.NoError(t, err)
277 | 						} else {
278 | 							data = response.Body.Bytes()
279 | 						}
280 | 
281 | 						blobSidecars := storage.BlobSidecars{}
282 | 
283 | 						if rf == "application/octet-stream" {
284 | 							res := api.BlobSidecars{}
285 | 							err = res.UnmarshalSSZ(data)
286 | 							blobSidecars.Data = res.Sidecars
287 | 						} else {
288 | 							err = json.Unmarshal(data, &blobSidecars)
289 | 						}
290 | 
291 | 						require.NoError(t, err)
292 | 						require.Equal(t, *test.expected, blobSidecars)
293 | 					} else if test.status != 200 && rf == "application/json" && test.errMessage != "" {
294 | 						var e httpError
295 | 						err := json.Unmarshal(response.Body.Bytes(), &e)
296 | 						require.NoError(t, err)
297 | 						require.Equal(t, test.status, e.Code)
298 | 						require.Equal(t, test.errMessage, e.Message)
299 | 					}
300 | 				})
301 | 			}
302 | 		}
303 | 	}
304 | }
305 | 
306 | func TestHealthHandler(t *testing.T) {
307 | 	a, _, _, cleanup := setup(t)
308 | 	defer cleanup()
309 | 
310 | 	request := httptest.NewRequest("GET", "/healthz", nil)
311 | 	response := httptest.NewRecorder()
312 | 
313 | 	a.router.ServeHTTP(response, request)
314 | 
315 | 	require.Equal(t, 200, response.Code)
316 | }
317 | 


--------------------------------------------------------------------------------
/api/service/service.go:
--------------------------------------------------------------------------------
 1 | package service
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"errors"
 6 | 	"fmt"
 7 | 	"sync/atomic"
 8 | 
 9 | 	"github.com/base-org/blob-archiver/api/flags"
10 | 	"github.com/ethereum-optimism/optimism/op-service/httputil"
11 | 	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
12 | 	"github.com/ethereum/go-ethereum/log"
13 | 	"github.com/prometheus/client_golang/prometheus"
14 | )
15 | 
16 | var ErrAlreadyStopped = errors.New("already stopped")
17 | 
18 | func NewService(l log.Logger, api *API, cfg flags.APIConfig, registry *prometheus.Registry) *APIService {
19 | 	return &APIService{
20 | 		log:      l,
21 | 		cfg:      cfg,
22 | 		registry: registry,
23 | 		api:      api,
24 | 	}
25 | }
26 | 
27 | type APIService struct {
28 | 	stopped       atomic.Bool
29 | 	log           log.Logger
30 | 	cfg           flags.APIConfig
31 | 	registry      *prometheus.Registry
32 | 	metricsServer *httputil.HTTPServer
33 | 	apiServer     *httputil.HTTPServer
34 | 	api           *API
35 | }
36 | 
37 | func (a *APIService) Start(ctx context.Context) error {
38 | 	if a.cfg.MetricsConfig.Enabled {
39 | 		a.log.Info("starting metrics server", "addr", a.cfg.MetricsConfig.ListenAddr, "port", a.cfg.MetricsConfig.ListenPort)
40 | 		srv, err := opmetrics.StartServer(a.registry, a.cfg.MetricsConfig.ListenAddr, a.cfg.MetricsConfig.ListenPort)
41 | 		if err != nil {
42 | 			return err
43 | 		}
44 | 
45 | 		a.log.Info("started metrics server", "addr", srv.Addr())
46 | 		a.metricsServer = srv
47 | 	}
48 | 
49 | 	a.log.Debug("starting API server", "address", a.cfg.ListenAddr)
50 | 
51 | 	srv, err := httputil.StartHTTPServer(a.cfg.ListenAddr, a.api.router)
52 | 	if err != nil {
53 | 		return fmt.Errorf("failed to start API server: %w", err)
54 | 	}
55 | 
56 | 	a.log.Info("API server started", "address", srv.Addr().String())
57 | 	a.apiServer = srv
58 | 	return nil
59 | }
60 | 
61 | func (a *APIService) Stop(ctx context.Context) error {
62 | 	if a.stopped.Load() {
63 | 		return ErrAlreadyStopped
64 | 	}
65 | 	a.log.Info("Stopping API")
66 | 	a.stopped.Store(true)
67 | 
68 | 	if a.apiServer != nil {
69 | 		if err := a.apiServer.Shutdown(ctx); err != nil {
70 | 			return err
71 | 		}
72 | 	}
73 | 
74 | 	if a.metricsServer != nil {
75 | 		if err := a.metricsServer.Stop(ctx); err != nil {
76 | 			return err
77 | 		}
78 | 	}
79 | 
80 | 	return nil
81 | }
82 | 
83 | func (a *APIService) Stopped() bool {
84 | 	return a.stopped.Load()
85 | }
86 | 


--------------------------------------------------------------------------------
/archiver/Makefile:
--------------------------------------------------------------------------------
 1 | blob-archiver:
 2 | 	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/blob-archiver ./cmd/main.go
 3 | 
 4 | clean:
 5 | 	rm -f bin/blob-archiver
 6 | 
 7 | test:
 8 | 	go test -v -race ./...
 9 | 
10 | .PHONY: \
11 | 	blob-archiver \
12 | 	clean \
13 | 	test


--------------------------------------------------------------------------------
/archiver/cmd/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"fmt"
 6 | 	"os"
 7 | 
 8 | 	"github.com/base-org/blob-archiver/archiver/flags"
 9 | 	"github.com/base-org/blob-archiver/archiver/metrics"
10 | 	"github.com/base-org/blob-archiver/archiver/service"
11 | 	"github.com/base-org/blob-archiver/common/beacon"
12 | 	"github.com/base-org/blob-archiver/common/storage"
13 | 	opservice "github.com/ethereum-optimism/optimism/op-service"
14 | 	"github.com/ethereum-optimism/optimism/op-service/cliapp"
15 | 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
16 | 	"github.com/ethereum/go-ethereum/log"
17 | 	"github.com/urfave/cli/v2"
18 | )
19 | 
20 | var (
21 | 	Version   = "v0.0.1"
22 | 	GitCommit = ""
23 | 	GitDate   = ""
24 | )
25 | 
26 | func main() {
27 | 	oplog.SetupDefaults()
28 | 
29 | 	app := cli.NewApp()
30 | 	app.Flags = cliapp.ProtectFlags(flags.Flags)
31 | 	app.Version = opservice.FormatVersion(Version, GitCommit, GitDate, "")
32 | 	app.Name = "blob-archiver"
33 | 	app.Usage = "Archiver service for Ethereum blobs"
34 | 	app.Description = "Service for fetching blobs and archiving them to a datastore"
35 | 	app.Action = cliapp.LifecycleCmd(Main())
36 | 
37 | 	err := app.Run(os.Args)
38 | 	if err != nil {
39 | 		log.Crit("Application failed", "message", err)
40 | 	}
41 | }
42 | 
43 | // Main is the entrypoint into the Archiver.
44 | // This method returns a cliapp.LifecycleAction, to create an op-service CLI-lifecycle-managed archiver.
45 | func Main() cliapp.LifecycleAction {
46 | 	return func(cliCtx *cli.Context, closeApp context.CancelCauseFunc) (cliapp.Lifecycle, error) {
47 | 		cfg := flags.ReadConfig(cliCtx)
48 | 
49 | 		if err := cfg.Check(); err != nil {
50 | 			return nil, fmt.Errorf("invalid CLI flags: %w", err)
51 | 		}
52 | 
53 | 		l := oplog.NewLogger(oplog.AppOut(cliCtx), cfg.LogConfig)
54 | 		oplog.SetGlobalLogHandler(l.Handler())
55 | 		opservice.ValidateEnvVars(flags.EnvVarPrefix, flags.Flags, l)
56 | 
57 | 		m := metrics.NewMetrics()
58 | 
59 | 		beaconClient, err := beacon.NewBeaconClient(context.Background(), cfg.BeaconConfig)
60 | 		if err != nil {
61 | 			return nil, err
62 | 		}
63 | 
64 | 		storageClient, err := storage.NewStorage(cfg.StorageConfig, l)
65 | 		if err != nil {
66 | 			return nil, err
67 | 		}
68 | 
69 | 		l.Info("Initializing Archiver Service")
70 | 		archiver, err := service.NewArchiver(l, cfg, storageClient, beaconClient, m)
71 | 		if err != nil {
72 | 			return nil, fmt.Errorf("failed to initialize archiver: %w", err)
73 | 		}
74 | 
75 | 		api := service.NewAPI(m, l, archiver)
76 | 
77 | 		return service.NewService(l, cfg, api, archiver, m)
78 | 	}
79 | }
80 | 


--------------------------------------------------------------------------------
/archiver/flags/config.go:
--------------------------------------------------------------------------------
 1 | package flags
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"time"
 6 | 
 7 | 	common "github.com/base-org/blob-archiver/common/flags"
 8 | 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
 9 | 	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
10 | 	geth "github.com/ethereum/go-ethereum/common"
11 | 	"github.com/urfave/cli/v2"
12 | )
13 | 
14 | type ArchiverConfig struct {
15 | 	LogConfig     oplog.CLIConfig
16 | 	MetricsConfig opmetrics.CLIConfig
17 | 	BeaconConfig  common.BeaconConfig
18 | 	StorageConfig common.StorageConfig
19 | 	PollInterval  time.Duration
20 | 	OriginBlock   geth.Hash
21 | 	ListenAddr    string
22 | }
23 | 
24 | func (c ArchiverConfig) Check() error {
25 | 	if err := c.StorageConfig.Check(); err != nil {
26 | 		return err
27 | 	}
28 | 
29 | 	if err := c.BeaconConfig.Check(); err != nil {
30 | 		return err
31 | 	}
32 | 
33 | 	if c.PollInterval == 0 {
34 | 		return fmt.Errorf("archiver poll interval must be set")
35 | 	}
36 | 
37 | 	if c.OriginBlock == (geth.Hash{}) {
38 | 		return fmt.Errorf("invalid origin block")
39 | 	}
40 | 
41 | 	if c.ListenAddr == "" {
42 | 		return fmt.Errorf("archiver listen address must be set")
43 | 	}
44 | 
45 | 	return nil
46 | }
47 | 
48 | func ReadConfig(cliCtx *cli.Context) ArchiverConfig {
49 | 	pollInterval, _ := time.ParseDuration(cliCtx.String(ArchiverPollIntervalFlag.Name))
50 | 	return ArchiverConfig{
51 | 		LogConfig:     oplog.ReadCLIConfig(cliCtx),
52 | 		MetricsConfig: opmetrics.ReadCLIConfig(cliCtx),
53 | 		BeaconConfig:  common.NewBeaconConfig(cliCtx),
54 | 		StorageConfig: common.NewStorageConfig(cliCtx),
55 | 		PollInterval:  pollInterval,
56 | 		OriginBlock:   geth.HexToHash(cliCtx.String(ArchiverOriginBlock.Name)),
57 | 		ListenAddr:    cliCtx.String(ArchiverListenAddrFlag.Name),
58 | 	}
59 | }
60 | 


--------------------------------------------------------------------------------
/archiver/flags/flags.go:
--------------------------------------------------------------------------------
 1 | package flags
 2 | 
 3 | import (
 4 | 	common "github.com/base-org/blob-archiver/common/flags"
 5 | 	opservice "github.com/ethereum-optimism/optimism/op-service"
 6 | 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
 7 | 	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
 8 | 	"github.com/urfave/cli/v2"
 9 | )
10 | 
11 | const EnvVarPrefix = "BLOB_ARCHIVER"
12 | 
13 | var (
14 | 	ArchiverPollIntervalFlag = &cli.StringFlag{
15 | 		Name:    "archiver-poll-interval",
16 | 		Usage:   "The interval at which the archiver polls for new blobs",
17 | 		EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "ARCHIVER_POLL_INTERVAL"),
18 | 		Value:   "6s",
19 | 	}
20 | 	ArchiverOriginBlock = &cli.StringFlag{
21 | 		Name:     "archiver-origin-block",
22 | 		Usage:    "The latest block hash that the archiver will walk back to",
23 | 		Required: true,
24 | 		EnvVars:  opservice.PrefixEnvVar(EnvVarPrefix, "ORIGIN_BLOCK"),
25 | 	}
26 | 	ArchiverListenAddrFlag = &cli.StringFlag{
27 | 		Name:    "archiver-listen-address",
28 | 		Usage:   "The address to list for new requests on",
29 | 		EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "LISTEN_ADDRESS"),
30 | 		Value:   "0.0.0.0:8000",
31 | 	}
32 | )
33 | 
34 | func init() {
35 | 	Flags = append(Flags, common.CLIFlags(EnvVarPrefix)...)
36 | 	Flags = append(Flags, opmetrics.CLIFlags(EnvVarPrefix)...)
37 | 	Flags = append(Flags, oplog.CLIFlags(EnvVarPrefix)...)
38 | 	Flags = append(Flags, ArchiverPollIntervalFlag, ArchiverOriginBlock, ArchiverListenAddrFlag)
39 | }
40 | 
41 | // Flags contains the list of configuration options available to the binary.
42 | var Flags []cli.Flag
43 | 


--------------------------------------------------------------------------------
/archiver/metrics/metrics.go:
--------------------------------------------------------------------------------
 1 | package metrics
 2 | 
 3 | import (
 4 | 	"github.com/ethereum-optimism/optimism/op-service/metrics"
 5 | 	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
 6 | 	"github.com/prometheus/client_golang/prometheus"
 7 | )
 8 | 
 9 | type BlockSource string
10 | 
11 | var (
12 | 	MetricsNamespace = "blob_archiver"
13 | 
14 | 	BlockSourceBackfill  BlockSource = "backfill"
15 | 	BlockSourceLive      BlockSource = "live"
16 | 	BlockSourceRearchive BlockSource = "rearchive"
17 | )
18 | 
19 | type Metricer interface {
20 | 	Registry() *prometheus.Registry
21 | 	RecordProcessedBlock(source BlockSource)
22 | 	RecordStoredBlobs(count int)
23 | }
24 | 
25 | type metricsRecorder struct {
26 | 	blockProcessedCounter *prometheus.CounterVec
27 | 	blobsStored           prometheus.Counter
28 | 	registry              *prometheus.Registry
29 | }
30 | 
31 | func NewMetrics() Metricer {
32 | 	registry := opmetrics.NewRegistry()
33 | 	factory := metrics.With(registry)
34 | 	return &metricsRecorder{
35 | 		registry: registry,
36 | 		blockProcessedCounter: factory.NewCounterVec(prometheus.CounterOpts{
37 | 			Namespace: MetricsNamespace,
38 | 			Name:      "blocks_processed",
39 | 			Help:      "number of times processing loop has run",
40 | 		}, []string{"source"}),
41 | 		blobsStored: factory.NewCounter(prometheus.CounterOpts{
42 | 			Namespace: MetricsNamespace,
43 | 			Name:      "blobs_stored",
44 | 			Help:      "number of blobs stored",
45 | 		}),
46 | 	}
47 | }
48 | 
49 | func (m *metricsRecorder) Registry() *prometheus.Registry {
50 | 	return m.registry
51 | }
52 | 
53 | func (m *metricsRecorder) RecordStoredBlobs(count int) {
54 | 	m.blobsStored.Add(float64(count))
55 | }
56 | 
57 | func (m *metricsRecorder) RecordProcessedBlock(source BlockSource) {
58 | 	m.blockProcessedCounter.WithLabelValues(string(source)).Inc()
59 | }
60 | 


--------------------------------------------------------------------------------
/archiver/service/api.go:
--------------------------------------------------------------------------------
  1 | package service
  2 | 
  3 | import (
  4 | 	"encoding/json"
  5 | 	"fmt"
  6 | 	"net/http"
  7 | 	"strconv"
  8 | 	"time"
  9 | 
 10 | 	m "github.com/base-org/blob-archiver/archiver/metrics"
 11 | 	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
 12 | 	"github.com/ethereum/go-ethereum/log"
 13 | 	"github.com/go-chi/chi/v5"
 14 | 	"github.com/go-chi/chi/v5/middleware"
 15 | )
 16 | 
 17 | const (
 18 | 	serverTimeout = 60 * time.Second
 19 | )
 20 | 
 21 | type API struct {
 22 | 	router   *chi.Mux
 23 | 	logger   log.Logger
 24 | 	metrics  m.Metricer
 25 | 	archiver *Archiver
 26 | }
 27 | 
 28 | // NewAPI creates a new Archiver API instance. This API exposes an admin interface to control the archiver.
 29 | func NewAPI(metrics m.Metricer, logger log.Logger, archiver *Archiver) *API {
 30 | 	result := &API{
 31 | 		router:   chi.NewRouter(),
 32 | 		archiver: archiver,
 33 | 		logger:   logger,
 34 | 		metrics:  metrics,
 35 | 	}
 36 | 
 37 | 	r := result.router
 38 | 	r.Use(middleware.Logger)
 39 | 	r.Use(middleware.Timeout(serverTimeout))
 40 | 	r.Use(middleware.Recoverer)
 41 | 	r.Use(middleware.Heartbeat("/healthz"))
 42 | 
 43 | 	recorder := opmetrics.NewPromHTTPRecorder(metrics.Registry(), m.MetricsNamespace)
 44 | 	r.Use(func(handler http.Handler) http.Handler {
 45 | 		return opmetrics.NewHTTPRecordingMiddleware(recorder, handler)
 46 | 	})
 47 | 
 48 | 	r.Get("/", http.NotFound)
 49 | 	r.Post("/rearchive", result.rearchiveBlocks)
 50 | 
 51 | 	return result
 52 | }
 53 | 
 54 | type rearchiveResponse struct {
 55 | 	Error      string `json:"error,omitempty"`
 56 | 	BlockStart uint64 `json:"blockStart"`
 57 | 	BlockEnd   uint64 `json:"blockEnd"`
 58 | }
 59 | 
 60 | func toSlot(input string) (uint64, error) {
 61 | 	if input == "" {
 62 | 		return 0, fmt.Errorf("must provide param")
 63 | 	}
 64 | 	res, err := strconv.ParseUint(input, 10, 64)
 65 | 	if err != nil {
 66 | 		return 0, fmt.Errorf("invalid slot: \"%s\"", input)
 67 | 	}
 68 | 	return res, nil
 69 | }
 70 | 
 71 | // rearchiveBlocks rearchives blobs from blocks between the given from and to slots.
 72 | // If any blocks are already archived, they will be overwritten with data from the beacon node.
 73 | func (a *API) rearchiveBlocks(w http.ResponseWriter, r *http.Request) {
 74 | 	from, err := toSlot(r.URL.Query().Get("from"))
 75 | 	if err != nil {
 76 | 		w.WriteHeader(http.StatusBadRequest)
 77 | 		_ = json.NewEncoder(w).Encode(rearchiveResponse{
 78 | 			Error: fmt.Sprintf("invalid from param: %v", err),
 79 | 		})
 80 | 		return
 81 | 	}
 82 | 
 83 | 	to, err := toSlot(r.URL.Query().Get("to"))
 84 | 	if err != nil {
 85 | 		w.WriteHeader(http.StatusBadRequest)
 86 | 		_ = json.NewEncoder(w).Encode(rearchiveResponse{
 87 | 			Error: fmt.Sprintf("invalid to param: %v", err),
 88 | 		})
 89 | 		return
 90 | 	}
 91 | 
 92 | 	if from > to {
 93 | 		w.WriteHeader(http.StatusBadRequest)
 94 | 		_ = json.NewEncoder(w).Encode(rearchiveResponse{
 95 | 			Error: fmt.Sprintf("invalid range: from %d to %d", from, to),
 96 | 		})
 97 | 		return
 98 | 	}
 99 | 
100 | 	blockStart, blockEnd, err := a.archiver.rearchiveRange(from, to)
101 | 	if err != nil {
102 | 		a.logger.Error("Failed to rearchive blocks", "err", err)
103 | 
104 | 		w.WriteHeader(http.StatusInternalServerError)
105 | 		err = json.NewEncoder(w).Encode(rearchiveResponse{
106 | 			Error:      err.Error(),
107 | 			BlockStart: blockStart,
108 | 			BlockEnd:   blockEnd,
109 | 		})
110 | 	} else {
111 | 		a.logger.Info("Rearchiving blocks complete")
112 | 		w.WriteHeader(http.StatusOK)
113 | 
114 | 		err = json.NewEncoder(w).Encode(rearchiveResponse{
115 | 			BlockStart: blockStart,
116 | 			BlockEnd:   blockEnd,
117 | 		})
118 | 	}
119 | 
120 | 	if err != nil {
121 | 		a.logger.Error("Failed to write response", "err", err)
122 | 		w.WriteHeader(http.StatusInternalServerError)
123 | 	}
124 | }
125 | 


--------------------------------------------------------------------------------
/archiver/service/api_test.go:
--------------------------------------------------------------------------------
  1 | package service
  2 | 
  3 | import (
  4 | 	"encoding/json"
  5 | 	"net/http/httptest"
  6 | 	"testing"
  7 | 	"time"
  8 | 
  9 | 	"github.com/base-org/blob-archiver/archiver/flags"
 10 | 	"github.com/base-org/blob-archiver/archiver/metrics"
 11 | 	"github.com/base-org/blob-archiver/common/storage/storagetest"
 12 | 	"github.com/ethereum-optimism/optimism/op-service/testlog"
 13 | 	"github.com/ethereum/go-ethereum/log"
 14 | 	"github.com/stretchr/testify/require"
 15 | )
 16 | 
 17 | func setupAPI(t *testing.T) (*API, *storagetest.TestFileStorage) {
 18 | 	logger := testlog.Logger(t, log.LvlInfo)
 19 | 	m := metrics.NewMetrics()
 20 | 	fs := storagetest.NewTestFileStorage(t, logger)
 21 | 	archiver, err := NewArchiver(logger, flags.ArchiverConfig{
 22 | 		PollInterval: 10 * time.Second,
 23 | 	}, fs, nil, m)
 24 | 	require.NoError(t, err)
 25 | 	return NewAPI(m, logger, archiver), fs
 26 | }
 27 | 
 28 | func TestHealthHandler(t *testing.T) {
 29 | 	a, _ := setupAPI(t)
 30 | 
 31 | 	request := httptest.NewRequest("GET", "/healthz", nil)
 32 | 	response := httptest.NewRecorder()
 33 | 
 34 | 	a.router.ServeHTTP(response, request)
 35 | 
 36 | 	require.Equal(t, 200, response.Code)
 37 | }
 38 | 
 39 | func TestRearchiveHandler(t *testing.T) {
 40 | 	a, _ := setupAPI(t)
 41 | 
 42 | 	tests := []struct {
 43 | 		name           string
 44 | 		path           string
 45 | 		expectedStatus int
 46 | 		error          string
 47 | 	}{
 48 | 		{
 49 | 			name:           "should fail with no params",
 50 | 			path:           "/rearchive",
 51 | 			expectedStatus: 400,
 52 | 			error:          "invalid from param: must provide param",
 53 | 		},
 54 | 		{
 55 | 			name:           "should fail with missing to param",
 56 | 			path:           "/rearchive?from=1",
 57 | 			expectedStatus: 400,
 58 | 			error:          "invalid to param: must provide param",
 59 | 		},
 60 | 		{
 61 | 			name:           "should fail with missing from param",
 62 | 			path:           "/rearchive?to=1",
 63 | 			expectedStatus: 400,
 64 | 			error:          "invalid from param: must provide param",
 65 | 		},
 66 | 		{
 67 | 			name:           "should fail with invalid from param",
 68 | 			path:           "/rearchive?from=blah&to=1",
 69 | 			expectedStatus: 400,
 70 | 			error:          "invalid from param: invalid slot: \"blah\"",
 71 | 		},
 72 | 		{
 73 | 			name:           "should fail with invalid to param",
 74 | 			path:           "/rearchive?from=1&to=blah",
 75 | 			expectedStatus: 400,
 76 | 			error:          "invalid to param: invalid slot: \"blah\"",
 77 | 		},
 78 | 		{
 79 | 			name:           "should fail with to greater than equal to from",
 80 | 			path:           "/rearchive?from=2&to=1",
 81 | 			expectedStatus: 400,
 82 | 			error:          "invalid range: from 2 to 1",
 83 | 		},
 84 | 	}
 85 | 
 86 | 	for _, tt := range tests {
 87 | 		test := tt
 88 | 		t.Run(test.name, func(t *testing.T) {
 89 | 			request := httptest.NewRequest("POST", test.path, nil)
 90 | 			response := httptest.NewRecorder()
 91 | 
 92 | 			a.router.ServeHTTP(response, request)
 93 | 
 94 | 			require.Equal(t, test.expectedStatus, response.Code)
 95 | 
 96 | 			var errResponse rearchiveResponse
 97 | 			err := json.NewDecoder(response.Body).Decode(&errResponse)
 98 | 			require.NoError(t, err)
 99 | 
100 | 			if test.error != "" {
101 | 				require.Equal(t, errResponse.Error, test.error)
102 | 			}
103 | 		})
104 | 	}
105 | }
106 | 


--------------------------------------------------------------------------------
/archiver/service/archiver.go:
--------------------------------------------------------------------------------
  1 | package service
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"errors"
  6 | 	"strconv"
  7 | 	"time"
  8 | 
  9 | 	client "github.com/attestantio/go-eth2-client"
 10 | 	"github.com/attestantio/go-eth2-client/api"
 11 | 	v1 "github.com/attestantio/go-eth2-client/api/v1"
 12 | 	"github.com/base-org/blob-archiver/archiver/flags"
 13 | 	"github.com/base-org/blob-archiver/archiver/metrics"
 14 | 	"github.com/base-org/blob-archiver/common/storage"
 15 | 	"github.com/ethereum-optimism/optimism/op-service/retry"
 16 | 	"github.com/ethereum/go-ethereum/common"
 17 | 	"github.com/ethereum/go-ethereum/log"
 18 | )
 19 | 
 20 | const (
 21 | 	liveFetchBlobMaximumRetries    = 10
 22 | 	startupFetchBlobMaximumRetries = 3
 23 | 	rearchiveMaximumRetries        = 3
 24 | 	backfillErrorRetryInterval     = 5 * time.Second
 25 | )
 26 | 
 27 | type BeaconClient interface {
 28 | 	client.BlobSidecarsProvider
 29 | 	client.BeaconBlockHeadersProvider
 30 | }
 31 | 
 32 | func NewArchiver(l log.Logger, cfg flags.ArchiverConfig, dataStoreClient storage.DataStore, client BeaconClient, m metrics.Metricer) (*Archiver, error) {
 33 | 	return &Archiver{
 34 | 		log:             l,
 35 | 		cfg:             cfg,
 36 | 		dataStoreClient: dataStoreClient,
 37 | 		metrics:         m,
 38 | 		beaconClient:    client,
 39 | 		stopCh:          make(chan struct{}),
 40 | 	}, nil
 41 | }
 42 | 
 43 | type Archiver struct {
 44 | 	log             log.Logger
 45 | 	cfg             flags.ArchiverConfig
 46 | 	dataStoreClient storage.DataStore
 47 | 	beaconClient    BeaconClient
 48 | 	metrics         metrics.Metricer
 49 | 	stopCh          chan struct{}
 50 | }
 51 | 
 52 | // Start starts archiving blobs. It begins polling the beacon node for the latest blocks and persisting blobs for
 53 | // them. Concurrently it'll also begin a backfill process (see backfillBlobs) to store all blobs from the current head
 54 | // to the previously stored blocks. This ensures that during restarts or outages of an archiver, any gaps will be
 55 | // filled in.
 56 | func (a *Archiver) Start(ctx context.Context) error {
 57 | 	currentBlock, _, err := retry.Do2(ctx, startupFetchBlobMaximumRetries, retry.Exponential(), func() (*v1.BeaconBlockHeader, bool, error) {
 58 | 		return a.persistBlobsForBlockToS3(ctx, "head", false)
 59 | 	})
 60 | 
 61 | 	if err != nil {
 62 | 		a.log.Error("failed to seed archiver with initial block", "err", err)
 63 | 		return err
 64 | 	}
 65 | 
 66 | 	go a.backfillBlobs(ctx, currentBlock)
 67 | 
 68 | 	return a.trackLatestBlocks(ctx)
 69 | }
 70 | 
 71 | // Stops the archiver service.
 72 | func (a *Archiver) Stop(ctx context.Context) error {
 73 | 	close(a.stopCh)
 74 | 	return nil
 75 | }
 76 | 
 77 | // persistBlobsForBlockToS3 fetches the blobs for a given block and persists them to S3. It returns the block header
 78 | // and a boolean indicating whether the blobs already existed in S3 and any errors that occur.
 79 | // If the blobs are already stored, it will not overwrite the data. Currently, the archiver does not
 80 | // perform any validation of the blobs, it assumes a trusted beacon node. See:
 81 | // https://github.com/base-org/blob-archiver/issues/4.
 82 | func (a *Archiver) persistBlobsForBlockToS3(ctx context.Context, blockIdentifier string, overwrite bool) (*v1.BeaconBlockHeader, bool, error) {
 83 | 	currentHeader, err := a.beaconClient.BeaconBlockHeader(ctx, &api.BeaconBlockHeaderOpts{
 84 | 		Block: blockIdentifier,
 85 | 	})
 86 | 
 87 | 	if err != nil {
 88 | 		a.log.Error("failed to fetch latest beacon block header", "err", err)
 89 | 		return nil, false, err
 90 | 	}
 91 | 
 92 | 	exists, err := a.dataStoreClient.Exists(ctx, common.Hash(currentHeader.Data.Root))
 93 | 	if err != nil {
 94 | 		a.log.Error("failed to check if blob exists", "err", err)
 95 | 		return nil, false, err
 96 | 	}
 97 | 
 98 | 	if exists && !overwrite {
 99 | 		a.log.Debug("blob already exists", "hash", currentHeader.Data.Root)
100 | 		return currentHeader.Data, true, nil
101 | 	}
102 | 
103 | 	blobSidecars, err := a.beaconClient.BlobSidecars(ctx, &api.BlobSidecarsOpts{
104 | 		Block: currentHeader.Data.Root.String(),
105 | 	})
106 | 
107 | 	if err != nil {
108 | 		a.log.Error("failed to fetch blob sidecars", "err", err)
109 | 		return nil, false, err
110 | 	}
111 | 
112 | 	a.log.Debug("fetched blob sidecars", "count", len(blobSidecars.Data))
113 | 
114 | 	blobData := storage.BlobData{
115 | 		Header: storage.Header{
116 | 			BeaconBlockHash: common.Hash(currentHeader.Data.Root),
117 | 		},
118 | 		BlobSidecars: storage.BlobSidecars{Data: blobSidecars.Data},
119 | 	}
120 | 
121 | 	// The blob that is being written has not been validated. It is assumed that the beacon node is trusted.
122 | 	err = a.dataStoreClient.Write(ctx, blobData)
123 | 
124 | 	if err != nil {
125 | 		a.log.Error("failed to write blob", "err", err)
126 | 		return nil, false, err
127 | 	}
128 | 
129 | 	a.metrics.RecordStoredBlobs(len(blobSidecars.Data))
130 | 
131 | 	return currentHeader.Data, exists, nil
132 | }
133 | 
134 | // backfillBlobs will persist all blobs from the provided beacon block header, to either the last block that was persisted
135 | // to the archivers storage or the origin block in the configuration. This is used to ensure that any gaps can be filled.
136 | // If an error is encountered persisting a block, it will retry after waiting for a period of time.
137 | func (a *Archiver) backfillBlobs(ctx context.Context, latest *v1.BeaconBlockHeader) {
138 | 	current, alreadyExists, err := latest, false, error(nil)
139 | 
140 | 	defer func() {
141 | 		a.log.Info("backfill complete", "endHash", current.Root.String(), "startHash", latest.Root.String())
142 | 	}()
143 | 
144 | 	for !alreadyExists {
145 | 		previous := current
146 | 
147 | 		if common.Hash(current.Root) == a.cfg.OriginBlock {
148 | 			a.log.Info("reached origin block", "hash", current.Root.String())
149 | 			return
150 | 		}
151 | 
152 | 		current, alreadyExists, err = a.persistBlobsForBlockToS3(ctx, previous.Header.Message.ParentRoot.String(), false)
153 | 		if err != nil {
154 | 			a.log.Error("failed to persist blobs for block, will retry", "err", err, "hash", previous.Header.Message.ParentRoot.String())
155 | 			// Revert back to block we failed to fetch
156 | 			current = previous
157 | 			time.Sleep(backfillErrorRetryInterval)
158 | 			continue
159 | 		}
160 | 
161 | 		if !alreadyExists {
162 | 			a.metrics.RecordProcessedBlock(metrics.BlockSourceBackfill)
163 | 		}
164 | 	}
165 | }
166 | 
167 | // trackLatestBlocks will poll the beacon node for the latest blocks and persist blobs for them.
168 | func (a *Archiver) trackLatestBlocks(ctx context.Context) error {
169 | 	t := time.NewTicker(a.cfg.PollInterval)
170 | 	defer t.Stop()
171 | 
172 | 	for {
173 | 		select {
174 | 		case <-ctx.Done():
175 | 			return nil
176 | 		case <-a.stopCh:
177 | 			return nil
178 | 		case <-t.C:
179 | 			a.processBlocksUntilKnownBlock(ctx)
180 | 		}
181 | 	}
182 | }
183 | 
184 | // processBlocksUntilKnownBlock will fetch and persist blobs for blocks until it finds a block that has been stored before.
185 | // In the case of a reorg, it will fetch the new head and then walk back the chain, storing all blobs until it finds a
186 | // known block -- that already exists in the archivers' storage.
187 | func (a *Archiver) processBlocksUntilKnownBlock(ctx context.Context) {
188 | 	a.log.Debug("refreshing live data")
189 | 
190 | 	var start *v1.BeaconBlockHeader
191 | 	currentBlockId := "head"
192 | 
193 | 	for {
194 | 		current, alreadyExisted, err := retry.Do2(ctx, liveFetchBlobMaximumRetries, retry.Exponential(), func() (*v1.BeaconBlockHeader, bool, error) {
195 | 			return a.persistBlobsForBlockToS3(ctx, currentBlockId, false)
196 | 		})
197 | 
198 | 		if err != nil {
199 | 			a.log.Error("failed to update live blobs for block", "err", err, "blockId", currentBlockId)
200 | 			return
201 | 		}
202 | 
203 | 		if start == nil {
204 | 			start = current
205 | 		}
206 | 
207 | 		if !alreadyExisted {
208 | 			a.metrics.RecordProcessedBlock(metrics.BlockSourceLive)
209 | 		} else {
210 | 			a.log.Debug("blob already exists", "hash", current.Root.String())
211 | 			break
212 | 		}
213 | 
214 | 		currentBlockId = current.Header.Message.ParentRoot.String()
215 | 	}
216 | 
217 | 	a.log.Info("live data refreshed", "startHash", start.Root.String(), "endHash", currentBlockId)
218 | }
219 | 
220 | // rearchiveRange will rearchive all blocks in the range from the given start to end. It returns the start and end of the
221 | // range that was successfully rearchived. On any persistent errors, it will halt archiving and return the range of blocks
222 | // that were rearchived and the error that halted the process.
223 | func (a *Archiver) rearchiveRange(from uint64, to uint64) (uint64, uint64, error) {
224 | 	for i := from; i <= to; i++ {
225 | 		id := strconv.FormatUint(i, 10)
226 | 
227 | 		l := a.log.New("slot", id)
228 | 
229 | 		l.Info("rearchiving block")
230 | 
231 | 		rewritten, err := retry.Do(context.Background(), rearchiveMaximumRetries, retry.Exponential(), func() (bool, error) {
232 | 			_, _, e := a.persistBlobsForBlockToS3(context.Background(), id, true)
233 | 
234 | 			// If the block is not found, we can assume that the slot has been skipped
235 | 			if e != nil {
236 | 				var apiErr *api.Error
237 | 				if errors.As(e, &apiErr) && apiErr.StatusCode == 404 {
238 | 					return false, nil
239 | 				}
240 | 
241 | 				return false, e
242 | 			}
243 | 
244 | 			return true, nil
245 | 		})
246 | 
247 | 		if err != nil {
248 | 			return from, i, err
249 | 		}
250 | 
251 | 		if !rewritten {
252 | 			l.Info("block not found during reachiving", "slot", id)
253 | 		}
254 | 
255 | 		a.metrics.RecordProcessedBlock(metrics.BlockSourceRearchive)
256 | 	}
257 | 
258 | 	return from, to, nil
259 | }
260 | 


--------------------------------------------------------------------------------
/archiver/service/archiver_test.go:
--------------------------------------------------------------------------------
  1 | package service
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"testing"
  6 | 	"time"
  7 | 
  8 | 	"github.com/base-org/blob-archiver/archiver/flags"
  9 | 	"github.com/base-org/blob-archiver/archiver/metrics"
 10 | 	"github.com/base-org/blob-archiver/common/beacon/beacontest"
 11 | 	"github.com/base-org/blob-archiver/common/blobtest"
 12 | 	"github.com/base-org/blob-archiver/common/storage"
 13 | 	"github.com/base-org/blob-archiver/common/storage/storagetest"
 14 | 	"github.com/ethereum-optimism/optimism/op-service/testlog"
 15 | 	"github.com/ethereum/go-ethereum/common"
 16 | 	"github.com/ethereum/go-ethereum/log"
 17 | 	"github.com/stretchr/testify/require"
 18 | )
 19 | 
 20 | func setup(t *testing.T, beacon *beacontest.StubBeaconClient) (*Archiver, *storagetest.TestFileStorage) {
 21 | 	l := testlog.Logger(t, log.LvlInfo)
 22 | 	fs := storagetest.NewTestFileStorage(t, l)
 23 | 	m := metrics.NewMetrics()
 24 | 
 25 | 	svc, err := NewArchiver(l, flags.ArchiverConfig{
 26 | 		PollInterval: 5 * time.Second,
 27 | 		OriginBlock:  blobtest.OriginBlock,
 28 | 	}, fs, beacon, m)
 29 | 	require.NoError(t, err)
 30 | 	return svc, fs
 31 | }
 32 | 
 33 | func TestArchiver_FetchAndPersist(t *testing.T) {
 34 | 	svc, fs := setup(t, beacontest.NewDefaultStubBeaconClient(t))
 35 | 
 36 | 	fs.CheckNotExistsOrFail(t, blobtest.OriginBlock)
 37 | 
 38 | 	header, alreadyExists, err := svc.persistBlobsForBlockToS3(context.Background(), blobtest.OriginBlock.String(), false)
 39 | 	require.False(t, alreadyExists)
 40 | 	require.NoError(t, err)
 41 | 	require.NotNil(t, header)
 42 | 	require.Equal(t, blobtest.OriginBlock.String(), common.Hash(header.Root).String())
 43 | 
 44 | 	fs.CheckExistsOrFail(t, blobtest.OriginBlock)
 45 | 
 46 | 	header, alreadyExists, err = svc.persistBlobsForBlockToS3(context.Background(), blobtest.OriginBlock.String(), false)
 47 | 	require.True(t, alreadyExists)
 48 | 	require.NoError(t, err)
 49 | 	require.NotNil(t, header)
 50 | 	require.Equal(t, blobtest.OriginBlock.String(), common.Hash(header.Root).String())
 51 | 
 52 | 	fs.CheckExistsOrFail(t, blobtest.OriginBlock)
 53 | }
 54 | 
 55 | func TestArchiver_FetchAndPersistOverwriting(t *testing.T) {
 56 | 	beacon := beacontest.NewDefaultStubBeaconClient(t)
 57 | 	svc, fs := setup(t, beacon)
 58 | 
 59 | 	// Blob 5 already exists
 60 | 	fs.WriteOrFail(t, storage.BlobData{
 61 | 		Header: storage.Header{
 62 | 			BeaconBlockHash: blobtest.Five,
 63 | 		},
 64 | 		BlobSidecars: storage.BlobSidecars{
 65 | 			Data: beacon.Blobs[blobtest.Five.String()],
 66 | 		},
 67 | 	})
 68 | 
 69 | 	require.Equal(t, fs.ReadOrFail(t, blobtest.Five).BlobSidecars.Data, beacon.Blobs[blobtest.Five.String()])
 70 | 
 71 | 	// change the blob data -- this isn't possible w/out changing the hash. But it allows us to test the overwrite
 72 | 	beacon.Blobs[blobtest.Five.String()] = blobtest.NewBlobSidecars(t, 6)
 73 | 
 74 | 	_, exists, err := svc.persistBlobsForBlockToS3(context.Background(), blobtest.Five.String(), true)
 75 | 	require.NoError(t, err)
 76 | 	require.True(t, exists)
 77 | 
 78 | 	// It should have overwritten the blob data
 79 | 	require.Equal(t, fs.ReadOrFail(t, blobtest.Five).BlobSidecars.Data, beacon.Blobs[blobtest.Five.String()])
 80 | 
 81 | 	// Overwriting a non-existent blob should return exists=false
 82 | 	_, exists, err = svc.persistBlobsForBlockToS3(context.Background(), blobtest.Four.String(), true)
 83 | 	require.NoError(t, err)
 84 | 	require.False(t, exists)
 85 | }
 86 | 
 87 | func TestArchiver_BackfillToOrigin(t *testing.T) {
 88 | 	beacon := beacontest.NewDefaultStubBeaconClient(t)
 89 | 	svc, fs := setup(t, beacon)
 90 | 
 91 | 	// We have the current head, which is block 5 written to storage
 92 | 	err := fs.Write(context.Background(), storage.BlobData{
 93 | 		Header: storage.Header{
 94 | 			BeaconBlockHash: blobtest.Five,
 95 | 		},
 96 | 		BlobSidecars: storage.BlobSidecars{
 97 | 			Data: beacon.Blobs[blobtest.Five.String()],
 98 | 		},
 99 | 	})
100 | 	require.NoError(t, err)
101 | 	// We expect to backfill all blocks to the origin
102 | 	expectedBlobs := []common.Hash{blobtest.Four, blobtest.Three, blobtest.Two, blobtest.One, blobtest.OriginBlock}
103 | 
104 | 	for _, blob := range expectedBlobs {
105 | 		fs.CheckNotExistsOrFail(t, blob)
106 | 	}
107 | 
108 | 	svc.backfillBlobs(context.Background(), beacon.Headers[blobtest.Five.String()])
109 | 
110 | 	for _, blob := range expectedBlobs {
111 | 		fs.CheckExistsOrFail(t, blob)
112 | 		data := fs.ReadOrFail(t, blob)
113 | 		require.Equal(t, data.BlobSidecars.Data, beacon.Blobs[blob.String()])
114 | 	}
115 | }
116 | 
117 | func TestArchiver_BackfillToExistingBlock(t *testing.T) {
118 | 	beacon := beacontest.NewDefaultStubBeaconClient(t)
119 | 	svc, fs := setup(t, beacon)
120 | 
121 | 	// We have the current head, which is block 5 written to storage
122 | 	err := fs.Write(context.Background(), storage.BlobData{
123 | 		Header: storage.Header{
124 | 			BeaconBlockHash: blobtest.Five,
125 | 		},
126 | 		BlobSidecars: storage.BlobSidecars{
127 | 			Data: beacon.Blobs[blobtest.Five.String()],
128 | 		},
129 | 	})
130 | 	require.NoError(t, err)
131 | 
132 | 	// We also have block 1 written to storage
133 | 	err = fs.Write(context.Background(), storage.BlobData{
134 | 		Header: storage.Header{
135 | 			BeaconBlockHash: blobtest.One,
136 | 		},
137 | 		BlobSidecars: storage.BlobSidecars{
138 | 			Data: beacon.Blobs[blobtest.One.String()],
139 | 		},
140 | 	})
141 | 	require.NoError(t, err)
142 | 
143 | 	// We expect to backfill all blobs between 5 and 1
144 | 	expectedBlobs := []common.Hash{blobtest.Four, blobtest.Three, blobtest.Two}
145 | 
146 | 	for _, blob := range expectedBlobs {
147 | 		exists, err := fs.Exists(context.Background(), blob)
148 | 		require.NoError(t, err)
149 | 		require.False(t, exists)
150 | 	}
151 | 
152 | 	svc.backfillBlobs(context.Background(), beacon.Headers[blobtest.Five.String()])
153 | 
154 | 	for _, blob := range expectedBlobs {
155 | 		exists, err := fs.Exists(context.Background(), blob)
156 | 		require.NoError(t, err)
157 | 		require.True(t, exists)
158 | 
159 | 		data, err := fs.Read(context.Background(), blob)
160 | 		require.NoError(t, err)
161 | 		require.NotNil(t, data)
162 | 		require.Equal(t, data.BlobSidecars.Data, beacon.Blobs[blob.String()])
163 | 	}
164 | }
165 | 
166 | func TestArchiver_LatestStopsAtExistingBlock(t *testing.T) {
167 | 	beacon := beacontest.NewDefaultStubBeaconClient(t)
168 | 	svc, fs := setup(t, beacon)
169 | 
170 | 	// 5 is the current head, if three already exists, we should write 5 and 4 and stop at three
171 | 	fs.WriteOrFail(t, storage.BlobData{
172 | 		Header: storage.Header{
173 | 			BeaconBlockHash: blobtest.Three,
174 | 		},
175 | 		BlobSidecars: storage.BlobSidecars{
176 | 			Data: beacon.Blobs[blobtest.Three.String()],
177 | 		},
178 | 	})
179 | 
180 | 	fs.CheckNotExistsOrFail(t, blobtest.Five)
181 | 	fs.CheckNotExistsOrFail(t, blobtest.Four)
182 | 	fs.CheckExistsOrFail(t, blobtest.Three)
183 | 
184 | 	svc.processBlocksUntilKnownBlock(context.Background())
185 | 
186 | 	fs.CheckExistsOrFail(t, blobtest.Five)
187 | 	five := fs.ReadOrFail(t, blobtest.Five)
188 | 	require.Equal(t, five.Header.BeaconBlockHash, blobtest.Five)
189 | 	require.Equal(t, five.BlobSidecars.Data, beacon.Blobs[blobtest.Five.String()])
190 | 
191 | 	fs.CheckExistsOrFail(t, blobtest.Four)
192 | 	four := fs.ReadOrFail(t, blobtest.Four)
193 | 	require.Equal(t, four.Header.BeaconBlockHash, blobtest.Four)
194 | 	require.Equal(t, five.BlobSidecars.Data, beacon.Blobs[blobtest.Five.String()])
195 | 
196 | 	fs.CheckExistsOrFail(t, blobtest.Three)
197 | 	three := fs.ReadOrFail(t, blobtest.Three)
198 | 	require.Equal(t, three.Header.BeaconBlockHash, blobtest.Three)
199 | 	require.Equal(t, five.BlobSidecars.Data, beacon.Blobs[blobtest.Five.String()])
200 | }
201 | 
202 | func TestArchiver_LatestNoNewData(t *testing.T) {
203 | 	beacon := beacontest.NewDefaultStubBeaconClient(t)
204 | 	svc, fs := setup(t, beacon)
205 | 
206 | 	// 5 is the current head, if 5 already exists, this should be a no-op
207 | 	fs.WriteOrFail(t, storage.BlobData{
208 | 		Header: storage.Header{
209 | 			BeaconBlockHash: common.Hash(beacon.Headers["head"].Root),
210 | 		},
211 | 		BlobSidecars: storage.BlobSidecars{
212 | 			Data: beacon.Blobs[blobtest.Three.String()],
213 | 		},
214 | 	})
215 | 
216 | 	fs.CheckExistsOrFail(t, blobtest.Five)
217 | 	fs.CheckNotExistsOrFail(t, blobtest.Four)
218 | 
219 | 	svc.processBlocksUntilKnownBlock(context.Background())
220 | 
221 | 	fs.CheckExistsOrFail(t, blobtest.Five)
222 | 	fs.CheckNotExistsOrFail(t, blobtest.Four)
223 | }
224 | 
225 | func TestArchiver_LatestConsumesNewBlocks(t *testing.T) {
226 | 	beacon := beacontest.NewDefaultStubBeaconClient(t)
227 | 	svc, fs := setup(t, beacon)
228 | 
229 | 	// set current head to 4, and write four
230 | 	beacon.Headers["head"] = beacon.Headers[blobtest.Four.String()]
231 | 	fs.WriteOrFail(t, storage.BlobData{
232 | 		Header: storage.Header{
233 | 			BeaconBlockHash: common.Hash(beacon.Headers[blobtest.Four.String()].Root),
234 | 		},
235 | 		BlobSidecars: storage.BlobSidecars{
236 | 			Data: beacon.Blobs[blobtest.Four.String()],
237 | 		},
238 | 	})
239 | 
240 | 	svc.processBlocksUntilKnownBlock(context.Background())
241 | 
242 | 	// No new data (5) is written and latest stops at known block (4), so 3 should not exist
243 | 	fs.CheckNotExistsOrFail(t, blobtest.Five)
244 | 	fs.CheckExistsOrFail(t, blobtest.Four)
245 | 	fs.CheckNotExistsOrFail(t, blobtest.Three)
246 | 
247 | 	// set current head to 5, and check it fetches new data
248 | 	beacon.Headers["head"] = beacon.Headers[blobtest.Five.String()]
249 | 
250 | 	svc.processBlocksUntilKnownBlock(context.Background())
251 | 	fs.CheckExistsOrFail(t, blobtest.Five)
252 | 	fs.CheckExistsOrFail(t, blobtest.Four)
253 | 	fs.CheckNotExistsOrFail(t, blobtest.Three)
254 | }
255 | 
256 | func TestArchiver_LatestStopsAtOrigin(t *testing.T) {
257 | 	beacon := beacontest.NewDefaultStubBeaconClient(t)
258 | 	svc, fs := setup(t, beacon)
259 | 
260 | 	// 5 is the current head, if origin already exists, we should stop at origin
261 | 	fs.WriteOrFail(t, storage.BlobData{
262 | 		Header: storage.Header{
263 | 			BeaconBlockHash: blobtest.OriginBlock,
264 | 		},
265 | 		BlobSidecars: storage.BlobSidecars{
266 | 			Data: beacon.Blobs[blobtest.OriginBlock.String()],
267 | 		},
268 | 	})
269 | 
270 | 	// Should write all blocks back to Origin
271 | 	toWrite := []common.Hash{blobtest.Five, blobtest.Four, blobtest.Three, blobtest.Two, blobtest.One}
272 | 	for _, hash := range toWrite {
273 | 		fs.CheckNotExistsOrFail(t, hash)
274 | 	}
275 | 
276 | 	svc.processBlocksUntilKnownBlock(context.Background())
277 | 
278 | 	for _, hash := range toWrite {
279 | 		fs.CheckExistsOrFail(t, hash)
280 | 		data := fs.ReadOrFail(t, hash)
281 | 		require.Equal(t, data.BlobSidecars.Data, beacon.Blobs[hash.String()])
282 | 	}
283 | }
284 | 
285 | func TestArchiver_LatestRetriesOnFailure(t *testing.T) {
286 | 	beacon := beacontest.NewDefaultStubBeaconClient(t)
287 | 	svc, fs := setup(t, beacon)
288 | 
289 | 	// 5 is the current head, if three already exists, we should write 5 and 4 and stop at three
290 | 	fs.WriteOrFail(t, storage.BlobData{
291 | 		Header: storage.Header{
292 | 			BeaconBlockHash: blobtest.Three,
293 | 		},
294 | 		BlobSidecars: storage.BlobSidecars{
295 | 			Data: beacon.Blobs[blobtest.Three.String()],
296 | 		},
297 | 	})
298 | 
299 | 	fs.CheckNotExistsOrFail(t, blobtest.Five)
300 | 	fs.CheckNotExistsOrFail(t, blobtest.Four)
301 | 	fs.CheckExistsOrFail(t, blobtest.Three)
302 | 
303 | 	// One failure is retried
304 | 	fs.WritesFailTimes(1)
305 | 	svc.processBlocksUntilKnownBlock(context.Background())
306 | 
307 | 	fs.CheckExistsOrFail(t, blobtest.Five)
308 | 	fs.CheckExistsOrFail(t, blobtest.Four)
309 | 	fs.CheckExistsOrFail(t, blobtest.Three)
310 | }
311 | 
312 | func TestArchiver_LatestHaltsOnPersistentError(t *testing.T) {
313 | 	beacon := beacontest.NewDefaultStubBeaconClient(t)
314 | 	svc, fs := setup(t, beacon)
315 | 
316 | 	// 5 is the current head, if three already exists, we should write 5 and 4 and stop at three
317 | 	fs.WriteOrFail(t, storage.BlobData{
318 | 		Header: storage.Header{
319 | 			BeaconBlockHash: blobtest.Three,
320 | 		},
321 | 		BlobSidecars: storage.BlobSidecars{
322 | 			Data: beacon.Blobs[blobtest.Three.String()],
323 | 		},
324 | 	})
325 | 
326 | 	fs.CheckNotExistsOrFail(t, blobtest.Five)
327 | 	fs.CheckNotExistsOrFail(t, blobtest.Four)
328 | 	fs.CheckExistsOrFail(t, blobtest.Three)
329 | 
330 | 	// Retries the maximum number of times, then fails and will not write the blobs
331 | 	fs.WritesFailTimes(liveFetchBlobMaximumRetries + 1)
332 | 	svc.processBlocksUntilKnownBlock(context.Background())
333 | 
334 | 	fs.CheckNotExistsOrFail(t, blobtest.Five)
335 | 	fs.CheckNotExistsOrFail(t, blobtest.Four)
336 | 	fs.CheckExistsOrFail(t, blobtest.Three)
337 | }
338 | 
339 | func TestArchiver_RearchiveRange(t *testing.T) {
340 | 	beacon := beacontest.NewDefaultStubBeaconClient(t)
341 | 	svc, fs := setup(t, beacon)
342 | 
343 | 	// 5 is the current head, if three already exists, we should write 5 and 4 and stop at three
344 | 	fs.WriteOrFail(t, storage.BlobData{
345 | 		Header: storage.Header{
346 | 			BeaconBlockHash: blobtest.Three,
347 | 		},
348 | 		BlobSidecars: storage.BlobSidecars{
349 | 			Data: beacon.Blobs[blobtest.Three.String()],
350 | 		},
351 | 	})
352 | 
353 | 	// startSlot+1 == One
354 | 	fs.CheckNotExistsOrFail(t, blobtest.One)
355 | 	fs.CheckNotExistsOrFail(t, blobtest.Two)
356 | 	fs.CheckExistsOrFail(t, blobtest.Three)
357 | 	fs.CheckNotExistsOrFail(t, blobtest.Four)
358 | 
359 | 	// this modifies the blobs at 3, purely to test the blob is rearchived
360 | 	beacon.Blobs[blobtest.Three.String()] = blobtest.NewBlobSidecars(t, 6)
361 | 
362 | 	from, to := blobtest.StartSlot+1, blobtest.StartSlot+4
363 | 
364 | 	actualFrom, actualTo, err := svc.rearchiveRange(from, to)
365 | 	// Should index the whole range
366 | 	require.NoError(t, err)
367 | 	require.Equal(t, from, actualFrom)
368 | 	require.Equal(t, to, actualTo)
369 | 
370 | 	// Should have written all the blobs
371 | 	fs.CheckExistsOrFail(t, blobtest.One)
372 | 	fs.CheckExistsOrFail(t, blobtest.Two)
373 | 	fs.CheckExistsOrFail(t, blobtest.Three)
374 | 	fs.CheckExistsOrFail(t, blobtest.Four)
375 | 
376 | 	// Should have overwritten any existing blobs
377 | 	require.Equal(t, fs.ReadOrFail(t, blobtest.Three).BlobSidecars.Data, beacon.Blobs[blobtest.Three.String()])
378 | }
379 | 


--------------------------------------------------------------------------------
/archiver/service/service.go:
--------------------------------------------------------------------------------
 1 | package service
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"errors"
 6 | 	"fmt"
 7 | 	"sync/atomic"
 8 | 
 9 | 	"github.com/base-org/blob-archiver/archiver/flags"
10 | 	"github.com/base-org/blob-archiver/archiver/metrics"
11 | 	"github.com/ethereum-optimism/optimism/op-service/httputil"
12 | 	opmetrics "github.com/ethereum-optimism/optimism/op-service/metrics"
13 | 	"github.com/ethereum/go-ethereum/log"
14 | )
15 | 
16 | var ErrAlreadyStopped = errors.New("already stopped")
17 | 
18 | func NewService(l log.Logger, cfg flags.ArchiverConfig, api *API, archiver *Archiver, m metrics.Metricer) (*ArchiverService, error) {
19 | 	return &ArchiverService{
20 | 		log:      l,
21 | 		cfg:      cfg,
22 | 		archiver: archiver,
23 | 		metrics:  m,
24 | 		api:      api,
25 | 	}, nil
26 | }
27 | 
28 | type ArchiverService struct {
29 | 	stopped       atomic.Bool
30 | 	log           log.Logger
31 | 	metricsServer *httputil.HTTPServer
32 | 	cfg           flags.ArchiverConfig
33 | 	metrics       metrics.Metricer
34 | 	api           *API
35 | 	archiver      *Archiver
36 | }
37 | 
38 | // Start starts the archiver service. It'll start the API's as well as the archiving process.
39 | func (a *ArchiverService) Start(ctx context.Context) error {
40 | 	if a.cfg.MetricsConfig.Enabled {
41 | 		a.log.Info("starting metrics server", "addr", a.cfg.MetricsConfig.ListenAddr, "port", a.cfg.MetricsConfig.ListenPort)
42 | 		srv, err := opmetrics.StartServer(a.metrics.Registry(), a.cfg.MetricsConfig.ListenAddr, a.cfg.MetricsConfig.ListenPort)
43 | 		if err != nil {
44 | 			return err
45 | 		}
46 | 
47 | 		a.log.Info("started metrics server", "addr", srv.Addr())
48 | 		a.metricsServer = srv
49 | 	}
50 | 
51 | 	srv, err := httputil.StartHTTPServer(a.cfg.ListenAddr, a.api.router)
52 | 	if err != nil {
53 | 		return fmt.Errorf("failed to start Archiver API server: %w", err)
54 | 	}
55 | 
56 | 	a.log.Info("Archiver API server started", "address", srv.Addr().String())
57 | 
58 | 	return a.archiver.Start(ctx)
59 | }
60 | 
61 | // Stops the archiver service.
62 | func (a *ArchiverService) Stop(ctx context.Context) error {
63 | 	if a.stopped.Load() {
64 | 		return ErrAlreadyStopped
65 | 	}
66 | 	a.log.Info("Stopping Archiver")
67 | 	a.stopped.Store(true)
68 | 
69 | 	if a.metricsServer != nil {
70 | 		if err := a.metricsServer.Stop(ctx); err != nil {
71 | 			return err
72 | 		}
73 | 	}
74 | 
75 | 	return a.archiver.Stop(ctx)
76 | }
77 | 
78 | func (a *ArchiverService) Stopped() bool {
79 | 	return a.stopped.Load()
80 | }
81 | 


--------------------------------------------------------------------------------
/common/beacon/beacontest/stub.go:
--------------------------------------------------------------------------------
  1 | package beacontest
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"fmt"
  6 | 	"strconv"
  7 | 	"testing"
  8 | 
  9 | 	"github.com/attestantio/go-eth2-client/api"
 10 | 	v1 "github.com/attestantio/go-eth2-client/api/v1"
 11 | 	"github.com/attestantio/go-eth2-client/spec/deneb"
 12 | 	"github.com/attestantio/go-eth2-client/spec/phase0"
 13 | 	"github.com/base-org/blob-archiver/common/blobtest"
 14 | 	"github.com/ethereum/go-ethereum/common"
 15 | )
 16 | 
 17 | type StubBeaconClient struct {
 18 | 	Headers map[string]*v1.BeaconBlockHeader
 19 | 	Blobs   map[string][]*deneb.BlobSidecar
 20 | }
 21 | 
 22 | func (s *StubBeaconClient) BeaconBlockHeader(ctx context.Context, opts *api.BeaconBlockHeaderOpts) (*api.Response[*v1.BeaconBlockHeader], error) {
 23 | 	header, found := s.Headers[opts.Block]
 24 | 	if !found {
 25 | 		return nil, fmt.Errorf("block not found")
 26 | 	}
 27 | 	return &api.Response[*v1.BeaconBlockHeader]{
 28 | 		Data: header,
 29 | 	}, nil
 30 | }
 31 | 
 32 | func (s *StubBeaconClient) BlobSidecars(ctx context.Context, opts *api.BlobSidecarsOpts) (*api.Response[[]*deneb.BlobSidecar], error) {
 33 | 	blobs, found := s.Blobs[opts.Block]
 34 | 	if !found {
 35 | 		return nil, fmt.Errorf("block not found")
 36 | 	}
 37 | 	return &api.Response[[]*deneb.BlobSidecar]{
 38 | 		Data: blobs,
 39 | 	}, nil
 40 | }
 41 | 
 42 | func NewEmptyStubBeaconClient() *StubBeaconClient {
 43 | 	return &StubBeaconClient{
 44 | 		Headers: make(map[string]*v1.BeaconBlockHeader),
 45 | 		Blobs:   make(map[string][]*deneb.BlobSidecar),
 46 | 	}
 47 | }
 48 | 
 49 | func NewDefaultStubBeaconClient(t *testing.T) *StubBeaconClient {
 50 | 	makeHeader := func(slot uint64, hash, parent common.Hash) *v1.BeaconBlockHeader {
 51 | 		return &v1.BeaconBlockHeader{
 52 | 			Root: phase0.Root(hash),
 53 | 			Header: &phase0.SignedBeaconBlockHeader{
 54 | 				Message: &phase0.BeaconBlockHeader{
 55 | 					Slot:       phase0.Slot(slot),
 56 | 					ParentRoot: phase0.Root(parent),
 57 | 				},
 58 | 			},
 59 | 		}
 60 | 	}
 61 | 
 62 | 	startSlot := blobtest.StartSlot
 63 | 
 64 | 	originBlobs := blobtest.NewBlobSidecars(t, 1)
 65 | 	oneBlobs := blobtest.NewBlobSidecars(t, 2)
 66 | 	twoBlobs := blobtest.NewBlobSidecars(t, 0)
 67 | 	threeBlobs := blobtest.NewBlobSidecars(t, 4)
 68 | 	fourBlobs := blobtest.NewBlobSidecars(t, 5)
 69 | 	fiveBlobs := blobtest.NewBlobSidecars(t, 6)
 70 | 
 71 | 	return &StubBeaconClient{
 72 | 		Headers: map[string]*v1.BeaconBlockHeader{
 73 | 			// Lookup by hash
 74 | 			blobtest.OriginBlock.String(): makeHeader(startSlot, blobtest.OriginBlock, common.Hash{9, 9, 9}),
 75 | 			blobtest.One.String():         makeHeader(startSlot+1, blobtest.One, blobtest.OriginBlock),
 76 | 			blobtest.Two.String():         makeHeader(startSlot+2, blobtest.Two, blobtest.One),
 77 | 			blobtest.Three.String():       makeHeader(startSlot+3, blobtest.Three, blobtest.Two),
 78 | 			blobtest.Four.String():        makeHeader(startSlot+4, blobtest.Four, blobtest.Three),
 79 | 			blobtest.Five.String():        makeHeader(startSlot+5, blobtest.Five, blobtest.Four),
 80 | 
 81 | 			// Lookup by identifier
 82 | 			"head":      makeHeader(startSlot+5, blobtest.Five, blobtest.Four),
 83 | 			"finalized": makeHeader(startSlot+3, blobtest.Three, blobtest.Two),
 84 | 
 85 | 			// Lookup by slot
 86 | 			strconv.FormatUint(startSlot, 10):   makeHeader(startSlot, blobtest.OriginBlock, common.Hash{9, 9, 9}),
 87 | 			strconv.FormatUint(startSlot+1, 10): makeHeader(startSlot+1, blobtest.One, blobtest.OriginBlock),
 88 | 			strconv.FormatUint(startSlot+2, 10): makeHeader(startSlot+2, blobtest.Two, blobtest.One),
 89 | 			strconv.FormatUint(startSlot+3, 10): makeHeader(startSlot+3, blobtest.Three, blobtest.Two),
 90 | 			strconv.FormatUint(startSlot+4, 10): makeHeader(startSlot+4, blobtest.Four, blobtest.Three),
 91 | 			strconv.FormatUint(startSlot+5, 10): makeHeader(startSlot+5, blobtest.Five, blobtest.Four),
 92 | 		},
 93 | 		Blobs: map[string][]*deneb.BlobSidecar{
 94 | 			// Lookup by hash
 95 | 			blobtest.OriginBlock.String(): originBlobs,
 96 | 			blobtest.One.String():         oneBlobs,
 97 | 			blobtest.Two.String():         twoBlobs,
 98 | 			blobtest.Three.String():       threeBlobs,
 99 | 			blobtest.Four.String():        fourBlobs,
100 | 			blobtest.Five.String():        fiveBlobs,
101 | 
102 | 			// Lookup by identifier
103 | 			"head":      fiveBlobs,
104 | 			"finalized": threeBlobs,
105 | 
106 | 			// Lookup by slot
107 | 			strconv.FormatUint(startSlot, 10):   originBlobs,
108 | 			strconv.FormatUint(startSlot+1, 10): oneBlobs,
109 | 			strconv.FormatUint(startSlot+2, 10): twoBlobs,
110 | 			strconv.FormatUint(startSlot+3, 10): threeBlobs,
111 | 			strconv.FormatUint(startSlot+4, 10): fourBlobs,
112 | 			strconv.FormatUint(startSlot+5, 10): fiveBlobs,
113 | 		},
114 | 	}
115 | }
116 | 


--------------------------------------------------------------------------------
/common/beacon/client.go:
--------------------------------------------------------------------------------
 1 | package beacon
 2 | 
 3 | import (
 4 | 	"context"
 5 | 
 6 | 	client "github.com/attestantio/go-eth2-client"
 7 | 	"github.com/attestantio/go-eth2-client/http"
 8 | 	"github.com/base-org/blob-archiver/common/flags"
 9 | 	"github.com/rs/zerolog"
10 | )
11 | 
12 | // Client is an interface that wraps the go-eth-2 interfaces that the blob archiver and api require.
13 | type Client interface {
14 | 	client.BeaconBlockHeadersProvider
15 | 	client.BlobSidecarsProvider
16 | }
17 | 
18 | // NewBeaconClient returns a new HTTP beacon client.
19 | func NewBeaconClient(ctx context.Context, cfg flags.BeaconConfig) (Client, error) {
20 | 	cctx, cancel := context.WithCancel(ctx)
21 | 	defer cancel()
22 | 
23 | 	c, err := http.New(cctx, http.WithAddress(cfg.BeaconURL), http.WithTimeout(cfg.BeaconClientTimeout), http.WithEnforceJSON(cfg.EnforceJSON), http.WithLogLevel(zerolog.ErrorLevel))
24 | 	if err != nil {
25 | 		return nil, err
26 | 	}
27 | 
28 | 	return c.(*http.Service), nil
29 | }
30 | 


--------------------------------------------------------------------------------
/common/blobtest/helpers.go:
--------------------------------------------------------------------------------
 1 | package blobtest
 2 | 
 3 | import (
 4 | 	"crypto/rand"
 5 | 	"testing"
 6 | 
 7 | 	"github.com/attestantio/go-eth2-client/spec/deneb"
 8 | 	"github.com/attestantio/go-eth2-client/spec/phase0"
 9 | 	"github.com/ethereum/go-ethereum/common"
10 | 	"github.com/stretchr/testify/require"
11 | )
12 | 
13 | var (
14 | 	OriginBlock = common.Hash{9, 9, 9, 9, 9}
15 | 	One         = common.Hash{1}
16 | 	Two         = common.Hash{2}
17 | 	Three       = common.Hash{3}
18 | 	Four        = common.Hash{4}
19 | 	Five        = common.Hash{5}
20 | 
21 | 	StartSlot = uint64(10)
22 | 	EndSlot   = uint64(15)
23 | )
24 | 
25 | func RandBytes(t *testing.T, size uint) []byte {
26 | 	randomBytes := make([]byte, size)
27 | 	_, err := rand.Read(randomBytes)
28 | 	require.NoError(t, err)
29 | 	return randomBytes
30 | }
31 | 
32 | func NewBlobSidecar(t *testing.T, i uint) *deneb.BlobSidecar {
33 | 	return &deneb.BlobSidecar{
34 | 		Index:         deneb.BlobIndex(i),
35 | 		Blob:          deneb.Blob(RandBytes(t, 131072)),
36 | 		KZGCommitment: deneb.KZGCommitment(RandBytes(t, 48)),
37 | 		KZGProof:      deneb.KZGProof(RandBytes(t, 48)),
38 | 		SignedBlockHeader: &phase0.SignedBeaconBlockHeader{
39 | 			Message: &phase0.BeaconBlockHeader{},
40 | 		},
41 | 	}
42 | }
43 | 
44 | func NewBlobSidecars(t *testing.T, count uint) []*deneb.BlobSidecar {
45 | 	result := make([]*deneb.BlobSidecar, count)
46 | 	for i := uint(0); i < count; i++ {
47 | 		result[i] = NewBlobSidecar(t, i)
48 | 	}
49 | 	return result
50 | }
51 | 


--------------------------------------------------------------------------------
/common/flags/config.go:
--------------------------------------------------------------------------------
  1 | package flags
  2 | 
  3 | import (
  4 | 	"errors"
  5 | 	"fmt"
  6 | 	"time"
  7 | 
  8 | 	"github.com/urfave/cli/v2"
  9 | )
 10 | 
 11 | type DataStorage string
 12 | type S3CredentialType string
 13 | 
 14 | const (
 15 | 	DataStorageUnknown  DataStorage      = "unknown"
 16 | 	DataStorageS3       DataStorage      = "s3"
 17 | 	DataStorageFile     DataStorage      = "file"
 18 | 	S3CredentialUnknown S3CredentialType = "unknown"
 19 | 	S3CredentialStatic  S3CredentialType = "static"
 20 | 	S3CredentialIAM     S3CredentialType = "iam"
 21 | )
 22 | 
 23 | type S3Config struct {
 24 | 	Endpoint string
 25 | 	UseHttps bool
 26 | 	Bucket   string
 27 | 
 28 | 	S3CredentialType S3CredentialType
 29 | 	AccessKey        string
 30 | 	SecretAccessKey  string
 31 | 	Compress         bool
 32 | }
 33 | 
 34 | func (c S3Config) check() error {
 35 | 	if c.Endpoint == "" {
 36 | 		return errors.New("s3 endpoint must be set")
 37 | 	}
 38 | 
 39 | 	if c.S3CredentialType == S3CredentialUnknown {
 40 | 		return errors.New("s3 credential type must be set")
 41 | 	}
 42 | 
 43 | 	if c.S3CredentialType == S3CredentialStatic {
 44 | 		if c.AccessKey == "" {
 45 | 			return errors.New("s3 access key must be set")
 46 | 		}
 47 | 
 48 | 		if c.SecretAccessKey == "" {
 49 | 			return errors.New("s3 secret access key must be set")
 50 | 		}
 51 | 	}
 52 | 
 53 | 	if c.Bucket == "" {
 54 | 		return errors.New("s3 bucket must be set")
 55 | 	}
 56 | 
 57 | 	return nil
 58 | }
 59 | 
 60 | type BeaconConfig struct {
 61 | 	BeaconURL           string
 62 | 	BeaconClientTimeout time.Duration
 63 | 	EnforceJSON         bool
 64 | }
 65 | 
 66 | type StorageConfig struct {
 67 | 	DataStorageType      DataStorage
 68 | 	S3Config             S3Config
 69 | 	FileStorageDirectory string
 70 | }
 71 | 
 72 | func NewBeaconConfig(cliCtx *cli.Context) BeaconConfig {
 73 | 	timeout, _ := time.ParseDuration(cliCtx.String(BeaconHttpClientTimeoutFlagName))
 74 | 
 75 | 	return BeaconConfig{
 76 | 		BeaconURL:           cliCtx.String(BeaconHttpFlagName),
 77 | 		BeaconClientTimeout: timeout,
 78 | 		EnforceJSON:         cliCtx.Bool(BeaconHttpEnforceJson),
 79 | 	}
 80 | }
 81 | 
 82 | func NewStorageConfig(cliCtx *cli.Context) StorageConfig {
 83 | 	return StorageConfig{
 84 | 		DataStorageType:      toDataStorage(cliCtx.String(DataStoreFlagName)),
 85 | 		S3Config:             readS3Config(cliCtx),
 86 | 		FileStorageDirectory: cliCtx.String(FileStorageDirectoryFlagName),
 87 | 	}
 88 | }
 89 | 
 90 | func toDataStorage(s string) DataStorage {
 91 | 	if s == string(DataStorageS3) {
 92 | 		return DataStorageS3
 93 | 	}
 94 | 
 95 | 	if s == string(DataStorageFile) {
 96 | 		return DataStorageFile
 97 | 	}
 98 | 
 99 | 	return DataStorageUnknown
100 | }
101 | 
102 | func readS3Config(ctx *cli.Context) S3Config {
103 | 	return S3Config{
104 | 		Endpoint:         ctx.String(S3EndpointFlagName),
105 | 		AccessKey:        ctx.String(S3AccessKeyFlagName),
106 | 		SecretAccessKey:  ctx.String(S3SecretAccessKeyFlagName),
107 | 		UseHttps:         ctx.Bool(S3EndpointHttpsFlagName),
108 | 		Bucket:           ctx.String(S3BucketFlagName),
109 | 		S3CredentialType: toS3CredentialType(ctx.String(S3CredentialTypeFlagName)),
110 | 		Compress:         ctx.Bool(S3CompressFlagName),
111 | 	}
112 | }
113 | 
114 | func toS3CredentialType(s string) S3CredentialType {
115 | 	if s == string(S3CredentialStatic) {
116 | 		return S3CredentialStatic
117 | 	} else if s == string(S3CredentialIAM) {
118 | 		return S3CredentialIAM
119 | 	}
120 | 	return S3CredentialUnknown
121 | }
122 | 
123 | func (c BeaconConfig) Check() error {
124 | 	if c.BeaconURL == "" {
125 | 		return errors.New("beacon url must be set")
126 | 	}
127 | 
128 | 	if c.BeaconClientTimeout == 0 {
129 | 		return errors.New("beacon client timeout must be set")
130 | 	}
131 | 
132 | 	return nil
133 | }
134 | 
135 | func (c StorageConfig) Check() error {
136 | 	if c.DataStorageType == DataStorageUnknown {
137 | 		return errors.New("unknown data-storage type")
138 | 	}
139 | 
140 | 	if c.DataStorageType == DataStorageS3 {
141 | 		if err := c.S3Config.check(); err != nil {
142 | 			return fmt.Errorf("s3 config check failed: %w", err)
143 | 		}
144 | 	} else if c.DataStorageType == DataStorageFile && c.FileStorageDirectory == "" {
145 | 		return errors.New("file storage directory must be set")
146 | 	}
147 | 
148 | 	return nil
149 | }
150 | 


--------------------------------------------------------------------------------
/common/flags/flags.go:
--------------------------------------------------------------------------------
  1 | package flags
  2 | 
  3 | import (
  4 | 	opservice "github.com/ethereum-optimism/optimism/op-service"
  5 | 	"github.com/urfave/cli/v2"
  6 | )
  7 | 
  8 | const (
  9 | 	BeaconHttpFlagName              = "l1-beacon-http"
 10 | 	BeaconHttpClientTimeoutFlagName = "l1-beacon-client-timeout"
 11 | 	BeaconHttpEnforceJson           = "l1-beacon-enforce-json"
 12 | 	DataStoreFlagName               = "data-store"
 13 | 	S3CredentialTypeFlagName        = "s3-credential-type"
 14 | 	S3EndpointFlagName              = "s3-endpoint"
 15 | 	S3EndpointHttpsFlagName         = "s3-endpoint-https"
 16 | 	S3CompressFlagName              = "s3-compress"
 17 | 	S3AccessKeyFlagName             = "s3-access-key"
 18 | 	S3SecretAccessKeyFlagName       = "s3-secret-access-key"
 19 | 	S3BucketFlagName                = "s3-bucket"
 20 | 	FileStorageDirectoryFlagName    = "file-directory"
 21 | )
 22 | 
 23 | func CLIFlags(envPrefix string) []cli.Flag {
 24 | 	return []cli.Flag{
 25 | 		// Required Flags
 26 | 		&cli.StringFlag{
 27 | 			Name:     BeaconHttpFlagName,
 28 | 			Usage:    "HTTP provider URL for L1 Beacon-node API",
 29 | 			Required: true,
 30 | 			EnvVars:  opservice.PrefixEnvVar(envPrefix, "L1_BEACON_HTTP"),
 31 | 		},
 32 | 		&cli.StringFlag{
 33 | 			Name:     DataStoreFlagName,
 34 | 			Usage:    "The type of data-store, options are [s3, file]",
 35 | 			Required: true,
 36 | 			EnvVars:  opservice.PrefixEnvVar(envPrefix, "DATA_STORE"),
 37 | 		},
 38 | 		// Optional Flags
 39 | 		// S3 Data Store Flags
 40 | 		&cli.StringFlag{
 41 | 			Name:    S3CredentialTypeFlagName,
 42 | 			Usage:   "The way to authenticate to S3, options are [iam, static]",
 43 | 			EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_CREDENTIAL_TYPE"),
 44 | 		},
 45 | 		&cli.StringFlag{
 46 | 			Name:    S3EndpointFlagName,
 47 | 			Usage:   "The URL for the S3 bucket (without the scheme http or https specified)",
 48 | 			EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_ENDPOINT"),
 49 | 		},
 50 | 		&cli.BoolFlag{
 51 | 			Name:    S3EndpointHttpsFlagName,
 52 | 			Usage:   "Whether to use https for the S3 bucket",
 53 | 			Value:   true,
 54 | 			EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_ENDPOINT_HTTPS"),
 55 | 		},
 56 | 		&cli.BoolFlag{
 57 | 			Name:    S3CompressFlagName,
 58 | 			Usage:   "Whether to compress data before storing in S3",
 59 | 			Value:   false,
 60 | 			EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_COMPRESS"),
 61 | 		},
 62 | 		&cli.StringFlag{
 63 | 			Name:    S3AccessKeyFlagName,
 64 | 			Usage:   "The S3 access key for the bucket",
 65 | 			Hidden:  true,
 66 | 			EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_ACCESS_KEY"),
 67 | 		},
 68 | 		&cli.StringFlag{
 69 | 			Name:    S3SecretAccessKeyFlagName,
 70 | 			Usage:   "The S3 secret access key for the bucket",
 71 | 			Hidden:  true,
 72 | 			EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_SECRET_ACCESS_KEY"),
 73 | 		},
 74 | 		&cli.StringFlag{
 75 | 			Name:    S3BucketFlagName,
 76 | 			Usage:   "The bucket to use",
 77 | 			Hidden:  true,
 78 | 			EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_BUCKET"),
 79 | 		},
 80 | 		// File Data Store Flags
 81 | 		&cli.StringFlag{
 82 | 			Name:    FileStorageDirectoryFlagName,
 83 | 			Usage:   "The path to the directory to use for storing blobs on the file system",
 84 | 			EnvVars: opservice.PrefixEnvVar(envPrefix, "FILE_DIRECTORY"),
 85 | 		},
 86 | 		// Beacon Client Settings
 87 | 		&cli.StringFlag{
 88 | 			Name:    BeaconHttpClientTimeoutFlagName,
 89 | 			Usage:   "The timeout duration for the beacon client",
 90 | 			Value:   "10s",
 91 | 			EnvVars: opservice.PrefixEnvVar(envPrefix, "L1_BEACON_CLIENT_TIMEOUT"),
 92 | 		},
 93 | 		&cli.BoolFlag{
 94 | 			Name:    BeaconHttpEnforceJson,
 95 | 			Usage:   "When true uses json for all requests/responses to the beacon node",
 96 | 			Value:   false,
 97 | 			EnvVars: opservice.PrefixEnvVar(envPrefix, "L1_BEACON_CLIENT_ENFORCE_JSON"),
 98 | 		},
 99 | 	}
100 | }
101 | 


--------------------------------------------------------------------------------
/common/storage/file.go:
--------------------------------------------------------------------------------
 1 | package storage
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"encoding/json"
 6 | 	"os"
 7 | 	"path"
 8 | 
 9 | 	"github.com/ethereum/go-ethereum/common"
10 | 	"github.com/ethereum/go-ethereum/log"
11 | )
12 | 
13 | type FileStorage struct {
14 | 	log       log.Logger
15 | 	directory string
16 | }
17 | 
18 | func NewFileStorage(dir string, l log.Logger) *FileStorage {
19 | 	return &FileStorage{
20 | 		log:       l,
21 | 		directory: dir,
22 | 	}
23 | }
24 | 
25 | func (s *FileStorage) Exists(_ context.Context, hash common.Hash) (bool, error) {
26 | 	_, err := os.Stat(s.fileName(hash))
27 | 	if err != nil {
28 | 		if os.IsNotExist(err) {
29 | 			return false, nil
30 | 		}
31 | 		return false, err
32 | 	}
33 | 	return true, nil
34 | }
35 | 
36 | func (s *FileStorage) Read(_ context.Context, hash common.Hash) (BlobData, error) {
37 | 	data, err := os.ReadFile(s.fileName(hash))
38 | 	if err != nil {
39 | 		if os.IsNotExist(err) {
40 | 			return BlobData{}, ErrNotFound
41 | 		}
42 | 
43 | 		return BlobData{}, err
44 | 	}
45 | 	var result BlobData
46 | 	err = json.Unmarshal(data, &result)
47 | 	if err != nil {
48 | 		s.log.Warn("error decoding blob", "err", err, "hash", hash.String())
49 | 		return BlobData{}, ErrMarshaling
50 | 	}
51 | 	return result, nil
52 | }
53 | 
54 | func (s *FileStorage) Write(_ context.Context, data BlobData) error {
55 | 	b, err := json.Marshal(data)
56 | 	if err != nil {
57 | 		s.log.Warn("error encoding blob", "err", err)
58 | 		return ErrMarshaling
59 | 	}
60 | 	err = os.WriteFile(s.fileName(data.Header.BeaconBlockHash), b, 0644)
61 | 	if err != nil {
62 | 		s.log.Warn("error writing blob", "err", err)
63 | 		return err
64 | 	}
65 | 
66 | 	s.log.Info("wrote blob", "hash", data.Header.BeaconBlockHash.String())
67 | 	return nil
68 | }
69 | 
70 | func (s *FileStorage) fileName(hash common.Hash) string {
71 | 	return path.Join(s.directory, hash.String())
72 | }
73 | 


--------------------------------------------------------------------------------
/common/storage/file_test.go:
--------------------------------------------------------------------------------
  1 | package storage
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"errors"
  6 | 	"os"
  7 | 	"testing"
  8 | 
  9 | 	"github.com/ethereum-optimism/optimism/op-service/testlog"
 10 | 	"github.com/ethereum/go-ethereum/common"
 11 | 	"github.com/ethereum/go-ethereum/log"
 12 | 	"github.com/stretchr/testify/require"
 13 | )
 14 | 
 15 | func setup(t *testing.T) (*FileStorage, func()) {
 16 | 	logger := testlog.Logger(t, log.LvlInfo)
 17 | 	tempDir, err := os.MkdirTemp("", "test")
 18 | 	require.NoError(t, err)
 19 | 	fs := NewFileStorage(tempDir, logger)
 20 | 	return fs, func() {
 21 | 		require.NoError(t, os.RemoveAll(tempDir))
 22 | 	}
 23 | }
 24 | 
 25 | func runTestExists(t *testing.T, s DataStore) {
 26 | 	id := common.Hash{1, 2, 3}
 27 | 
 28 | 	exists, err := s.Exists(context.Background(), id)
 29 | 	require.NoError(t, err)
 30 | 	require.False(t, exists)
 31 | 
 32 | 	err = s.Write(context.Background(), BlobData{
 33 | 		Header: Header{
 34 | 			BeaconBlockHash: id,
 35 | 		},
 36 | 		BlobSidecars: BlobSidecars{},
 37 | 	})
 38 | 	require.NoError(t, err)
 39 | 
 40 | 	exists, err = s.Exists(context.Background(), id)
 41 | 	require.NoError(t, err)
 42 | 	require.True(t, exists)
 43 | }
 44 | 
 45 | func TestExists(t *testing.T) {
 46 | 	fs, cleanup := setup(t)
 47 | 	defer cleanup()
 48 | 
 49 | 	runTestExists(t, fs)
 50 | }
 51 | 
 52 | func runTestRead(t *testing.T, s DataStore) {
 53 | 	id := common.Hash{1, 2, 3}
 54 | 
 55 | 	_, err := s.Read(context.Background(), id)
 56 | 	require.Error(t, err)
 57 | 	require.True(t, errors.Is(err, ErrNotFound))
 58 | 
 59 | 	err = s.Write(context.Background(), BlobData{
 60 | 		Header: Header{
 61 | 			BeaconBlockHash: id,
 62 | 		},
 63 | 		BlobSidecars: BlobSidecars{},
 64 | 	})
 65 | 	require.NoError(t, err)
 66 | 
 67 | 	data, err := s.Read(context.Background(), id)
 68 | 	require.NoError(t, err)
 69 | 	require.Equal(t, id, data.Header.BeaconBlockHash)
 70 | }
 71 | 
 72 | func TestRead(t *testing.T) {
 73 | 	fs, cleanup := setup(t)
 74 | 	defer cleanup()
 75 | 
 76 | 	runTestRead(t, fs)
 77 | }
 78 | 
 79 | func TestBrokenStorage(t *testing.T) {
 80 | 	fs, cleanup := setup(t)
 81 | 
 82 | 	id := common.Hash{1, 2, 3}
 83 | 
 84 | 	// Delete the directory to simulate broken storage
 85 | 	cleanup()
 86 | 
 87 | 	_, err := fs.Read(context.Background(), id)
 88 | 	require.Error(t, err)
 89 | 
 90 | 	exists, err := fs.Exists(context.Background(), id)
 91 | 	require.False(t, exists)
 92 | 	require.NoError(t, err) // No error should be returned, as in this test we've just delted the directory
 93 | 
 94 | 	err = fs.Write(context.Background(), BlobData{
 95 | 		Header: Header{
 96 | 			BeaconBlockHash: id,
 97 | 		},
 98 | 		BlobSidecars: BlobSidecars{},
 99 | 	})
100 | 	require.Error(t, err)
101 | }
102 | 
103 | func TestReadInvalidData(t *testing.T) {
104 | 	fs, cleanup := setup(t)
105 | 	defer cleanup()
106 | 
107 | 	id := common.Hash{1, 2, 3}
108 | 
109 | 	err := os.WriteFile(fs.fileName(id), []byte("invalid json"), 0644)
110 | 	require.NoError(t, err)
111 | 
112 | 	_, err = fs.Read(context.Background(), id)
113 | 	require.Error(t, err)
114 | 	require.True(t, errors.Is(err, ErrMarshaling))
115 | }
116 | 


--------------------------------------------------------------------------------
/common/storage/s3.go:
--------------------------------------------------------------------------------
  1 | package storage
  2 | 
  3 | import (
  4 | 	"bytes"
  5 | 	"compress/gzip"
  6 | 	"context"
  7 | 	"encoding/json"
  8 | 	"io"
  9 | 
 10 | 	"github.com/base-org/blob-archiver/common/flags"
 11 | 	"github.com/ethereum/go-ethereum/common"
 12 | 	"github.com/ethereum/go-ethereum/log"
 13 | 	"github.com/minio/minio-go/v7"
 14 | 	"github.com/minio/minio-go/v7/pkg/credentials"
 15 | )
 16 | 
 17 | type S3Storage struct {
 18 | 	s3       *minio.Client
 19 | 	bucket   string
 20 | 	log      log.Logger
 21 | 	compress bool
 22 | }
 23 | 
 24 | func NewS3Storage(cfg flags.S3Config, l log.Logger) (*S3Storage, error) {
 25 | 	var c *credentials.Credentials
 26 | 	if cfg.S3CredentialType == flags.S3CredentialStatic {
 27 | 		c = credentials.NewStaticV4(cfg.AccessKey, cfg.SecretAccessKey, "")
 28 | 	} else {
 29 | 		c = credentials.NewIAM("")
 30 | 	}
 31 | 
 32 | 	client, err := minio.New(cfg.Endpoint, &minio.Options{
 33 | 		Creds:  c,
 34 | 		Secure: cfg.UseHttps,
 35 | 	})
 36 | 
 37 | 	if err != nil {
 38 | 		return nil, err
 39 | 	}
 40 | 
 41 | 	return &S3Storage{
 42 | 		s3:       client,
 43 | 		bucket:   cfg.Bucket,
 44 | 		log:      l,
 45 | 		compress: cfg.Compress,
 46 | 	}, nil
 47 | }
 48 | 
 49 | func (s *S3Storage) Exists(ctx context.Context, hash common.Hash) (bool, error) {
 50 | 	_, err := s.s3.StatObject(ctx, s.bucket, hash.String(), minio.StatObjectOptions{})
 51 | 	if err != nil {
 52 | 		errResponse := minio.ToErrorResponse(err)
 53 | 		if errResponse.Code == "NoSuchKey" {
 54 | 			return false, nil
 55 | 		} else {
 56 | 			return false, err
 57 | 		}
 58 | 	}
 59 | 
 60 | 	return true, nil
 61 | }
 62 | 
 63 | func (s *S3Storage) Read(ctx context.Context, hash common.Hash) (BlobData, error) {
 64 | 	res, err := s.s3.GetObject(ctx, s.bucket, hash.String(), minio.GetObjectOptions{})
 65 | 	if err != nil {
 66 | 		s.log.Info("unexpected error fetching blob", "hash", hash.String(), "err", err)
 67 | 		return BlobData{}, ErrStorage
 68 | 	}
 69 | 	defer res.Close()
 70 | 	stat, err := res.Stat()
 71 | 	if err != nil {
 72 | 		errResponse := minio.ToErrorResponse(err)
 73 | 		if errResponse.Code == "NoSuchKey" {
 74 | 			s.log.Info("unable to find blob", "hash", hash.String())
 75 | 			return BlobData{}, ErrNotFound
 76 | 		} else {
 77 | 			s.log.Info("unexpected error fetching blob", "hash", hash.String(), "err", err)
 78 | 			return BlobData{}, ErrStorage
 79 | 		}
 80 | 	}
 81 | 
 82 | 	var reader io.ReadCloser = res
 83 | 	defer reader.Close()
 84 | 
 85 | 	if stat.Metadata.Get("Content-Encoding") == "gzip" {
 86 | 		reader, err = gzip.NewReader(reader)
 87 | 		if err != nil {
 88 | 			s.log.Warn("error creating gzip reader", "hash", hash.String(), "err", err)
 89 | 			return BlobData{}, ErrMarshaling
 90 | 		}
 91 | 	}
 92 | 
 93 | 	var data BlobData
 94 | 	err = json.NewDecoder(reader).Decode(&data)
 95 | 	if err != nil {
 96 | 		s.log.Warn("error decoding blob", "hash", hash.String(), "err", err)
 97 | 		return BlobData{}, ErrMarshaling
 98 | 	}
 99 | 
100 | 	return data, nil
101 | }
102 | 
103 | func (s *S3Storage) Write(ctx context.Context, data BlobData) error {
104 | 	b, err := json.Marshal(data)
105 | 	if err != nil {
106 | 		s.log.Warn("error encoding blob", "err", err)
107 | 		return ErrMarshaling
108 | 	}
109 | 
110 | 	options := minio.PutObjectOptions{
111 | 		ContentType: "application/json",
112 | 	}
113 | 
114 | 	if s.compress {
115 | 		b, err = compress(b)
116 | 		if err != nil {
117 | 			s.log.Warn("error compressing blob", "err", err)
118 | 			return ErrCompress
119 | 		}
120 | 		options.ContentEncoding = "gzip"
121 | 	}
122 | 
123 | 	reader := bytes.NewReader(b)
124 | 
125 | 	_, err = s.s3.PutObject(ctx, s.bucket, data.Header.BeaconBlockHash.String(), reader, int64(len(b)), options)
126 | 
127 | 	if err != nil {
128 | 		s.log.Warn("error writing blob", "err", err)
129 | 		return ErrStorage
130 | 	}
131 | 
132 | 	s.log.Info("wrote blob", "hash", data.Header.BeaconBlockHash.String())
133 | 	return nil
134 | }
135 | 
136 | func compress(in []byte) ([]byte, error) {
137 | 	var buf bytes.Buffer
138 | 	gz := gzip.NewWriter(&buf)
139 | 	_, err := gz.Write(in)
140 | 	if err != nil {
141 | 		return nil, err
142 | 	}
143 | 	err = gz.Close()
144 | 	if err != nil {
145 | 		return nil, err
146 | 	}
147 | 	return buf.Bytes(), nil
148 | }
149 | 


--------------------------------------------------------------------------------
/common/storage/s3_test.go:
--------------------------------------------------------------------------------
 1 | package storage
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"os"
 6 | 	"testing"
 7 | 
 8 | 	"github.com/base-org/blob-archiver/common/flags"
 9 | 	"github.com/ethereum-optimism/optimism/op-service/testlog"
10 | 	"github.com/ethereum/go-ethereum/log"
11 | 	"github.com/minio/minio-go/v7"
12 | 	"github.com/stretchr/testify/require"
13 | )
14 | 
15 | // Prior to running these tests, a local Minio server must be running.
16 | // You can accomplish this with:
17 | // docker-compose down # shut down any running services
18 | // docker-compose up minio create-buckets # start the minio service
19 | func setupS3(t *testing.T) *S3Storage {
20 | 	if os.Getenv("RUN_INTEGRATION_TESTS") == "" {
21 | 		t.Skip("skipping integration tests: set RUN_INTEGRATION_TESTS environment variable")
22 | 	}
23 | 
24 | 	l := testlog.Logger(t, log.LvlInfo)
25 | 
26 | 	s3, err := NewS3Storage(flags.S3Config{
27 | 		Endpoint:         "localhost:9000",
28 | 		AccessKey:        "admin",
29 | 		SecretAccessKey:  "password",
30 | 		UseHttps:         false,
31 | 		Bucket:           "blobs",
32 | 		S3CredentialType: flags.S3CredentialStatic,
33 | 	}, l)
34 | 
35 | 	require.NoError(t, err)
36 | 
37 | 	for object := range s3.s3.ListObjects(context.Background(), "blobs", minio.ListObjectsOptions{}) {
38 | 		err = s3.s3.RemoveObject(context.Background(), "blobs", object.Key, minio.RemoveObjectOptions{})
39 | 		require.NoError(t, err)
40 | 	}
41 | 
42 | 	require.NoError(t, err)
43 | 	return s3
44 | }
45 | 
46 | func TestS3Exists(t *testing.T) {
47 | 	s3 := setupS3(t)
48 | 
49 | 	runTestExists(t, s3)
50 | }
51 | 
52 | func TestS3Read(t *testing.T) {
53 | 	s3 := setupS3(t)
54 | 
55 | 	runTestRead(t, s3)
56 | }
57 | 


--------------------------------------------------------------------------------
/common/storage/storage.go:
--------------------------------------------------------------------------------
  1 | package storage
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"errors"
  6 | 
  7 | 	"github.com/attestantio/go-eth2-client/spec/deneb"
  8 | 	"github.com/base-org/blob-archiver/common/flags"
  9 | 	"github.com/ethereum/go-ethereum/common"
 10 | 	"github.com/ethereum/go-ethereum/log"
 11 | )
 12 | 
 13 | const (
 14 | 	blobSidecarSize = 131928
 15 | )
 16 | 
 17 | var (
 18 | 	// ErrNotFound is returned when a blob is not found in the storage.
 19 | 	ErrNotFound = errors.New("blob not found")
 20 | 	// ErrStorage is returned when there is an error accessing the storage.
 21 | 	ErrStorage = errors.New("error accessing storage")
 22 | 	// ErrMarshaling is returned when there is an error in (un)marshaling the blob
 23 | 	ErrMarshaling = errors.New("error encoding/decoding blob")
 24 | 	// ErrCompress is returned when there is an error gzipping the data
 25 | 	ErrCompress = errors.New("error compressing blob")
 26 | )
 27 | 
 28 | type Header struct {
 29 | 	BeaconBlockHash common.Hash `json:"beacon_block_hash"`
 30 | }
 31 | 
 32 | type BlobSidecars struct {
 33 | 	Data []*deneb.BlobSidecar `json:"data"`
 34 | }
 35 | 
 36 | // MarshalSSZ marshals the blob sidecars into SSZ. As the blob sidecars are a single list of fixed size elements, we can
 37 | // simply concatenate the marshaled sidecars together.
 38 | func (b *BlobSidecars) MarshalSSZ() ([]byte, error) {
 39 | 	result := make([]byte, b.SizeSSZ())
 40 | 
 41 | 	for i, sidecar := range b.Data {
 42 | 		sidecarBytes, err := sidecar.MarshalSSZ()
 43 | 		if err != nil {
 44 | 			return nil, err
 45 | 		}
 46 | 
 47 | 		from := i * len(sidecarBytes)
 48 | 		to := (i + 1) * len(sidecarBytes)
 49 | 
 50 | 		copy(result[from:to], sidecarBytes)
 51 | 	}
 52 | 
 53 | 	return result, nil
 54 | }
 55 | 
 56 | func (b *BlobSidecars) SizeSSZ() int {
 57 | 	return len(b.Data) * blobSidecarSize
 58 | }
 59 | 
 60 | type BlobData struct {
 61 | 	Header       Header       `json:"header"`
 62 | 	BlobSidecars BlobSidecars `json:"blob_sidecars"`
 63 | }
 64 | 
 65 | // DataStoreReader is the interface for reading from a data store.
 66 | type DataStoreReader interface {
 67 | 	// Exists returns true if the given blob hash exists in the data store, false otherwise.
 68 | 	// It should return one of the following:
 69 | 	// - nil: the existence check was successful. In this case the boolean should also be set correctly.
 70 | 	// - ErrStorage: there was an error accessing the data store.
 71 | 	Exists(ctx context.Context, hash common.Hash) (bool, error)
 72 | 	// Read reads the blob data for the given beacon block hash from the data store.
 73 | 	// It should return one of the following:
 74 | 	// - nil: reading the blob was successful. The blob data is also returned.
 75 | 	// - ErrNotFound: the blob data was not found in the data store.
 76 | 	// - ErrStorage: there was an error accessing the data store.
 77 | 	// - ErrMarshaling: there was an error decoding the blob data.
 78 | 	Read(ctx context.Context, hash common.Hash) (BlobData, error)
 79 | }
 80 | 
 81 | // DataStoreWriter is the interface for writing to a data store.
 82 | type DataStoreWriter interface {
 83 | 	// Write writes the given blob data to the data store. It should return one of the following errors:
 84 | 	// - nil: writing the blob was successful.
 85 | 	// - ErrStorage: there was an error accessing the data store.
 86 | 	// - ErrMarshaling: there was an error encoding the blob data.
 87 | 	Write(ctx context.Context, data BlobData) error
 88 | }
 89 | 
 90 | // DataStore is the interface for a data store that can be both written to and read from.
 91 | type DataStore interface {
 92 | 	DataStoreReader
 93 | 	DataStoreWriter
 94 | }
 95 | 
 96 | func NewStorage(cfg flags.StorageConfig, l log.Logger) (DataStore, error) {
 97 | 	if cfg.DataStorageType == flags.DataStorageS3 {
 98 | 		return NewS3Storage(cfg.S3Config, l)
 99 | 	} else {
100 | 		return NewFileStorage(cfg.FileStorageDirectory, l), nil
101 | 	}
102 | }
103 | 


--------------------------------------------------------------------------------
/common/storage/storage_test.go:
--------------------------------------------------------------------------------
 1 | package storage
 2 | 
 3 | import (
 4 | 	"testing"
 5 | 
 6 | 	"github.com/attestantio/go-eth2-client/api"
 7 | 	"github.com/attestantio/go-eth2-client/spec/deneb"
 8 | 	"github.com/base-org/blob-archiver/common/blobtest"
 9 | 	"github.com/stretchr/testify/require"
10 | )
11 | 
12 | func TestMarshalSSZ(t *testing.T) {
13 | 	b := &BlobSidecars{
14 | 		Data: []*deneb.BlobSidecar{
15 | 			{
16 | 				Index:         1,
17 | 				Blob:          deneb.Blob(blobtest.RandBytes(t, 131072)),
18 | 				KZGCommitment: deneb.KZGCommitment(blobtest.RandBytes(t, 48)),
19 | 				KZGProof:      deneb.KZGProof(blobtest.RandBytes(t, 48)),
20 | 			},
21 | 			{
22 | 				Index:         2,
23 | 				Blob:          deneb.Blob(blobtest.RandBytes(t, 131072)),
24 | 				KZGCommitment: deneb.KZGCommitment(blobtest.RandBytes(t, 48)),
25 | 				KZGProof:      deneb.KZGProof(blobtest.RandBytes(t, 48)),
26 | 			},
27 | 		},
28 | 	}
29 | 
30 | 	data, err := b.MarshalSSZ()
31 | 	require.NoError(t, err)
32 | 
33 | 	sidecars := api.BlobSidecars{}
34 | 	err = sidecars.UnmarshalSSZ(data)
35 | 	require.NoError(t, err)
36 | 
37 | 	require.Equal(t, len(b.Data), len(sidecars.Sidecars))
38 | 	for i := range b.Data {
39 | 		require.Equal(t, b.Data[i].Index, sidecars.Sidecars[i].Index)
40 | 		require.Equal(t, b.Data[i].Blob, sidecars.Sidecars[i].Blob)
41 | 		require.Equal(t, b.Data[i].KZGCommitment, sidecars.Sidecars[i].KZGCommitment)
42 | 		require.Equal(t, b.Data[i].KZGProof, sidecars.Sidecars[i].KZGProof)
43 | 	}
44 | }
45 | 


--------------------------------------------------------------------------------
/common/storage/storagetest/stub.go:
--------------------------------------------------------------------------------
 1 | package storagetest
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"testing"
 6 | 
 7 | 	"github.com/base-org/blob-archiver/common/storage"
 8 | 	"github.com/ethereum/go-ethereum/common"
 9 | 	"github.com/ethereum/go-ethereum/log"
10 | 	"github.com/stretchr/testify/require"
11 | )
12 | 
13 | type TestFileStorage struct {
14 | 	*storage.FileStorage
15 | 	writeFailCount int
16 | }
17 | 
18 | func NewTestFileStorage(t *testing.T, l log.Logger) *TestFileStorage {
19 | 	dir := t.TempDir()
20 | 	return &TestFileStorage{
21 | 		FileStorage: storage.NewFileStorage(dir, l),
22 | 	}
23 | }
24 | 
25 | func (s *TestFileStorage) WritesFailTimes(times int) {
26 | 	s.writeFailCount = times
27 | }
28 | 
29 | func (s *TestFileStorage) Write(_ context.Context, data storage.BlobData) error {
30 | 	if s.writeFailCount > 0 {
31 | 		s.writeFailCount--
32 | 		return storage.ErrStorage
33 | 	}
34 | 
35 | 	return s.FileStorage.Write(context.Background(), data)
36 | }
37 | 
38 | func (fs *TestFileStorage) CheckExistsOrFail(t *testing.T, hash common.Hash) {
39 | 	exists, err := fs.Exists(context.Background(), hash)
40 | 	require.NoError(t, err)
41 | 	require.True(t, exists)
42 | }
43 | 
44 | func (fs *TestFileStorage) CheckNotExistsOrFail(t *testing.T, hash common.Hash) {
45 | 	exists, err := fs.Exists(context.Background(), hash)
46 | 	require.NoError(t, err)
47 | 	require.False(t, exists)
48 | }
49 | 
50 | func (fs *TestFileStorage) WriteOrFail(t *testing.T, data storage.BlobData) {
51 | 	err := fs.Write(context.Background(), data)
52 | 	require.NoError(t, err)
53 | }
54 | 
55 | func (fs *TestFileStorage) ReadOrFail(t *testing.T, hash common.Hash) storage.BlobData {
56 | 	data, err := fs.Read(context.Background(), hash)
57 | 	require.NoError(t, err)
58 | 	require.NotNil(t, data)
59 | 	return data
60 | }
61 | 


--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
 1 | version: "3.7"
 2 | 
 3 | services:
 4 |   api:
 5 |     build:
 6 |       context: .
 7 |       dockerfile: Dockerfile
 8 |     env_file:
 9 |       - .env
10 |     command:
11 |       - "blob-api"
12 |     depends_on:
13 |       - minio
14 |       - create-buckets
15 |   archiver:
16 |     build:
17 |       context: .
18 |       dockerfile: Dockerfile
19 |     command:
20 |       - "blob-archiver"
21 |     env_file:
22 |       - .env
23 |     depends_on:
24 |       - minio
25 |       - create-buckets
26 |   minio:
27 |     restart: unless-stopped
28 |     image: minio/minio:latest
29 |     ports:
30 |       - "9000:9000"
31 |       - "9999:9999"
32 |     environment:
33 |       MINIO_ROOT_USER: admin
34 |       MINIO_ROOT_PASSWORD: password
35 |     entrypoint: minio server /data --console-address ":9999"
36 |   create-buckets:
37 |     image: minio/mc
38 |     depends_on:
39 |       - minio
40 |     entrypoint: >
41 |       /bin/sh -c "
42 |       /usr/bin/mc alias set minio http://minio:9000 admin password;
43 |       /usr/bin/mc mb minio/blobs;
44 |       /usr/bin/mc policy set public minio/blobs;
45 |       exit 0;
46 |       "


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
  1 | module github.com/base-org/blob-archiver
  2 | 
  3 | go 1.21.6
  4 | 
  5 | require (
  6 | 	github.com/attestantio/go-eth2-client v0.21.1
  7 | 	github.com/ethereum-optimism/optimism v1.7.2
  8 | 	github.com/ethereum/go-ethereum v1.13.8
  9 | 	github.com/go-chi/chi/v5 v5.0.12
 10 | 	github.com/minio/minio-go/v7 v7.0.66
 11 | 	github.com/prometheus/client_golang v1.19.0
 12 | 	github.com/stretchr/testify v1.9.0
 13 | 	github.com/urfave/cli/v2 v2.27.1
 14 | )
 15 | 
 16 | require (
 17 | 	github.com/DataDog/zstd v1.5.2 // indirect
 18 | 	github.com/Microsoft/go-winio v0.6.1 // indirect
 19 | 	github.com/VictoriaMetrics/fastcache v1.12.1 // indirect
 20 | 	github.com/beorn7/perks v1.0.1 // indirect
 21 | 	github.com/bits-and-blooms/bitset v1.10.0 // indirect
 22 | 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
 23 | 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 24 | 	github.com/cockroachdb/errors v1.11.1 // indirect
 25 | 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 26 | 	github.com/cockroachdb/pebble v0.0.0-20231018212520-f6cde3fc2fa4 // indirect
 27 | 	github.com/cockroachdb/redact v1.1.5 // indirect
 28 | 	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
 29 | 	github.com/consensys/bavard v0.1.13 // indirect
 30 | 	github.com/consensys/gnark-crypto v0.12.1 // indirect
 31 | 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 32 | 	github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect
 33 | 	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
 34 | 	github.com/davecgh/go-spew v1.1.1 // indirect
 35 | 	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
 36 | 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 37 | 	github.com/dustin/go-humanize v1.0.1 // indirect
 38 | 	github.com/ethereum-optimism/superchain-registry/superchain v0.0.0-20240318114348-52d3dbd1605d // indirect
 39 | 	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
 40 | 	github.com/fatih/color v1.16.0 // indirect
 41 | 	github.com/ferranbt/fastssz v0.1.3 // indirect
 42 | 	github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect
 43 | 	github.com/getsentry/sentry-go v0.18.0 // indirect
 44 | 	github.com/go-logr/logr v1.2.4 // indirect
 45 | 	github.com/go-logr/stdr v1.2.2 // indirect
 46 | 	github.com/go-ole/go-ole v1.3.0 // indirect
 47 | 	github.com/goccy/go-yaml v1.9.2 // indirect
 48 | 	github.com/gofrs/flock v0.8.1 // indirect
 49 | 	github.com/gogo/protobuf v1.3.2 // indirect
 50 | 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 51 | 	github.com/google/uuid v1.6.0 // indirect
 52 | 	github.com/gorilla/websocket v1.5.0 // indirect
 53 | 	github.com/holiman/uint256 v1.2.4 // indirect
 54 | 	github.com/huandu/go-clone v1.6.0 // indirect
 55 | 	github.com/json-iterator/go v1.1.12 // indirect
 56 | 	github.com/klauspost/compress v1.17.4 // indirect
 57 | 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 58 | 	github.com/kr/pretty v0.3.1 // indirect
 59 | 	github.com/kr/text v0.2.0 // indirect
 60 | 	github.com/mattn/go-colorable v0.1.13 // indirect
 61 | 	github.com/mattn/go-isatty v0.0.20 // indirect
 62 | 	github.com/mattn/go-runewidth v0.0.14 // indirect
 63 | 	github.com/minio/md5-simd v1.1.2 // indirect
 64 | 	github.com/minio/sha256-simd v1.0.1 // indirect
 65 | 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 66 | 	github.com/mmcloughlin/addchain v0.4.0 // indirect
 67 | 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 68 | 	github.com/modern-go/reflect2 v1.0.2 // indirect
 69 | 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 70 | 	github.com/pkg/errors v0.9.1 // indirect
 71 | 	github.com/pmezard/go-difflib v1.0.0 // indirect
 72 | 	github.com/prometheus/client_model v0.5.0 // indirect
 73 | 	github.com/prometheus/common v0.48.0 // indirect
 74 | 	github.com/prometheus/procfs v0.12.0 // indirect
 75 | 	github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 // indirect
 76 | 	github.com/r3labs/sse/v2 v2.10.0 // indirect
 77 | 	github.com/rivo/uniseg v0.4.3 // indirect
 78 | 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 79 | 	github.com/rs/xid v1.5.0 // indirect
 80 | 	github.com/rs/zerolog v1.32.0 // indirect
 81 | 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 82 | 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 83 | 	github.com/sirupsen/logrus v1.9.3 // indirect
 84 | 	github.com/supranational/blst v0.3.11 // indirect
 85 | 	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
 86 | 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 87 | 	github.com/tklauser/numcpus v0.6.1 // indirect
 88 | 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 89 | 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 90 | 	go.opentelemetry.io/otel v1.16.0 // indirect
 91 | 	go.opentelemetry.io/otel/metric v1.16.0 // indirect
 92 | 	go.opentelemetry.io/otel/trace v1.16.0 // indirect
 93 | 	golang.org/x/crypto v0.21.0 // indirect
 94 | 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
 95 | 	golang.org/x/mod v0.14.0 // indirect
 96 | 	golang.org/x/net v0.21.0 // indirect
 97 | 	golang.org/x/sync v0.6.0 // indirect
 98 | 	golang.org/x/sys v0.18.0 // indirect
 99 | 	golang.org/x/term v0.18.0 // indirect
100 | 	golang.org/x/text v0.14.0 // indirect
101 | 	golang.org/x/tools v0.16.1 // indirect
102 | 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
103 | 	google.golang.org/protobuf v1.32.0 // indirect
104 | 	gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
105 | 	gopkg.in/ini.v1 v1.67.0 // indirect
106 | 	gopkg.in/yaml.v2 v2.4.0 // indirect
107 | 	gopkg.in/yaml.v3 v3.0.1 // indirect
108 | 	rsc.io/tmplfunc v0.0.3 // indirect
109 | )
110 | 
111 | replace github.com/ethereum/go-ethereum v1.13.8 => github.com/ethereum-optimism/op-geth v1.101308.3-rc.1
112 | 


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
  1 | github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
  2 | github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
  3 | github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
  4 | github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
  5 | github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40=
  6 | github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o=
  7 | github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
  8 | github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc=
  9 | github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 10 | github.com/attestantio/go-eth2-client v0.21.1 h1:yvsMd/azPUbxiJzWZhgqfOJJRNF1zLvAJpcBXTHzyh8=
 11 | github.com/attestantio/go-eth2-client v0.21.1/go.mod h1:Tb412NpzhsC0sbtpXS4D51y5se6nDkWAi6amsJrqX9c=
 12 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 13 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 14 | github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88=
 15 | github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
 16 | github.com/btcsuite/btcd v0.24.0 h1:gL3uHE/IaFj6fcZSu03SvqPMSx7s/dPzfpG/atRwWdo=
 17 | github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k=
 18 | github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU=
 19 | github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ=
 20 | github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 21 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 22 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 23 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 24 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 25 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 26 | github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
 27 | github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
 28 | github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8=
 29 | github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw=
 30 | github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 31 | github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
 32 | github.com/cockroachdb/pebble v0.0.0-20231018212520-f6cde3fc2fa4 h1:PuHFhOUMnD62r80dN+Ik5qco2drekgsUSVdcHsvllec=
 33 | github.com/cockroachdb/pebble v0.0.0-20231018212520-f6cde3fc2fa4/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E=
 34 | github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 35 | github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 36 | github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
 37 | github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
 38 | github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ=
 39 | github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
 40 | github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M=
 41 | github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY=
 42 | github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 43 | github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 44 | github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 45 | github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ=
 46 | github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs=
 47 | github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA=
 48 | github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc=
 49 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 50 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 51 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 52 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 53 | github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI=
 54 | github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
 55 | github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y=
 56 | github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
 57 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
 58 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
 59 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 60 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 61 | github.com/ethereum-optimism/op-geth v1.101308.3-rc.1 h1:mC8PrDNTZJr7sYcm+FgaWFUy/LOZ0sS8+BxkglUoPkg=
 62 | github.com/ethereum-optimism/op-geth v1.101308.3-rc.1/go.mod h1:k0UbrLuOITLD8goCyA2xWebAL03n2BZUCfwos0rxz60=
 63 | github.com/ethereum-optimism/optimism v1.7.2 h1:9S1Qi9Ns4eGuFtfpIG4OnX/CDzA0dx4mHGIv5oaYeEw=
 64 | github.com/ethereum-optimism/optimism v1.7.2/go.mod h1:ZwYKTK1oLHPnVsNvGbVXCn2FNwdacfizPAAEDHVh6Ck=
 65 | github.com/ethereum-optimism/superchain-registry/superchain v0.0.0-20240318114348-52d3dbd1605d h1:K7HdD/ZAcSFhcqqnUAbvU+8vsg0DzL8pvetHw5vRLCc=
 66 | github.com/ethereum-optimism/superchain-registry/superchain v0.0.0-20240318114348-52d3dbd1605d/go.mod h1:7xh2awFQqsiZxFrHKTgEd+InVfDRrkKVUIuK8SAFHp0=
 67 | github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY=
 68 | github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0=
 69 | github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 70 | github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 71 | github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 72 | github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo=
 73 | github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE=
 74 | github.com/fjl/memsize v0.0.1 h1:+zhkb+dhUgx0/e+M8sF0QqiouvMQUiKR+QYvdxIOKcQ=
 75 | github.com/fjl/memsize v0.0.1/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0=
 76 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 77 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 78 | github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 79 | github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 80 | github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 81 | github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays=
 82 | github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww=
 83 | github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 h1:BAIP2GihuqhwdILrV+7GJel5lyPV3u1+PgzrWLc0TkE=
 84 | github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46/go.mod h1:QNpY22eby74jVhqH4WhDLDwxc/vqsern6pW+u2kbkpc=
 85 | github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0=
 86 | github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ=
 87 | github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s=
 88 | github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 89 | github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 90 | github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 91 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 92 | github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 93 | github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 94 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 95 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 96 | github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 97 | github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
 98 | github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
 99 | github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
100 | github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
101 | github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
102 | github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
103 | github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
104 | github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
105 | github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
106 | github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
107 | github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ=
108 | github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU=
109 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
110 | github.com/goccy/go-yaml v1.9.2 h1:2Njwzw+0+pjU2gb805ZC1B/uBuAs2VcZ3K+ZgHwDs7w=
111 | github.com/goccy/go-yaml v1.9.2/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
112 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
113 | github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
114 | github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
115 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
116 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
117 | github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
118 | github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
119 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
120 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
121 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
122 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
123 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
124 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
125 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
126 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
127 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
128 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
129 | github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk=
130 | github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
131 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
132 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
133 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
134 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
135 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
136 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
137 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
138 | github.com/google/gofuzz v1.2.1-0.20220503160820-4a35382e8fc8 h1:Ep/joEub9YwcjRY6ND3+Y/w0ncE540RtGatVhtZL0/Q=
139 | github.com/google/gofuzz v1.2.1-0.20220503160820-4a35382e8fc8/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
140 | github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
141 | github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
142 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
143 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
144 | github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
145 | github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
146 | github.com/hashicorp/go-bexpr v0.1.11 h1:6DqdA/KBjurGby9yTY0bmkathya0lfwF2SeuubCI7dY=
147 | github.com/hashicorp/go-bexpr v0.1.11/go.mod h1:f03lAo0duBlDIUMGCuad8oLcgejw4m7U+N8T+6Kz1AE=
148 | github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 h1:3JQNjnMRil1yD0IfZKHF9GxxWKDJGj8I0IqOUol//sw=
149 | github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc=
150 | github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao=
151 | github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
152 | github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU=
153 | github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E=
154 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
155 | github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c=
156 | github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U=
157 | github.com/huandu/go-clone v1.6.0 h1:HMo5uvg4wgfiy5FoGOqlFLQED/VGRm2D9Pi8g1FXPGc=
158 | github.com/huandu/go-clone v1.6.0/go.mod h1:ReGivhG6op3GYr+UY3lS6mxjKp7MIGTknuU5TbTVaXE=
159 | github.com/huandu/go-clone/generic v1.6.0 h1:Wgmt/fUZ28r16F2Y3APotFD59sHk1p78K0XLdbUYN5U=
160 | github.com/huandu/go-clone/generic v1.6.0/go.mod h1:xgd9ZebcMsBWWcBx5mVMCoqMX24gLWr5lQicr+nVXNs=
161 | github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc=
162 | github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
163 | github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
164 | github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
165 | github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
166 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
167 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
168 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
169 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
170 | github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
171 | github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
172 | github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
173 | github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
174 | github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
175 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
176 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
177 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
178 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
179 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
180 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
181 | github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
182 | github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
183 | github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
184 | github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
185 | github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
186 | github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
187 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
188 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
189 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
190 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
191 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
192 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
193 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
194 | github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
195 | github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
196 | github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
197 | github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
198 | github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
199 | github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
200 | github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs=
201 | github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
202 | github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
203 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
204 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
205 | github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQhoHZAz4x7TIw=
206 | github.com/mitchellh/pointerstructure v1.2.1/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4=
207 | github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY=
208 | github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU=
209 | github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU=
210 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
211 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
212 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
213 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
214 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
215 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
216 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
217 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
218 | github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
219 | github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
220 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
221 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
222 | github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
223 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
224 | github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
225 | github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
226 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
227 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
228 | github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
229 | github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
230 | github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo=
231 | github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0=
232 | github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
233 | github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
234 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
235 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
236 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
237 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
238 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
239 | github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
240 | github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
241 | github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
242 | github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
243 | github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
244 | github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
245 | github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
246 | github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
247 | github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 h1:0tVE4tdWQK9ZpYygoV7+vS6QkDvQVySboMVEIxBJmXw=
248 | github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7/go.mod h1:wmuf/mdK4VMD+jA9ThwcUKjg3a2XWM9cVfFYjDyY4j4=
249 | github.com/r3labs/sse/v2 v2.10.0 h1:hFEkLLFY4LDifoHdiCN/LlGBAdVJYsANaLqNYa1l/v0=
250 | github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I=
251 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
252 | github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw=
253 | github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
254 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
255 | github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
256 | github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
257 | github.com/rs/cors v1.9.0 h1:l9HGsTsHJcvW14Nk7J9KFz8bzeAWXn3CG6bgt7LsrAE=
258 | github.com/rs/cors v1.9.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
259 | github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
260 | github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
261 | github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
262 | github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
263 | github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
264 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
265 | github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
266 | github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
267 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
268 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
269 | github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA=
270 | github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg=
271 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
272 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
273 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
274 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
275 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
276 | github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
277 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
278 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
279 | github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4=
280 | github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
281 | github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI=
282 | github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48=
283 | github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
284 | github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
285 | github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
286 | github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
287 | github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8=
288 | github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U=
289 | github.com/umbracle/gohashtree v0.0.2-alpha.0.20230207094856-5b775a815c10 h1:CQh33pStIp/E30b7TxDlXfM0145bn2e8boI30IxAhTg=
290 | github.com/umbracle/gohashtree v0.0.2-alpha.0.20230207094856-5b775a815c10/go.mod h1:x/Pa0FF5Te9kdrlZKJK82YmAkvL8+f989USgz6Jiw7M=
291 | github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
292 | github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
293 | github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
294 | github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
295 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
296 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
297 | github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
298 | github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
299 | go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s=
300 | go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=
301 | go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo=
302 | go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4=
303 | go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs=
304 | go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0=
305 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
306 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
307 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
308 | golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
309 | golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
310 | golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
311 | golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE=
312 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
313 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
314 | golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
315 | golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
316 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
317 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
318 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
319 | golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
320 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
321 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
322 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
323 | golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
324 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
325 | golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
326 | golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
327 | golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
328 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
329 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
330 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
331 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
332 | golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
333 | golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
334 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
335 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
336 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
337 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
338 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
339 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
340 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
341 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
342 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
343 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
344 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
345 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
346 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
347 | golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
348 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
349 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
350 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
351 | golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
352 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
353 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
354 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
355 | golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
356 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
357 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
358 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
359 | golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
360 | golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
361 | golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
362 | golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
363 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
364 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
365 | golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
366 | golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
367 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
368 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
369 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
370 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
371 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
372 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
373 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
374 | golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
375 | golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
376 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
377 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
378 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
379 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
380 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
381 | golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA=
382 | golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
383 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
384 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
385 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
386 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
387 | golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
388 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
389 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
390 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
391 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
392 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
393 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
394 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
395 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
396 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
397 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
398 | google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
399 | google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
400 | gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y=
401 | gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI=
402 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
403 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
404 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
405 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
406 | gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
407 | gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
408 | gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
409 | gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
410 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
411 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
412 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
413 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
414 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
415 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
416 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
417 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
418 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
419 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
420 | rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
421 | rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=
422 | 


--------------------------------------------------------------------------------
/validator/Makefile:
--------------------------------------------------------------------------------
 1 | blob-validator:
 2 | 	env GO111MODULE=on GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o ./bin/blob-validator ./cmd/main.go
 3 | 
 4 | clean:
 5 | 	rm -f bin/blob-validator
 6 | 
 7 | test:
 8 | 	go test -v -race ./...
 9 | 
10 | .PHONY: \
11 | 	blob-validator \
12 | 	clean \
13 | 	test
14 | 


--------------------------------------------------------------------------------
/validator/cmd/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"context"
 5 | 	"fmt"
 6 | 	"os"
 7 | 
 8 | 	"github.com/base-org/blob-archiver/common/beacon"
 9 | 	"github.com/base-org/blob-archiver/validator/flags"
10 | 	"github.com/base-org/blob-archiver/validator/service"
11 | 	opservice "github.com/ethereum-optimism/optimism/op-service"
12 | 	"github.com/ethereum-optimism/optimism/op-service/cliapp"
13 | 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
14 | 	"github.com/ethereum/go-ethereum/log"
15 | 	"github.com/urfave/cli/v2"
16 | )
17 | 
18 | var (
19 | 	Version   = "v0.0.1"
20 | 	GitCommit = ""
21 | 	GitDate   = ""
22 | )
23 | 
24 | func main() {
25 | 	oplog.SetupDefaults()
26 | 
27 | 	app := cli.NewApp()
28 | 	app.Flags = cliapp.ProtectFlags(flags.Flags)
29 | 	app.Version = opservice.FormatVersion(Version, GitCommit, GitDate, "")
30 | 	app.Name = "blob-validator"
31 | 	app.Usage = "Job that checks the validity of blobs"
32 | 	app.Description = "The blob-validator is a job that checks the validity of blobs"
33 | 	app.Action = cliapp.LifecycleCmd(Main())
34 | 
35 | 	err := app.Run(os.Args)
36 | 	if err != nil {
37 | 		log.Crit("Application failed", "message", err)
38 | 	}
39 | }
40 | 
41 | // Main is the entrypoint into the API.
42 | // This method returns a cliapp.LifecycleAction, to create an op-service CLI-lifecycle-managed API Server.
43 | func Main() cliapp.LifecycleAction {
44 | 	return func(cliCtx *cli.Context, closeApp context.CancelCauseFunc) (cliapp.Lifecycle, error) {
45 | 		cfg := flags.ReadConfig(cliCtx)
46 | 		if err := cfg.Check(); err != nil {
47 | 			return nil, fmt.Errorf("config check failed: %w", err)
48 | 		}
49 | 
50 | 		l := oplog.NewLogger(oplog.AppOut(cliCtx), cfg.LogConfig)
51 | 		oplog.SetGlobalLogHandler(l.Handler())
52 | 		opservice.ValidateEnvVars(flags.EnvVarPrefix, flags.Flags, l)
53 | 
54 | 		headerClient, err := beacon.NewBeaconClient(cliCtx.Context, cfg.BeaconConfig)
55 | 		if err != nil {
56 | 			return nil, fmt.Errorf("failed to create beacon client: %w", err)
57 | 		}
58 | 
59 | 		beaconClient := service.NewBlobSidecarClient(cfg.BeaconConfig.BeaconURL)
60 | 		blobClient := service.NewBlobSidecarClient(cfg.BlobConfig.BeaconURL)
61 | 
62 | 		return service.NewValidator(l, headerClient, beaconClient, blobClient, closeApp), nil
63 | 	}
64 | }
65 | 


--------------------------------------------------------------------------------
/validator/flags/config.go:
--------------------------------------------------------------------------------
 1 | package flags
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"time"
 6 | 
 7 | 	common "github.com/base-org/blob-archiver/common/flags"
 8 | 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
 9 | 	"github.com/urfave/cli/v2"
10 | )
11 | 
12 | type ValidatorConfig struct {
13 | 	LogConfig    oplog.CLIConfig
14 | 	BeaconConfig common.BeaconConfig
15 | 	BlobConfig   common.BeaconConfig
16 | }
17 | 
18 | func (c ValidatorConfig) Check() error {
19 | 	if err := c.BeaconConfig.Check(); err != nil {
20 | 		return fmt.Errorf("beacon config check failed: %w", err)
21 | 	}
22 | 
23 | 	if err := c.BlobConfig.Check(); err != nil {
24 | 		return fmt.Errorf("blob config check failed: %w", err)
25 | 	}
26 | 
27 | 	return nil
28 | }
29 | 
30 | func ReadConfig(cliCtx *cli.Context) ValidatorConfig {
31 | 	timeout, _ := time.ParseDuration(cliCtx.String(BeaconClientTimeoutFlag.Name))
32 | 
33 | 	return ValidatorConfig{
34 | 		LogConfig: oplog.ReadCLIConfig(cliCtx),
35 | 		BeaconConfig: common.BeaconConfig{
36 | 			BeaconURL:           cliCtx.String(L1BeaconClientUrlFlag.Name),
37 | 			BeaconClientTimeout: timeout,
38 | 		},
39 | 		BlobConfig: common.BeaconConfig{
40 | 			BeaconURL:           cliCtx.String(BlobApiClientUrlFlag.Name),
41 | 			BeaconClientTimeout: timeout,
42 | 		},
43 | 	}
44 | }
45 | 


--------------------------------------------------------------------------------
/validator/flags/flags.go:
--------------------------------------------------------------------------------
 1 | package flags
 2 | 
 3 | import (
 4 | 	opservice "github.com/ethereum-optimism/optimism/op-service"
 5 | 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
 6 | 	"github.com/urfave/cli/v2"
 7 | )
 8 | 
 9 | const EnvVarPrefix = "BLOB_VALIDATOR"
10 | 
11 | var (
12 | 	BeaconClientTimeoutFlag = &cli.StringFlag{
13 | 		Name:    "beacon-client-timeout",
14 | 		Usage:   "The timeout duration for the beacon client",
15 | 		Value:   "10s",
16 | 		EnvVars: opservice.PrefixEnvVar(EnvVarPrefix, "CLIENT_TIMEOUT"),
17 | 	}
18 | 	L1BeaconClientUrlFlag = &cli.StringFlag{
19 | 		Name:     "l1-beacon-http",
20 | 		Usage:    "URL for a L1 Beacon-node API",
21 | 		Required: true,
22 | 		EnvVars:  opservice.PrefixEnvVar(EnvVarPrefix, "L1_BEACON_HTTP"),
23 | 	}
24 | 	BlobApiClientUrlFlag = &cli.StringFlag{
25 | 		Name:     "blob-api-http",
26 | 		Usage:    "URL for a Blob API",
27 | 		Required: true,
28 | 		EnvVars:  opservice.PrefixEnvVar(EnvVarPrefix, "BLOB_API_HTTP"),
29 | 	}
30 | )
31 | 
32 | func init() {
33 | 	Flags = append(Flags, oplog.CLIFlags(EnvVarPrefix)...)
34 | 	Flags = append(Flags, BeaconClientTimeoutFlag, L1BeaconClientUrlFlag, BlobApiClientUrlFlag)
35 | }
36 | 
37 | // Flags contains the list of configuration options available to the binary.
38 | var Flags []cli.Flag
39 | 


--------------------------------------------------------------------------------
/validator/service/client.go:
--------------------------------------------------------------------------------
 1 | package service
 2 | 
 3 | import (
 4 | 	"encoding/json"
 5 | 	"fmt"
 6 | 	"io"
 7 | 	"net/http"
 8 | 
 9 | 	"github.com/attestantio/go-eth2-client/api"
10 | 	"github.com/base-org/blob-archiver/common/storage"
11 | )
12 | 
13 | type Format string
14 | 
15 | const (
16 | 	// FormatJson instructs the client to request the response in JSON format
17 | 	FormatJson Format = "application/json"
18 | 	// FormatSSZ instructs the client to request the response in SSZ format
19 | 	FormatSSZ Format = "application/octet-stream"
20 | )
21 | 
22 | // BlobSidecarClient is a minimal client for fetching sidecars from the blob service. This client is used instead of an
23 | // existing client for two reasons.
24 | // 1) Does not require any endpoints except /eth/v1/blob_sidecar, which is the only endpoint that the Blob API supports
25 | // 2) Exposes implementation details, e.g. status code, as well as allowing us to specify the format
26 | type BlobSidecarClient interface {
27 | 	// FetchSidecars fetches the sidecars for a given slot from the blob sidecar API. It returns the HTTP status code and
28 | 	// the sidecars.
29 | 	FetchSidecars(id string, format Format) (int, storage.BlobSidecars, error)
30 | }
31 | 
32 | type httpBlobSidecarClient struct {
33 | 	url    string
34 | 	client *http.Client
35 | }
36 | 
37 | // NewBlobSidecarClient creates a new BlobSidecarClient that fetches sidecars from the given URL.
38 | func NewBlobSidecarClient(url string) BlobSidecarClient {
39 | 	return &httpBlobSidecarClient{
40 | 		url:    url,
41 | 		client: &http.Client{},
42 | 	}
43 | }
44 | 
45 | func (c *httpBlobSidecarClient) FetchSidecars(id string, format Format) (int, storage.BlobSidecars, error) {
46 | 	url := fmt.Sprintf("%s/eth/v1/beacon/blob_sidecars/%s", c.url, id)
47 | 	req, err := http.NewRequest("GET", url, nil)
48 | 	if err != nil {
49 | 		return http.StatusInternalServerError, storage.BlobSidecars{}, fmt.Errorf("failed to create request: %w", err)
50 | 	}
51 | 
52 | 	req.Header.Set("Accept", string(format))
53 | 
54 | 	response, err := c.client.Do(req)
55 | 	if err != nil {
56 | 		return http.StatusInternalServerError, storage.BlobSidecars{}, fmt.Errorf("failed to fetch sidecars: %w", err)
57 | 	}
58 | 
59 | 	if response.StatusCode != http.StatusOK {
60 | 		return response.StatusCode, storage.BlobSidecars{}, nil
61 | 	}
62 | 
63 | 	defer response.Body.Close()
64 | 
65 | 	var sidecars storage.BlobSidecars
66 | 	if format == FormatJson {
67 | 		if err := json.NewDecoder(response.Body).Decode(&sidecars); err != nil {
68 | 			return response.StatusCode, storage.BlobSidecars{}, fmt.Errorf("failed to decode json response: %w", err)
69 | 		}
70 | 	} else {
71 | 		body, err := io.ReadAll(response.Body)
72 | 		if err != nil {
73 | 			return response.StatusCode, storage.BlobSidecars{}, fmt.Errorf("failed to read response: %w", err)
74 | 		}
75 | 
76 | 		s := api.BlobSidecars{}
77 | 		if err := s.UnmarshalSSZ(body); err != nil {
78 | 			return response.StatusCode, storage.BlobSidecars{}, fmt.Errorf("failed to decode ssz response: %w", err)
79 | 		}
80 | 
81 | 		sidecars.Data = s.Sidecars
82 | 	}
83 | 
84 | 	return response.StatusCode, sidecars, nil
85 | }
86 | 


--------------------------------------------------------------------------------
/validator/service/service.go:
--------------------------------------------------------------------------------
  1 | package service
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"errors"
  6 | 	"fmt"
  7 | 	"net/http"
  8 | 	"reflect"
  9 | 	"strconv"
 10 | 	"sync/atomic"
 11 | 
 12 | 	client "github.com/attestantio/go-eth2-client"
 13 | 	"github.com/attestantio/go-eth2-client/api"
 14 | 	v1 "github.com/attestantio/go-eth2-client/api/v1"
 15 | 	"github.com/attestantio/go-eth2-client/spec/phase0"
 16 | 	"github.com/base-org/blob-archiver/common/storage"
 17 | 	"github.com/ethereum-optimism/optimism/op-service/retry"
 18 | 	"github.com/ethereum/go-ethereum/log"
 19 | )
 20 | 
 21 | var ErrAlreadyStopped = errors.New("already stopped")
 22 | 
 23 | const (
 24 | 	// 5 blocks per minute, 120 minutes
 25 | 	twoHoursOfBlocks = 5 * 120
 26 | 	// finalized l1 offset
 27 | 	finalizedL1Offset = 64
 28 | 	// Known log for any validation errors
 29 | 	validationErrorLog = "validation error"
 30 | 	// Number of attempts to fetch blobs from blob-api and beacon-node
 31 | 	retryAttempts = 10
 32 | )
 33 | 
 34 | func NewValidator(l log.Logger, headerClient client.BeaconBlockHeadersProvider, beaconAPI BlobSidecarClient, blobAPI BlobSidecarClient, app context.CancelCauseFunc) *ValidatorService {
 35 | 	return &ValidatorService{
 36 | 		log:          l,
 37 | 		headerClient: headerClient,
 38 | 		beaconAPI:    beaconAPI,
 39 | 		blobAPI:      blobAPI,
 40 | 		closeApp:     app,
 41 | 	}
 42 | }
 43 | 
 44 | type ValidatorService struct {
 45 | 	stopped      atomic.Bool
 46 | 	log          log.Logger
 47 | 	headerClient client.BeaconBlockHeadersProvider
 48 | 	beaconAPI    BlobSidecarClient
 49 | 	blobAPI      BlobSidecarClient
 50 | 	closeApp     context.CancelCauseFunc
 51 | }
 52 | 
 53 | // Start starts the validator service. This will fetch the current range of blocks to validate and start the validation
 54 | // process.
 55 | func (a *ValidatorService) Start(ctx context.Context) error {
 56 | 	header, err := retry.Do(ctx, retryAttempts, retry.Exponential(), func() (*api.Response[*v1.BeaconBlockHeader], error) {
 57 | 		return a.headerClient.BeaconBlockHeader(ctx, &api.BeaconBlockHeaderOpts{
 58 | 			Block: "head",
 59 | 		})
 60 | 	})
 61 | 
 62 | 	if err != nil {
 63 | 		return fmt.Errorf("failed to get beacon block header: %w", err)
 64 | 	}
 65 | 
 66 | 	end := header.Data.Header.Message.Slot - finalizedL1Offset
 67 | 	start := end - twoHoursOfBlocks
 68 | 
 69 | 	go a.checkBlobs(ctx, start, end)
 70 | 
 71 | 	return nil
 72 | }
 73 | 
 74 | // Stops the validator service.
 75 | func (a *ValidatorService) Stop(ctx context.Context) error {
 76 | 	if a.stopped.Load() {
 77 | 		return ErrAlreadyStopped
 78 | 	}
 79 | 
 80 | 	a.log.Info("Stopping validator")
 81 | 	a.stopped.Store(true)
 82 | 
 83 | 	return nil
 84 | }
 85 | 
 86 | func (a *ValidatorService) Stopped() bool {
 87 | 	return a.stopped.Load()
 88 | }
 89 | 
 90 | // CheckBlobResult contains the summary of the blob checks
 91 | type CheckBlobResult struct {
 92 | 	// ErrorFetching contains the list of slots for which the blob-api or beacon-node returned an error
 93 | 	ErrorFetching []string
 94 | 	// MismatchedStatus contains the list of slots for which the status code from the blob-api and beacon-node did not match
 95 | 	MismatchedStatus []string
 96 | 	// MismatchedData contains the list of slots for which the data from the blob-api and beacon-node did not match
 97 | 	MismatchedData []string
 98 | }
 99 | 
100 | // shouldRetry returns true if the status code is one of the retryable status codes
101 | func shouldRetry(status int) bool {
102 | 	switch status {
103 | 	case http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout, http.StatusTooManyRequests:
104 | 		return true
105 | 	default:
106 | 		return false
107 | 	}
108 | }
109 | 
110 | // fetchWithRetries fetches the sidecar and handles retryable error cases (5xx status codes + 429 + connection errors)
111 | func fetchWithRetries(ctx context.Context, endpoint BlobSidecarClient, id string, format Format) (int, storage.BlobSidecars, error) {
112 | 	return retry.Do2(ctx, retryAttempts, retry.Exponential(), func() (int, storage.BlobSidecars, error) {
113 | 		status, resp, err := endpoint.FetchSidecars(id, format)
114 | 
115 | 		if err == nil && status != http.StatusOK && shouldRetry(status) {
116 | 			err = fmt.Errorf("retryable status code: %d", status)
117 | 		}
118 | 
119 | 		return status, resp, err
120 | 	})
121 | }
122 | 
123 | // checkBlobs iterates all blocks in the range start:end and checks that the blobs from the beacon-node and blob-api
124 | // are identical, when encoded in both JSON and SSZ.
125 | func (a *ValidatorService) checkBlobs(ctx context.Context, start phase0.Slot, end phase0.Slot) CheckBlobResult {
126 | 	var result CheckBlobResult
127 | 
128 | 	for slot := start; slot <= end; slot++ {
129 | 		for _, format := range []Format{FormatJson, FormatSSZ} {
130 | 			id := strconv.FormatUint(uint64(slot), 10)
131 | 
132 | 			l := a.log.New("format", format, "slot", slot)
133 | 
134 | 			blobStatus, blobResponse, blobError := fetchWithRetries(ctx, a.blobAPI, id, format)
135 | 
136 | 			if blobError != nil {
137 | 				result.ErrorFetching = append(result.ErrorFetching, id)
138 | 				l.Error(validationErrorLog, "reason", "error-blob-api", "error", blobError, "status", blobStatus)
139 | 				continue
140 | 			}
141 | 
142 | 			beaconStatus, beaconResponse, beaconErr := fetchWithRetries(ctx, a.beaconAPI, id, format)
143 | 
144 | 			if beaconErr != nil {
145 | 				result.ErrorFetching = append(result.ErrorFetching, id)
146 | 				l.Error(validationErrorLog, "reason", "error-beacon-api", "error", beaconErr, "status", beaconStatus)
147 | 				continue
148 | 			}
149 | 
150 | 			if beaconStatus != blobStatus {
151 | 				result.MismatchedStatus = append(result.MismatchedStatus, id)
152 | 				l.Error(validationErrorLog, "reason", "status-code-mismatch", "beaconStatus", beaconStatus, "blobStatus", blobStatus)
153 | 				continue
154 | 			}
155 | 
156 | 			if beaconStatus != http.StatusOK {
157 | 				// This can happen if the slot has been missed
158 | 				l.Info("matching error status", "beaconStatus", beaconStatus, "blobStatus", blobStatus)
159 | 				continue
160 | 
161 | 			}
162 | 
163 | 			if !reflect.DeepEqual(beaconResponse, blobResponse) {
164 | 				result.MismatchedData = append(result.MismatchedData, id)
165 | 				l.Error(validationErrorLog, "reason", "response-mismatch")
166 | 			}
167 | 
168 | 			l.Info("completed blob check", "blobs", len(beaconResponse.Data))
169 | 		}
170 | 
171 | 		// Check if we should stop validation otherwise continue
172 | 		select {
173 | 		case <-ctx.Done():
174 | 			return result
175 | 		default:
176 | 			continue
177 | 		}
178 | 	}
179 | 
180 | 	// Validation is complete, shutdown the app
181 | 	a.closeApp(nil)
182 | 
183 | 	return result
184 | }
185 | 


--------------------------------------------------------------------------------
/validator/service/service_test.go:
--------------------------------------------------------------------------------
  1 | package service
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"encoding/json"
  6 | 	"fmt"
  7 | 	"strconv"
  8 | 	"testing"
  9 | 
 10 | 	"github.com/attestantio/go-eth2-client/spec/deneb"
 11 | 	"github.com/attestantio/go-eth2-client/spec/phase0"
 12 | 	"github.com/base-org/blob-archiver/common/beacon/beacontest"
 13 | 	"github.com/base-org/blob-archiver/common/blobtest"
 14 | 	"github.com/base-org/blob-archiver/common/storage"
 15 | 	"github.com/ethereum-optimism/optimism/op-service/testlog"
 16 | 	"github.com/ethereum/go-ethereum/log"
 17 | 	"github.com/stretchr/testify/require"
 18 | )
 19 | 
 20 | var (
 21 | 	blockOne = strconv.FormatUint(blobtest.StartSlot+1, 10)
 22 | )
 23 | 
 24 | type response struct {
 25 | 	data       storage.BlobSidecars
 26 | 	err        error
 27 | 	statusCode int
 28 | }
 29 | 
 30 | type stubBlobSidecarClient struct {
 31 | 	data map[string]response
 32 | }
 33 | 
 34 | // setResponses configures the stub to return the same data as the beacon client for all FetchSidecars invocations
 35 | func (s *stubBlobSidecarClient) setResponses(sbc *beacontest.StubBeaconClient) {
 36 | 	for k, v := range sbc.Blobs {
 37 | 		s.data[k] = response{
 38 | 			data:       storage.BlobSidecars{Data: v},
 39 | 			err:        nil,
 40 | 			statusCode: 200,
 41 | 		}
 42 | 	}
 43 | }
 44 | 
 45 | // setResponse overrides a single FetchSidecars response
 46 | func (s *stubBlobSidecarClient) setResponse(id string, statusCode int, data storage.BlobSidecars, err error) {
 47 | 	s.data[id] = response{
 48 | 		data:       data,
 49 | 		err:        err,
 50 | 		statusCode: statusCode,
 51 | 	}
 52 | }
 53 | 
 54 | func (s *stubBlobSidecarClient) FetchSidecars(id string, format Format) (int, storage.BlobSidecars, error) {
 55 | 	response, ok := s.data[id]
 56 | 	if !ok {
 57 | 		return 0, storage.BlobSidecars{}, fmt.Errorf("not found")
 58 | 	}
 59 | 	return response.statusCode, response.data, response.err
 60 | }
 61 | 
 62 | func setup(t *testing.T) (*ValidatorService, *beacontest.StubBeaconClient, *stubBlobSidecarClient, *stubBlobSidecarClient) {
 63 | 	l := testlog.Logger(t, log.LvlInfo)
 64 | 	headerClient := beacontest.NewDefaultStubBeaconClient(t)
 65 | 	cancel := func(error) {}
 66 | 
 67 | 	beacon := &stubBlobSidecarClient{
 68 | 		data: make(map[string]response),
 69 | 	}
 70 | 	blob := &stubBlobSidecarClient{
 71 | 		data: make(map[string]response),
 72 | 	}
 73 | 
 74 | 	return NewValidator(l, headerClient, beacon, blob, cancel), headerClient, beacon, blob
 75 | }
 76 | 
 77 | func TestValidatorService_OnFetchError(t *testing.T) {
 78 | 	validator, _, _, _ := setup(t)
 79 | 
 80 | 	result := validator.checkBlobs(context.Background(), phase0.Slot(blobtest.StartSlot), phase0.Slot(blobtest.StartSlot+1))
 81 | 
 82 | 	// Expect an error for both SSZ and JSON
 83 | 	startSlot := strconv.FormatUint(blobtest.StartSlot, 10)
 84 | 	endSlot := strconv.FormatUint(blobtest.StartSlot+1, 10)
 85 | 	require.Equal(t, result.ErrorFetching, []string{startSlot, startSlot, endSlot, endSlot})
 86 | 	require.Empty(t, result.MismatchedStatus)
 87 | 	require.Empty(t, result.MismatchedData)
 88 | }
 89 | 
 90 | func TestValidatorService_AllMatch(t *testing.T) {
 91 | 	validator, headers, beacon, blob := setup(t)
 92 | 
 93 | 	// Set the beacon + blob APIs to return the same data
 94 | 	beacon.setResponses(headers)
 95 | 	blob.setResponses(headers)
 96 | 
 97 | 	result := validator.checkBlobs(context.Background(), phase0.Slot(blobtest.StartSlot), phase0.Slot(blobtest.EndSlot))
 98 | 
 99 | 	require.Empty(t, result.MismatchedStatus)
100 | 	require.Empty(t, result.MismatchedData)
101 | 	require.Empty(t, result.ErrorFetching)
102 | }
103 | 
104 | func TestValidatorService_MismatchedStatus(t *testing.T) {
105 | 	validator, headers, beacon, blob := setup(t)
106 | 
107 | 	// Set the blob API to return a 404 for blob=1
108 | 	beacon.setResponses(headers)
109 | 	blob.setResponses(headers)
110 | 	blob.setResponse(blockOne, 404, storage.BlobSidecars{}, nil)
111 | 
112 | 	result := validator.checkBlobs(context.Background(), phase0.Slot(blobtest.StartSlot), phase0.Slot(blobtest.EndSlot))
113 | 
114 | 	require.Empty(t, result.MismatchedData)
115 | 	require.Empty(t, result.ErrorFetching)
116 | 	require.Len(t, result.MismatchedStatus, 2)
117 | 	// The first mismatch is the JSON format, the second is the SSZ format
118 | 	require.Equal(t, result.MismatchedStatus, []string{blockOne, blockOne})
119 | }
120 | 
121 | func TestValidatorService_CompletelyDifferentBlobData(t *testing.T) {
122 | 	validator, headers, beacon, blob := setup(t)
123 | 
124 | 	// Modify the blobs for block 1 to be new random data
125 | 	beacon.setResponses(headers)
126 | 	blob.setResponses(headers)
127 | 	blob.setResponse(blockOne, 200, storage.BlobSidecars{
128 | 		Data: blobtest.NewBlobSidecars(t, 1),
129 | 	}, nil)
130 | 
131 | 	result := validator.checkBlobs(context.Background(), phase0.Slot(blobtest.StartSlot), phase0.Slot(blobtest.EndSlot))
132 | 
133 | 	require.Empty(t, result.MismatchedStatus)
134 | 	require.Empty(t, result.ErrorFetching)
135 | 	require.Len(t, result.MismatchedData, 2)
136 | 	// The first mismatch is the JSON format, the second is the SSZ format
137 | 	require.Equal(t, result.MismatchedData, []string{blockOne, blockOne})
138 | }
139 | 
140 | func TestValidatorService_MistmatchedBlobFields(t *testing.T) {
141 | 	tests := []struct {
142 | 		name         string
143 | 		modification func(i *[]*deneb.BlobSidecar)
144 | 	}{
145 | 		{
146 | 			name: "mismatched index",
147 | 			modification: func(i *[]*deneb.BlobSidecar) {
148 | 				(*i)[0].Index = deneb.BlobIndex(9)
149 | 			},
150 | 		},
151 | 		{
152 | 			name: "mismatched blob",
153 | 			modification: func(i *[]*deneb.BlobSidecar) {
154 | 				(*i)[0].Blob = deneb.Blob{0, 0, 0}
155 | 			},
156 | 		},
157 | 		{
158 | 			name: "mismatched kzg commitment",
159 | 			modification: func(i *[]*deneb.BlobSidecar) {
160 | 				(*i)[0].KZGCommitment = deneb.KZGCommitment{0, 0, 0}
161 | 			},
162 | 		},
163 | 		{
164 | 			name: "mismatched kzg proof",
165 | 			modification: func(i *[]*deneb.BlobSidecar) {
166 | 				(*i)[0].KZGProof = deneb.KZGProof{0, 0, 0}
167 | 			},
168 | 		},
169 | 		{
170 | 			name: "mismatched signed block header",
171 | 			modification: func(i *[]*deneb.BlobSidecar) {
172 | 				(*i)[0].SignedBlockHeader = nil
173 | 			},
174 | 		},
175 | 		{
176 | 			name: "mismatched kzg commitment inclusion proof",
177 | 			modification: func(i *[]*deneb.BlobSidecar) {
178 | 				(*i)[0].KZGCommitmentInclusionProof = deneb.KZGCommitmentInclusionProof{{1, 2, 9}}
179 | 			},
180 | 		},
181 | 	}
182 | 
183 | 	for _, test := range tests {
184 | 		t.Run(test.name, func(t *testing.T) {
185 | 			validator, headers, beacon, blob := setup(t)
186 | 
187 | 			// Modify the blobs for block 1 to be new random data
188 | 			beacon.setResponses(headers)
189 | 			blob.setResponses(headers)
190 | 
191 | 			// Deep copy the blob data
192 | 			d, err := json.Marshal(headers.Blobs[blockOne])
193 | 			require.NoError(t, err)
194 | 			var c []*deneb.BlobSidecar
195 | 			err = json.Unmarshal(d, &c)
196 | 			require.NoError(t, err)
197 | 
198 | 			test.modification(&c)
199 | 
200 | 			blob.setResponse(blockOne, 200, storage.BlobSidecars{
201 | 				Data: c,
202 | 			}, nil)
203 | 
204 | 			result := validator.checkBlobs(context.Background(), phase0.Slot(blobtest.StartSlot), phase0.Slot(blobtest.EndSlot))
205 | 
206 | 			require.Empty(t, result.MismatchedStatus)
207 | 			require.Empty(t, result.ErrorFetching)
208 | 			require.Len(t, result.MismatchedData, 2)
209 | 			// The first mismatch is the JSON format, the second is the SSZ format
210 | 			require.Equal(t, result.MismatchedData, []string{blockOne, blockOne})
211 | 		})
212 | 	}
213 | }
214 | 


--------------------------------------------------------------------------------