├── .github └── workflows │ ├── go-test.yml │ └── golangci-lint.yml ├── .gitignore ├── .golangci.yml ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── SECURITY.md ├── cmd └── pessimism │ ├── bootstrap.go │ └── main.go ├── config.env.template ├── docs ├── README.md ├── alerting.md ├── api.md ├── architecture.md ├── assets │ └── high_level_diagram.png ├── engine.md ├── etl.md ├── index.html ├── invariants.md └── openapi │ └── swagger.yml ├── go.mod ├── go.sum ├── internal ├── alert │ ├── interpolator.go │ ├── interpolator_test.go │ ├── manager.go │ ├── store.go │ └── store_test.go ├── api │ ├── handlers │ │ ├── handlers.go │ │ ├── handlers_test.go │ │ ├── health.go │ │ ├── health_test.go │ │ ├── invariant.go │ │ ├── invariant_test.go │ │ └── middleware │ │ │ └── middleware.go │ ├── models │ │ ├── health.go │ │ ├── invariant.go │ │ └── models.go │ ├── server │ │ └── server.go │ └── service │ │ ├── health.go │ │ ├── health_test.go │ │ ├── invariant.go │ │ ├── invariant_test.go │ │ ├── service.go │ │ └── service_test.go ├── app │ └── app.go ├── client │ ├── eth_client.go │ └── slack_client.go ├── common │ ├── common.go │ └── common_test.go ├── config │ └── config.go ├── core │ ├── alert.go │ ├── config.go │ ├── constants.go │ ├── core.go │ ├── etl.go │ ├── id.go │ ├── id_test.go │ ├── register.go │ └── state.go ├── engine │ ├── addressing.go │ ├── addressing_test.go │ ├── engine.go │ ├── invariant │ │ ├── config.go │ │ └── invariant.go │ ├── manager.go │ ├── registry │ │ ├── balance.go │ │ ├── balance_test.go │ │ ├── event_log.go │ │ ├── event_log_test.go │ │ └── registry.go │ ├── store.go │ └── store_test.go ├── etl │ ├── component │ │ ├── aggregator.go │ │ ├── component.go │ │ ├── component_test.go │ │ ├── egress.go │ │ ├── egress_test.go │ │ ├── ingress.go │ │ ├── ingress_test.go │ │ ├── oracle.go │ │ ├── pipe.go │ │ ├── pipe_test.go │ │ └── types.go │ ├── pipeline │ │ ├── analysis.go │ │ ├── analysis_test.go │ │ ├── graph.go │ │ ├── graph_test.go │ │ ├── manager.go │ │ ├── manager_test.go │ │ ├── pipeline.go │ │ ├── pipeline_test.go │ │ ├── store.go │ │ ├── store_test.go │ │ └── types.go │ └── registry │ │ ├── oracle │ │ ├── account_balance.go │ │ ├── geth_block.go │ │ ├── geth_block_test.go │ │ └── types.go │ │ ├── pipe │ │ └── event_log.go │ │ ├── registry.go │ │ └── registry_test.go ├── logging │ └── logger.go ├── mocks │ ├── alert_manager.go │ ├── api_service.go │ ├── engine_manager.go │ ├── eth_client.go │ ├── etl_manager.go │ ├── oracle.go │ ├── pipe.go │ └── slack_client.go ├── state │ ├── memory.go │ ├── memory_test.go │ └── state.go └── subsystem │ ├── manager.go │ └── manager_test.go └── pull_request_template.md /.github/workflows/go-test.yml: -------------------------------------------------------------------------------- 1 | # Go test workflow 2 | name: go-test 3 | 4 | on: 5 | push: 6 | branches: [ "master", "development" ] 7 | pull_request: 8 | branches: [ "master", "development" ] 9 | 10 | jobs: 11 | 12 | build: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v2 16 | 17 | - name: Set up Go 18 | uses: actions/setup-go@v2 19 | with: 20 | go-version: 1.19 21 | 22 | - name: Run Unit Tests 23 | run: make test 24 | -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | # Linting with golangci-lint 2 | name: golangci-lint 3 | on: 4 | 5 | push: 6 | branches: [ "master", "development" ] 7 | 8 | pull_request: 9 | branches: [ "master", "development" ] 10 | 11 | jobs: 12 | golangci: 13 | name: lint 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | - name: golangci-lint 18 | uses: golangci/golangci-lint-action@v3 19 | with: 20 | # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version 21 | version: v1.52.1 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /bin/ 2 | config.env 3 | /.vscode/ 4 | /.idea -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Pessimism 2 | 3 | ## Code of Conduct 4 | 5 | All interactions with this project follow our [Code of Conduct][code-of-conduct]. 6 | By participating, you are expected to honor this code. Violators can be banned 7 | from further participation in this project, or potentially all Base and/or 8 | Coinbase projects. 9 | 10 | [code-of-conduct]: https://github.com/coinbase/code-of-conduct 11 | 12 | ## Before You Start 13 | 14 | Ensure that you have read and understand the project's README file and the 15 | contribution guidelines. Search the issues tracker to see if the issue you 16 | want to work on has already been reported or if someone is already working 17 | on it. If you find an existing issue that you would like to work on, request 18 | to be assigned to the issue. If you cannot find an existing issue that matches 19 | what you want to work on, create a new issue and wait for it to be assigned to 20 | you before starting work on it. 21 | 22 | ## Bug Reports 23 | 24 | * Ensure your issue [has not already been reported][1]. It may already be fixed! 25 | * Include the steps you carried out to produce the problem. 26 | * Include the behavior you observed along with the behavior you expected, and 27 | why you expected it. 28 | * Include any relevant stack traces or debugging output. 29 | 30 | ## Feature Requests 31 | 32 | We welcome feedback with or without pull requests. If you have an idea for how 33 | to improve the project, great! All we ask is that you take the time to write a 34 | clear and concise explanation of what need you are trying to solve. If you have 35 | thoughts on _how_ it can be solved, include those too! 36 | 37 | The best way to see a feature added, however, is to submit a pull request. 38 | 39 | ## Pull Requests 40 | 41 | * Before creating your pull request, it's usually worth asking if the code 42 | you're planning on writing will actually be considered for merging. You can 43 | do this by [opening an issue][1] and asking. It may also help give the 44 | maintainers context for when the time comes to review your code. 45 | 46 | * Ensure your [commit messages are well-written][2]. This can double as your 47 | pull request message, so it pays to take the time to write a clear message. 48 | 49 | * Add tests for your feature. You should be able to look at other tests for 50 | examples. If you're unsure, don't hesitate to [open an issue][1] and ask! 51 | 52 | * Ensure that your code is well-documented and meets the project's coding standards. 53 | 54 | * Provide a reference to the issue you worked on and provide a brief description of the 55 | changes you made. 56 | 57 | * Submit your pull request! 58 | 59 | ## Contributing to an Existing Issue 60 | 61 | If you have been assigned an issue, please confirm that the issue is still open 62 | and has not already been resolved. If you have any questions about the issue, 63 | please ask on the issue thread before starting work on it. Once you are assigned 64 | to an issue, you can start working on a solution for it. Please note that it 65 | is important to communicate regularly with the project maintainers and update 66 | them on your progress. If you are no longer able to work on an issue, please 67 | let us know as soon as possible so we can reassign it. 68 | 69 | ## Support Requests 70 | 71 | For security reasons, any communication referencing support tickets for Coinbase 72 | products will be ignored. The request will have its content redacted and will 73 | be locked to prevent further discussion. 74 | 75 | All support requests must be made via [our support team][3]. 76 | 77 | [1]: https://github.com/base-org/pessimism/issues 78 | [2]: https://medium.com/brigade-engineering/the-secrets-to-great-commit-messages-106fc0a92a25 79 | [3]: https://support.coinbase.com/customer/en/portal/articles/2288496-how-can-i-contact-coinbase-support- -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Coinbase 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | APP_NAME = pessimism 2 | 3 | LINTER_VERSION = v1.52.1 4 | LINTER_URL = https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh 5 | 6 | GET_LINT_CMD = "curl -sSfL $(LINTER_URL) | sh -s -- -b $(go env GOPATH)/bin $(LINTER_VERSION)" 7 | 8 | RED = \033[0;34m 9 | GREEN = \033[0;32m 10 | BLUE = \033[0;34m 11 | COLOR_END = \033[0;39m 12 | 13 | TEST_LIMIT = 10s 14 | 15 | build-app: 16 | @echo "$(BLUE)» building application binary... $(COLOR_END)" 17 | @CGO_ENABLED=0 go build -a -tags netgo -o bin/$(APP_NAME) ./cmd/pessimism/ 18 | @echo "Binary successfully built" 19 | 20 | run-app: 21 | @./bin/${APP_NAME} 22 | 23 | .PHONY: go-gen-mocks 24 | go-gen-mocks: 25 | @echo "generating go mocks..." 26 | @GO111MODULE=on go generate --run "mockgen*" ./... 27 | 28 | .PHONY: test 29 | test: 30 | @ go test ./... -timeout $(TEST_LIMIT) 31 | 32 | .PHONY: lint 33 | lint: 34 | @echo "$(GREEN) Linting repository Go code...$(COLOR_END)" 35 | @if ! command -v golangci-lint &> /dev/null; \ 36 | then \ 37 | echo "golangci-lint command could not be found...."; \ 38 | echo "\nTo install, please run $(GREEN) $(GET_LINT_CMD) $(COLOR_END)"; \ 39 | echo "\nBuild instructions can be found at: https://golangci-lint.run/usage/install/."; \ 40 | exit 1; \ 41 | fi 42 | 43 | @golangci-lint run 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pessimism 2 | __Because you can't always be optimistic__ 3 | 4 | _Pessimism_ is a public good monitoring service that allows for [Op-Stack](https://stack.optimism.io/) and EVM compatible blockchains to be continously assessed for real-time threats using customly defined user invariant rulesets. To learn about Pessimism's architecture, please advise the documentation. 5 | 6 | 7 | 8 | [![GitHub contributors](https://img.shields.io/github/contributors/base-org/pessimism)](https://github.com/base-org/pessimism/graphs/contributors) 9 | [![GitHub commit activity](https://img.shields.io/github/commit-activity/w/base-org/pessimism)](https://github.com/base-org/pessimism/graphs/contributors) 10 | [![GitHub Stars](https://img.shields.io/github/stars/base-org/pessimism.svg)](https://github.com/base-org/pessimism/stargazers) 11 | ![GitHub repo size](https://img.shields.io/github/repo-size/base-org/pessimism) 12 | [![GitHub](https://img.shields.io/github/license/base-org/pessimism?color=blue)](https://github.com/base-org/pessimism/blob/main/LICENSE) 13 | 14 | 15 | 16 | [![GitHub pull requests by-label](https://img.shields.io/github/issues-pr-raw/base-org/pessimism)](https://github.com/base-org/pessimism/pulls) 17 | [![GitHub Issues](https://img.shields.io/github/issues-raw/base-org/pessimism.svg)](https://github.com/base-org/pessimism/issues) 18 | 19 | **Warning:** 20 | Pessimism is currently experimental and very much in development. It means Pessimism is currently unstable, so code will change and builds can break over the coming months. If you come across problems, it would help greatly to open issues so that we can fix them as quickly as possible. 21 | 22 | ## Setup 23 | To use the template, run the following the command(s): 24 | 1. Create local config file (`config.env`) to store all necessary environmental variables. There's already an example `config.env.template` in the repo that stores default env vars. 25 | 26 | 2. [Download](https://go.dev/doc/install) or upgrade to `golang 1.19`. 27 | 28 | 3. Install all project golang dependencies by running `go mod download`. 29 | 30 | # To Run 31 | 1. Compile pessimism to machine binary by running the following project level command(s): 32 | * Using Make: `make build-app` 33 | 34 | 2. To run the compiled binary, you can use the following project level command(s): 35 | * Using Make: `make run-app` 36 | * Direct Call: `./bin/pessimism` 37 | 38 | ## Linting 39 | [golangci-lint](https://golangci-lint.run/) is used to perform code linting. Configurations are defined in [.golangci.yml](./.golangci.yml) 40 | It can be ran using the following project level command(s): 41 | * Using Make: `make lint` 42 | * Direct Call: `golangci-lint run` 43 | 44 | ## Testing 45 | 46 | ### Unit Tests 47 | Unit tests are written using the native [go test](https://pkg.go.dev/testing) library with test mocks generated using the golang native [mock](https://github.com/golang/mock) library. 48 | 49 | Unit tests can ran using the following project level command(s): 50 | * Using Make: `make test` 51 | * Direct Call: `go test ./...` 52 | 53 | ### Integration Tests 54 | TBD 55 | 56 | ## Bootstrap Config 57 | A bootstrap config file is used to define the initial state of the pessimism service. The file must be `json` formatted with it's directive defined in the `BOOTSTRAP_PATH` env var. 58 | 59 | ### Example 60 | ``` 61 | [ 62 | { 63 | "network": "layer1", 64 | "pipeline_type": "live", 65 | "type": "contract_event", 66 | "start_height": null, 67 | "alert_destination": "slack", 68 | "invariant_params": { 69 | "address": "0xfC0157aA4F5DB7177830ACddB3D5a9BB5BE9cc5e", 70 | "args": ["Transfer(address, address, uint256)"] 71 | } 72 | }, 73 | { 74 | "network": "layer1", 75 | "pipeline_type": "live", 76 | "type": "balance_enforcement", 77 | "start_height": null, 78 | "alert_destination": "slack", 79 | "invariant_params": { 80 | "address": "0xfC0157aA4F5DB7177830ACddB3D5a9BB5BE9cc5e", 81 | "lower": 1, 82 | "upper": 2 83 | } 84 | } 85 | ] 86 | ``` 87 | 88 | 89 | 90 | ## Spawning an invariant session 91 | To learn about the currently supported invariants and how to spawn them, please advise the [invariants documentation](./docs/invariants.md). -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Pessimism Security Policy 2 | 3 | ## Reporting a Security Bug 4 | If you think you have discovered a security issue within any part of this codebase, please let us know. We take security bugs seriously; upon investigating and confirming one, we will patch it within a reasonable amount of time, and ultimately release a public security bulletin, in which we discuss the impact and credit the discoverer. 5 | 6 | Report your findings to our H1 program: https://hackerone.com/coinbase 7 | -------------------------------------------------------------------------------- /cmd/pessimism/bootstrap.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "strings" 8 | 9 | "github.com/base-org/pessimism/internal/app" 10 | ) 11 | 12 | const ( 13 | extJSON = ".json" 14 | ) 15 | 16 | // fetchBootSessions ... Loads the bootstrap file 17 | func fetchBootSessions(path string) ([]app.BootSession, error) { 18 | if !strings.HasSuffix(path, extJSON) { 19 | return nil, fmt.Errorf("invalid bootstrap file format; expected %s", extJSON) 20 | } 21 | 22 | file, err := os.ReadFile(path) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | data := []app.BootSession{} 28 | 29 | err = json.Unmarshal(file, &data) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | return data, nil 35 | } 36 | -------------------------------------------------------------------------------- /cmd/pessimism/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/base-org/pessimism/internal/alert" 8 | "github.com/base-org/pessimism/internal/api/handlers" 9 | "github.com/base-org/pessimism/internal/api/server" 10 | "github.com/base-org/pessimism/internal/api/service" 11 | "github.com/base-org/pessimism/internal/app" 12 | "github.com/base-org/pessimism/internal/client" 13 | "github.com/base-org/pessimism/internal/core" 14 | "github.com/base-org/pessimism/internal/engine" 15 | "github.com/base-org/pessimism/internal/state" 16 | "go.uber.org/zap" 17 | 18 | "github.com/base-org/pessimism/internal/subsystem" 19 | 20 | "github.com/base-org/pessimism/internal/config" 21 | 22 | "github.com/base-org/pessimism/internal/etl/pipeline" 23 | "github.com/base-org/pessimism/internal/etl/registry" 24 | "github.com/base-org/pessimism/internal/logging" 25 | ) 26 | 27 | const ( 28 | // cfgPath ... env file path 29 | cfgPath = "config.env" 30 | ) 31 | 32 | // initializeServer ... Performs dependency injection to build server struct 33 | func initializeServer(ctx context.Context, cfg *config.Config, 34 | m subsystem.Manager) (*server.Server, func(), error) { 35 | ethClient := client.NewEthClient() 36 | apiService := service.New(ctx, cfg.SvcConfig, m, ethClient) 37 | handler, err := handlers.New(ctx, apiService) 38 | if err != nil { 39 | return nil, nil, err 40 | } 41 | 42 | server, cleanup, err := server.New(ctx, cfg.ServerConfig, handler) 43 | if err != nil { 44 | return nil, nil, err 45 | } 46 | return server, cleanup, nil 47 | } 48 | 49 | /* 50 | Subsystem initialization functions 51 | */ 52 | 53 | // initializeAlerting ... Performs dependency injection to build alerting struct 54 | func initializeAlerting(ctx context.Context, cfg *config.Config) alert.Manager { 55 | sc := client.NewSlackClient(cfg.SlackURL) 56 | return alert.NewManager(ctx, sc) 57 | } 58 | 59 | // initalizeETL ... Performs dependency injection to build etl struct 60 | func initalizeETL(ctx context.Context, transit chan core.InvariantInput) pipeline.Manager { 61 | compRegistry := registry.NewRegistry() 62 | analyzer := pipeline.NewAnalyzer(compRegistry) 63 | store := pipeline.NewEtlStore() 64 | dag := pipeline.NewComponentGraph() 65 | 66 | return pipeline.NewManager(ctx, analyzer, compRegistry, store, dag, transit) 67 | } 68 | 69 | // initializeEngine ... Performs dependency injection to build engine struct 70 | func initializeEngine(ctx context.Context, transit chan core.Alert) engine.Manager { 71 | store := engine.NewSessionStore() 72 | am := engine.NewAddressingMap() 73 | re := engine.NewHardCodedEngine() 74 | 75 | return engine.NewManager(ctx, re, am, store, transit) 76 | } 77 | 78 | // main ... Application driver 79 | func main() { 80 | ctx := context.WithValue( 81 | context.Background(), state.Default, state.NewMemState()) 82 | 83 | cfg := config.NewConfig(cfgPath) // Load env vars 84 | 85 | logging.NewLogger(cfg.LoggerConfig, cfg.IsProduction()) 86 | 87 | logger := logging.WithContext(ctx) 88 | logger.Info("Bootstrapping pessimsim monitoring application") 89 | 90 | alrt := initializeAlerting(ctx, cfg) 91 | eng := initializeEngine(ctx, alrt.Transit()) 92 | etl := initalizeETL(ctx, eng.Transit()) 93 | 94 | m := subsystem.NewManager(ctx, etl, eng, alrt) 95 | srver, shutdownServer, err := initializeServer(ctx, cfg, m) 96 | if err != nil { 97 | logger.Error("Error initializing server", zap.Error(err)) 98 | os.Exit(1) 99 | } 100 | 101 | pessimism := app.New(ctx, cfg, m, srver) 102 | 103 | logger.Info("Starting pessimism application") 104 | if err := pessimism.Start(); err != nil { 105 | logger.Error("Error starting pessimism application", zap.Error(err)) 106 | os.Exit(1) 107 | } 108 | 109 | if cfg.IsBootstrap() { 110 | logger.Debug("Bootstrapping application state") 111 | 112 | sessions, err := fetchBootSessions(cfg.BootStrapPath) 113 | if err != nil { 114 | logger.Error("Error loading bootstrap file", zap.Error(err)) 115 | panic(err) 116 | } 117 | 118 | if err := pessimism.BootStrap(sessions); err != nil { 119 | logger.Error("Error bootstrapping application state", zap.Error(err)) 120 | panic(err) 121 | } 122 | 123 | logger.Debug("Application state successfully bootstrapped") 124 | } 125 | 126 | pessimism.ListenForShutdown(func() { 127 | err := m.Shutdown() 128 | if err != nil { 129 | logger.Error("Error shutting down subsystems", zap.Error(err)) 130 | } 131 | 132 | shutdownServer() 133 | }) 134 | 135 | logger.Debug("Waiting for all application threads to end") 136 | 137 | logger.Info("Successful pessimism shutdown") 138 | os.Exit(0) 139 | } 140 | -------------------------------------------------------------------------------- /config.env.template: -------------------------------------------------------------------------------- 1 | # GETH compliant RPC APIs for layer 1 & 2 blockchains 2 | L1_RPC_ENDPOINT="" 3 | L2_RPC_ENDPOINT="" 4 | 5 | # Oracle Geth Block Poll Intervals (ms) 6 | L1_POLL_INTERVAL=5000 7 | L2_POLL_INTERVAL=5000 8 | 9 | # Environment 10 | ENV=local # local,development,production 11 | BOOTSTRAP_PATH="" 12 | 13 | # Custom Logger Configs 14 | LOGGER_USE_CUSTOM=0 # 0 or 1 15 | LOGGER_LEVEL=-1 # -1 (debug), 0 (info), 1 (warn), 2 (error), 3 (dpanic), 4 (panic), 5 (fatal) 16 | LOGGER_DISABLE_CALLER=0 # 0 or 1 17 | LOGGER_DISABLE_STACKTRACE=0 # 0 or 1 18 | LOGGER_ENCODING=console # json,console 19 | LOGGER_OUTPUT_PATHS=stderr # comma separated paths 20 | LOGGER_ERROR_OUTPUT_PATHS=stderr # comma separated paths 21 | 22 | # Server configurations 23 | 24 | SERVER_HOST="localhost" 25 | SERVER_PORT=8080 26 | SERVER_KEEP_ALIVE_TIME=10 27 | SERVER_READ_TIMEOUT=10 28 | SERVER_WRITE_TIMEOUT=10 29 | SERVER_SHUTDOWN_TIME=10 30 | 31 | ## Output Configs 32 | SLACK_ENDPOINT="" 33 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Pessimism Documentation 2 | 3 | This directory contains the english specs for the Pessimism application. 4 | 5 | ## Contents 6 | - [Architecture](architecture.md) 7 | - [JSON-RPC API](api.md) 8 | - [ETL Subsystem](etl.md) 9 | - [Engine Subsystem](engine.md) 10 | - [Alerting Subsystem](alerting.md) 11 | 12 | ## Github Pages 13 | The Pessimism documentation is hosted on Github Pages. To view the documentation, please visit [https://base-org.github.io/pessimism/architecture](https://base-org.github.io/pessimism/architecture). It's worth noting that as of now, the mermaid diagrams fail to generate on Github Pages ([#63](https://github.com/base-org/pessimism/issues/63)). 14 | 15 | ## Contributing 16 | If you would like to contribute to the Pessimism documentation, please advise the guidelines stipulated in the [CONTRIBUTING.md](../CONTRIBUTING.md) file __before__ submitting a pull request. 17 | -------------------------------------------------------------------------------- /docs/alerting.md: -------------------------------------------------------------------------------- 1 | # Alerting 2 | 3 | ## Overview 4 | The alerting subsystem will receive alerts from the `EngineManager` and publish them to the appropriate alerting destinations. The alerting subsystem will also be responsible for managing the lifecycle of alerts. This includes creating, updating, and removing alerting entries for invariant sessions. 5 | 6 | ## Diagram 7 | ```mermaid 8 | graph LR 9 | 10 | subgraph EM["Engine Manager"] 11 | alertingRelay 12 | end 13 | 14 | subgraph AM["Alerting Manager"] 15 | alertingRelay --> |Alert|EL 16 | EL[eventLoop] --> |Alert SUUID|AS["AlertStore"] 17 | AS --> |Alert Destination|EL 18 | EL --> |Submit alert|AD[Destination] 19 | AD --> |Slack|SH["Slack Handler"] 20 | AD --> |counterParty|CPH["Counterparty Handler"] 21 | 22 | end 23 | CPH --> |"HTTP POST"|TPH["Third Party API"] 24 | SH --> |"HTTP POST"|SlackAPI("Slack Webhook API") 25 | 26 | ``` 27 | 28 | ### Alert 29 | An `Alert` type stores all necessary metadata for external consumption by a downstream entity. 30 | ### Alert Store 31 | The alert store is a persistent storage layer that is used to store alerting entries. As of now, the alert store only supports configurable alert destinations for each alerting entry. Ie: 32 | ``` 33 | (SUUID) --> (AlertDestination) 34 | ``` 35 | 36 | ### Alert Destinations 37 | An alert destination is a configurable destination that an alert can be sent to. As of now this only includes _Slack_. In the future however, this will include other third party integrations. 38 | 39 | #### Slack 40 | The Slack alert destination is a configurable destination that allows alerts to be sent to a specific Slack channel. The Slack alert destination will be configured with a Slack webhook URL. The Slack alert destination will then use this URL to send alerts to the specified Slack channel. 41 | 42 | **NOTE: As of now Pessimism can only post alerts to a single slack channel** 43 | 44 | ### Cooldown 45 | **NOTE: This is currently unimplemented** 46 | To ensure that alerts aren't spammed to destinations once invoked, a time based cooldown value should exist that specifies how long an invariantSession must wait before it can propagate a trigged alert again. This value should be configurable by the user via a JSON-RPC request. -------------------------------------------------------------------------------- /docs/api.md: -------------------------------------------------------------------------------- 1 | ## Pessimism API 2 | 3 | ### Overview 4 | The Pessimism API is a RESTful HTTP API that allows users to interact with the Pessimism application. The API is built using the [go-chi](https://github.com/go-chi/chi) framework and is served using the native [http package](https://pkg.go.dev/net/http). The API is designed to be modular and extensible, allowing for the addition of new endpoints and functionality with relative ease. 5 | 6 | Currently, interactive endpoint documentation is hosted via [Swagger UI](https://swagger.io/tools/swagger-ui/) at [https://base-org.github.io/pessimism/](https://base-org.github.io/pessimism/). 7 | 8 | ### Configuration 9 | The API can be customly configured using environment variables stored in a `config.env` file. The following environment variables are used to configure the API: 10 | - `SERVER_HOST`: The host address to serve the API on (eg. `localhost`) 11 | - `SERVER_PORT`: The port to serve the API on (eg. `8080`) 12 | - `SERVER_KEEP_ALIVE`: The keep alive second duration for the server (eg. `10`) 13 | - `SERVER_READ_TIMEOUT`: The read timeout second duration for the server (eg. `10`) 14 | - `SERVER_WRITE_TIMEOUT`: The write timeout second duration for the server (eg. `10`) 15 | 16 | ### Components 17 | The Pessimism API is broken down into the following constituent components: 18 | * `handlers`: The handlers package contains the HTTP handlers for the API. Each handler is responsible for handling a specific endpoint and is responsible for parsing the request, calling the appropriate service method, and renders a response. 19 | * `service`: The service package contains the business logic for the API. The service is responsible for handling calls to the core Pessimism subsystems and is responsible for validating incoming requests. 20 | * `models`: The models package contains the data models used by the API. Each model is responsible for representing a specific data type and is used to parse and validate incoming requests. 21 | * `server`: The server package contains the HTTP server for the API. The server is responsible for serving the API and is responsible for handling incoming requests and dispatching them to the appropriate handler function. 22 | 23 | ### Authorization and Authentication 24 | TBD -------------------------------------------------------------------------------- /docs/architecture.md: -------------------------------------------------------------------------------- 1 | # Pessimism Architecture 2 | 3 | ## Overview 4 | There are *three subsystems* that drive Pessimism’s architecture: 5 | 1. [ETL](./etl.md) - Modularized data extraction system for retrieving and processing external chain data in the form of a DAG known as the Pipeline DAG 6 | 2. [Risk Engine](./engine.md) - Logical execution platform that runs a set of invariants on the data funneled from the Pipeline DAG 7 | 3. [Alerting](./alerting.md) - Alerting system that is used to notify users of invariant failures 8 | 9 | These systems will be accessible by a client through the use of a JSON-RPC API that has unilateral access to all three primary subsystems. 10 | 11 | The API will be supported to allow Pessimism users via client to: 12 | 1. Start invariant sessions 13 | 2. Update existing invariant sessions 14 | 3. Remove invariant sessions 15 | 16 | ## Diagram 17 | The following diagram illustrates the core interaction flow between the three primary subsystems, API, and external data sources: 18 | ![high level component diagram](./assets/high_level_diagram.png) 19 | 20 | ## Shared State 21 | To provide context about specific data values (ie. addresses to monitor) between subsystems, Pessimism uses a shared state store. The shared state store will be a non-persistent storage layer. This means that the data will not be persisted to disk and will be lost upon restart of the Pessimism service. 22 | 23 | **NOTE: As of now, the shared state store only supports an in-memory representation and fails to leverage more proper cache solutions like Redis** 24 | -------------------------------------------------------------------------------- /docs/assets/high_level_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feiliang91/pessimism/b572538ba9bfd0c502ac4c098c12a2523d20e71d/docs/assets/high_level_diagram.png -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Pessimism API 9 | 10 | 11 |
12 | 30 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /docs/invariants.md: -------------------------------------------------------------------------------- 1 | # Invariants 2 | 3 | 4 | ## Balance Enforcement 5 | The hardcoded `balance_enforcement` invariant checks the native ETH balance of some address every `n` milliseconds and alerts to slack if the account's balance is ever less than `lower` or greater than `upper` value. This invariant is useful for monitoring hot wallets and other accounts that should always have a balance above a certain threshold. 6 | 7 | 8 | ### Parameters 9 | | Name | Type | Description | 10 | | ---- | ---- | ----------- | 11 | | address | string | The address to check the balance of | 12 | | lower | float | The ETH lower bound of the balance | 13 | | upper | float | The ETH upper bound of the balance | 14 | 15 | 16 | ### Example Deploy Request 17 | ``` 18 | curl --location --request POST 'http://localhost:8080/v0/invariant' \ 19 | --header 'Content-Type: text/plain' \ 20 | --data-raw '{ 21 | "method": "run", 22 | "params": { 23 | "network": "layer1", 24 | "pipeline_type": "live", 25 | "type": "balance_enforcement", 26 | "start_height": null, 27 | "alert_destination": "slack", 28 | "invariant_params": { 29 | "address": "0xfC0157aA4F5DB7177830ACddB3D5a9BB5BE9cc5e", 30 | "lower": 1, 31 | "upper": 2 32 | } 33 | } 34 | }' 35 | ``` 36 | 37 | ## Contract Event 38 | The hardcoded `contract_event` invariant scans newly produced blocks for a specific contract event and alerts to slack if the event is found. This invariant is useful for monitoring for specific contract events that should never occur. 39 | 40 | ### Parameters 41 | | Name | Type | Description | 42 | | ---- | ---- | ----------- | 43 | | address | string | The address of the contract to scan for the events | 44 | | args | []string | The event signatures to scan for | 45 | 46 | **NOTE:** The `args` field is an array of string event declarations (eg. `Transfer(address,address,uint256)`). Currently Pessimism makes no use of contract ABIs so the manually specified event declarations are not validated for correctness. If the event declaration is incorrect, the invariant session will never alert but will continue to scan. 47 | 48 | 49 | ### Example Deploy Request 50 | ``` 51 | curl --location --request POST 'http://localhost:8080/v0/invariant' \ 52 | --header 'Content-Type: text/plain' \ 53 | --data-raw '{ 54 | "method": "run", 55 | "params": { 56 | "network": "layer1", 57 | "pipeline_type": "live", 58 | "type": "contract_event", 59 | "start_height": null, 60 | "alert_destination": "slack", 61 | "invariant_params": { 62 | "address": "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", 63 | "args": ["Transfer(address,address,uint256)"] 64 | } 65 | } 66 | }' -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/base-org/pessimism 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/ethereum/go-ethereum v1.11.6 7 | github.com/go-chi/chi v1.5.4 8 | github.com/go-chi/render v1.0.2 9 | github.com/golang/mock v1.6.0 10 | github.com/google/uuid v1.3.0 11 | github.com/joho/godotenv v1.5.1 12 | github.com/stretchr/testify v1.8.2 13 | go.uber.org/zap v1.24.0 14 | ) 15 | 16 | require ( 17 | github.com/DataDog/zstd v1.5.2 // indirect 18 | github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect 19 | github.com/VictoriaMetrics/fastcache v1.6.0 // indirect 20 | github.com/ajg/form v1.5.1 // indirect 21 | github.com/beorn7/perks v1.0.1 // indirect 22 | github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect 23 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 24 | github.com/cockroachdb/errors v1.9.1 // indirect 25 | github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect 26 | github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 // indirect 27 | github.com/cockroachdb/redact v1.1.3 // indirect 28 | github.com/davecgh/go-spew v1.1.1 // indirect 29 | github.com/deckarep/golang-set/v2 v2.1.0 // indirect 30 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect 31 | github.com/getsentry/sentry-go v0.18.0 // indirect 32 | github.com/go-ole/go-ole v1.2.1 // indirect 33 | github.com/go-stack/stack v1.8.1 // indirect 34 | github.com/gofrs/flock v0.8.1 // indirect 35 | github.com/gogo/protobuf v1.3.2 // indirect 36 | github.com/golang/protobuf v1.5.2 // indirect 37 | github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect 38 | github.com/gorilla/websocket v1.4.2 // indirect 39 | github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect 40 | github.com/klauspost/compress v1.15.15 // indirect 41 | github.com/kr/pretty v0.3.1 // indirect 42 | github.com/kr/text v0.2.0 // indirect 43 | github.com/mattn/go-runewidth v0.0.9 // indirect 44 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 45 | github.com/olekukonko/tablewriter v0.0.5 // indirect 46 | github.com/pkg/errors v0.9.1 // indirect 47 | github.com/pmezard/go-difflib v1.0.0 // indirect 48 | github.com/prometheus/client_golang v1.14.0 // indirect 49 | github.com/prometheus/client_model v0.3.0 // indirect 50 | github.com/prometheus/common v0.39.0 // indirect 51 | github.com/prometheus/procfs v0.9.0 // indirect 52 | github.com/rogpeppe/go-internal v1.9.0 // indirect 53 | github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect 54 | github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect 55 | github.com/tklauser/go-sysconf v0.3.5 // indirect 56 | github.com/tklauser/numcpus v0.2.2 // indirect 57 | go.uber.org/atomic v1.7.0 // indirect 58 | go.uber.org/multierr v1.6.0 // indirect 59 | golang.org/x/crypto v0.1.0 // indirect 60 | golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect 61 | golang.org/x/sys v0.6.0 // indirect 62 | golang.org/x/text v0.8.0 // indirect 63 | google.golang.org/protobuf v1.28.1 // indirect 64 | gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect 65 | gopkg.in/yaml.v3 v3.0.1 // indirect 66 | ) 67 | 68 | replace github.com/ethereum/go-ethereum v1.11.6 => github.com/ethereum-optimism/op-geth v1.101105.2-0.20230502202351-9cc072e922f6 69 | -------------------------------------------------------------------------------- /internal/alert/interpolator.go: -------------------------------------------------------------------------------- 1 | package alert 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | ) 8 | 9 | // TODO: add timestamp to the message 10 | const ( 11 | CodeBlockFmt = "```%s```" 12 | 13 | // slackMsgFmt ... Slack message format 14 | SlackMsgFmt = ` 15 | ⚠️🚨 Pessimism Alert: %s Invariant Invalidation 🚨⚠️ 16 | 17 | _Invariant invalidation conditions met_ 18 | 19 | _Network:_ %s 20 | _Session UUID:_ %s 21 | 22 | *Assessment Content:* 23 | %s 24 | ` 25 | ) 26 | 27 | // Interpolator ... Interface for interpolating messages 28 | type Interpolator interface { 29 | InterpolateSlackMessage(sUUID core.SUUID, message string) string 30 | } 31 | 32 | // interpolator ... Interpolator implementation 33 | type interpolator struct{} 34 | 35 | // NewInterpolator ... Initializer 36 | func NewInterpolator() Interpolator { 37 | return &interpolator{} 38 | } 39 | 40 | // InterpolateSlackMessage ... Interpolates a slack message with the given invariant session UUID and message 41 | func (*interpolator) InterpolateSlackMessage(sUUID core.SUUID, message string) string { 42 | return fmt.Sprintf(SlackMsgFmt, 43 | sUUID.PID.InvType().String(), 44 | sUUID.PID.Network(), 45 | sUUID.String(), 46 | fmt.Sprintf(CodeBlockFmt, message)) 47 | } 48 | -------------------------------------------------------------------------------- /internal/alert/interpolator_test.go: -------------------------------------------------------------------------------- 1 | package alert_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/base-org/pessimism/internal/alert" 7 | "github.com/base-org/pessimism/internal/core" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func Test_InterpolateSlackMessage(t *testing.T) { 12 | sUUID := core.NilSUUID() 13 | 14 | msg := "Friedrich Nietzsche" 15 | 16 | expected := "\n\t⚠️🚨 Pessimism Alert: unknown Invariant Invalidation 🚨⚠️\n\n\t_Invariant invalidation conditions met_\n\n\t_Network:_ unknown\n\t_Session UUID:_ unknown:unknown:unknown::000000000\n\n\t*Assessment Content:* \n\t```Friedrich Nietzsche```\t\n\t" 17 | 18 | actual := alert.NewInterpolator(). 19 | InterpolateSlackMessage(sUUID, msg) 20 | 21 | assert.Equal(t, expected, actual, "should be equal") 22 | } 23 | -------------------------------------------------------------------------------- /internal/alert/manager.go: -------------------------------------------------------------------------------- 1 | //go:generate mockgen -package mocks --destination ../mocks/alert_manager.go --mock_names Manager=AlertManager . Manager 2 | 3 | package alert 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | 9 | "github.com/base-org/pessimism/internal/client" 10 | "github.com/base-org/pessimism/internal/core" 11 | "github.com/base-org/pessimism/internal/logging" 12 | "go.uber.org/zap" 13 | ) 14 | 15 | // Manager ... Interface for alert manager 16 | type Manager interface { 17 | AddInvariantSession(core.SUUID, core.AlertDestination) error 18 | Transit() chan core.Alert 19 | 20 | core.Subsystem 21 | } 22 | 23 | // alertManager ... Alert manager implementation 24 | type alertManager struct { 25 | ctx context.Context 26 | cancel context.CancelFunc 27 | 28 | sc client.SlackClient 29 | store Store 30 | interpolator Interpolator 31 | 32 | alertTransit chan core.Alert 33 | } 34 | 35 | // NewManager ... Instantiates a new alert manager 36 | func NewManager(ctx context.Context, sc client.SlackClient) Manager { 37 | // NOTE - Consider constructing dependencies in higher level 38 | // abstraction and passing them in 39 | 40 | ctx, cancel := context.WithCancel(ctx) 41 | 42 | am := &alertManager{ 43 | ctx: ctx, 44 | cancel: cancel, 45 | 46 | sc: sc, 47 | interpolator: NewInterpolator(), 48 | store: NewStore(), 49 | alertTransit: make(chan core.Alert), 50 | } 51 | 52 | return am 53 | } 54 | 55 | // AddInvariantSession ... Adds an invariant session to the alert manager store 56 | func (am *alertManager) AddInvariantSession(sUUID core.SUUID, alertDestination core.AlertDestination) error { 57 | return am.store.AddAlertDestination(sUUID, alertDestination) 58 | } 59 | 60 | // Transit ... Returns inter-subsystem transit channel for receiving alerts 61 | func (am *alertManager) Transit() chan core.Alert { 62 | return am.alertTransit 63 | } 64 | 65 | // handleSlackPost ... Handles posting an alert to slack channel 66 | func (am *alertManager) handleSlackPost(alert core.Alert) error { 67 | slackMsg := am.interpolator.InterpolateSlackMessage(alert.SUUID, alert.Content) 68 | 69 | resp, err := am.sc.PostData(am.ctx, slackMsg) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | if !resp.Ok && resp.Err != "" { 75 | return fmt.Errorf(resp.Err) 76 | } 77 | 78 | return nil 79 | } 80 | 81 | // EventLoop ... Event loop for alert manager subsystem 82 | func (am *alertManager) EventLoop() error { 83 | logger := logging.WithContext(am.ctx) 84 | 85 | for { 86 | select { 87 | case <-am.ctx.Done(): 88 | return nil 89 | 90 | case alert := <-am.alertTransit: 91 | logger.Info("received alert", 92 | zap.String(core.SUUIDKey, alert.SUUID.String())) 93 | 94 | alertDest, err := am.store.GetAlertDestination(alert.SUUID) 95 | if err != nil { 96 | logger.Error("Could not determine alerting destination", zap.Error(err)) 97 | continue 98 | } 99 | 100 | switch alertDest { 101 | case core.Slack: // TODO: add more alert destinations 102 | logger.Debug("Attempting to post alert to slack") 103 | 104 | err := am.handleSlackPost(alert) 105 | if err != nil { 106 | logger.Error("Could not post alert to slack", zap.Error(err)) 107 | } 108 | 109 | case core.ThirdParty: 110 | logger.Error("Attempting to post alert to third_party which is not yet supported") 111 | 112 | default: 113 | logger.Error("Attempting to post alert to unknown destination", 114 | zap.String("destination", alertDest.String())) 115 | } 116 | } 117 | } 118 | } 119 | 120 | func (am *alertManager) Shutdown() error { 121 | am.cancel() 122 | return nil 123 | } 124 | -------------------------------------------------------------------------------- /internal/alert/store.go: -------------------------------------------------------------------------------- 1 | package alert 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | ) 8 | 9 | // TODO(#81): No Support for Multiple Alerting Destinations for an Invariant Session 10 | 11 | // Store ... Interface for alert store 12 | // NOTE - This is a simple in-memory store, using this interface 13 | // we can easily swap it out for a persistent store 14 | type Store interface { 15 | AddAlertDestination(core.SUUID, core.AlertDestination) error 16 | GetAlertDestination(sUUID core.SUUID) (core.AlertDestination, error) 17 | } 18 | 19 | // store ... Alert store implementation 20 | type store struct { 21 | invariantstore map[core.SUUID]core.AlertDestination 22 | } 23 | 24 | // Newstore ... Initializer 25 | func NewStore() Store { 26 | return &store{ 27 | invariantstore: make(map[core.SUUID]core.AlertDestination), 28 | } 29 | } 30 | 31 | // AddAlertDestination ... Adds an alert destination for the given invariant session UUID 32 | // NOTE - There can only be one alert destination per invariant session UUID 33 | func (am *store) AddAlertDestination(sUUID core.SUUID, 34 | alertDestination core.AlertDestination) error { 35 | if _, exists := am.invariantstore[sUUID]; exists { 36 | return fmt.Errorf("alert destination already exists for invariant session %s", sUUID.String()) 37 | } 38 | 39 | am.invariantstore[sUUID] = alertDestination 40 | return nil 41 | } 42 | 43 | // GetAlertDestination ... Returns the alert destination for the given invariant session UUID 44 | func (am *store) GetAlertDestination(sUUID core.SUUID) (core.AlertDestination, error) { 45 | alertDestination, exists := am.invariantstore[sUUID] 46 | if !exists { 47 | return 0, fmt.Errorf("alert destination does not exist for invariant session %s", sUUID.String()) 48 | } 49 | 50 | return alertDestination, nil 51 | } 52 | -------------------------------------------------------------------------------- /internal/alert/store_test.go: -------------------------------------------------------------------------------- 1 | package alert_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/base-org/pessimism/internal/alert" 8 | "github.com/base-org/pessimism/internal/core" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func Test_Store(t *testing.T) { 13 | var tests = []struct { 14 | name string 15 | description string 16 | testLogic func(t *testing.T) 17 | }{ 18 | { 19 | name: "Test Get Alert Destintation Success", 20 | description: "Test GetAlertDestination", 21 | testLogic: func(t *testing.T) { 22 | am := alert.NewStore() 23 | 24 | sUUID := core.MakeSUUID(core.Layer1, core.Live, core.BalanceEnforcement) 25 | alertDestination := core.Slack 26 | 27 | err := am.AddAlertDestination(sUUID, alertDestination) 28 | assert.NoError(t, err, "failed to add alert destination") 29 | 30 | actualAlertDest, err := am.GetAlertDestination(sUUID) 31 | assert.NoError(t, err, "failed to get alert destination") 32 | assert.Equal(t, alertDestination, actualAlertDest, "alert destination mismatch") 33 | }, 34 | }, 35 | { 36 | name: "Test Add Alert Destination Success", 37 | description: "Test adding of arbitrary alert destinations", 38 | testLogic: func(t *testing.T) { 39 | am := alert.NewStore() 40 | 41 | sUUID := core.MakeSUUID(core.Layer1, core.Live, core.BalanceEnforcement) 42 | alertDestination := core.Slack 43 | 44 | err := am.AddAlertDestination(sUUID, alertDestination) 45 | assert.NoError(t, err, "failed to add alert destination") 46 | 47 | // add again 48 | err = am.AddAlertDestination(sUUID, alertDestination) 49 | assert.Error(t, err, "failed to add alert destination") 50 | }, 51 | }, 52 | { 53 | name: "Test NewStore", 54 | description: "Test NewStore logic", 55 | testLogic: func(t *testing.T) { 56 | am := alert.NewStore() 57 | assert.NotNil(t, am, "failed to instantiate alert store") 58 | }, 59 | }, 60 | } 61 | 62 | for i, test := range tests { 63 | t.Run(fmt.Sprintf("%s:%d", test.name, i), func(t *testing.T) { 64 | test.testLogic(t) 65 | }) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /internal/api/handlers/handlers.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | pess_middleware "github.com/base-org/pessimism/internal/api/handlers/middleware" 8 | "github.com/base-org/pessimism/internal/api/service" 9 | "github.com/base-org/pessimism/internal/logging" 10 | "github.com/go-chi/chi" 11 | chi_middleware "github.com/go-chi/chi/middleware" 12 | ) 13 | 14 | type Handlers interface { 15 | HealthCheck(w http.ResponseWriter, r *http.Request) 16 | RunInvariant(w http.ResponseWriter, r *http.Request) 17 | 18 | ServeHTTP(w http.ResponseWriter, r *http.Request) 19 | } 20 | 21 | // PessimismHandler ... Server handler logic 22 | type PessimismHandler struct { 23 | ctx context.Context 24 | service service.Service 25 | router *chi.Mux 26 | } 27 | 28 | type Route = string 29 | 30 | const ( 31 | healthRoute = "/health" 32 | invariantRoute = "/v0/invariant" 33 | ) 34 | 35 | // New ... Initializer 36 | func New(ctx context.Context, service service.Service) (Handlers, error) { 37 | handlers := &PessimismHandler{ctx: ctx, service: service} 38 | router := chi.NewRouter() 39 | 40 | router.Use(chi_middleware.Recoverer) 41 | 42 | router.Use(pess_middleware.InjectedLogging(logging.NoContext())) 43 | 44 | registerEndpoint(healthRoute, router.Get, handlers.HealthCheck) 45 | registerEndpoint(invariantRoute, router.Post, handlers.RunInvariant) 46 | 47 | handlers.router = router 48 | 49 | return handlers, nil 50 | } 51 | 52 | // ServeHTTP ... Serves a http request given a response builder and request 53 | func (ph *PessimismHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 54 | ph.router.ServeHTTP(w, r) 55 | } 56 | 57 | // registerEndpoint ... Registers an endpoint to the router for a specified method type and handlerFunction 58 | func registerEndpoint(endpoint string, routeMethod func(pattern string, handlerFn http.HandlerFunc), 59 | handlerFunc func(w http.ResponseWriter, r *http.Request)) { 60 | routeMethod(endpoint, http.HandlerFunc(handlerFunc).ServeHTTP) 61 | } 62 | -------------------------------------------------------------------------------- /internal/api/handlers/handlers_test.go: -------------------------------------------------------------------------------- 1 | package handlers_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/base-org/pessimism/internal/api/handlers" 9 | "github.com/base-org/pessimism/internal/core" 10 | "github.com/base-org/pessimism/internal/mocks" 11 | "github.com/golang/mock/gomock" 12 | ) 13 | 14 | func testSUUID1() core.SUUID { 15 | return core.MakeSUUID(1, 1, 1) 16 | } 17 | 18 | func testError1() error { 19 | return fmt.Errorf("test error 1") 20 | } 21 | 22 | type testSuite struct { 23 | mockSvc mocks.MockService 24 | 25 | testHandler handlers.Handlers 26 | ctrl *gomock.Controller 27 | } 28 | 29 | func createTestSuite(t *testing.T) testSuite { 30 | ctrl := gomock.NewController(t) 31 | 32 | mockSvc := mocks.NewMockService(ctrl) 33 | testHandler, err := handlers.New(context.Background(), mockSvc) 34 | 35 | if err != nil { 36 | panic(err) 37 | } 38 | 39 | return testSuite{ 40 | mockSvc: *mockSvc, 41 | testHandler: testHandler, 42 | ctrl: ctrl, 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /internal/api/handlers/health.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/go-chi/render" 7 | ) 8 | 9 | // HealthCheck ... Handle health check 10 | func (ph *PessimismHandler) HealthCheck(w http.ResponseWriter, r *http.Request) { 11 | render.JSON(w, r, ph.service.CheckHealth()) 12 | } 13 | -------------------------------------------------------------------------------- /internal/api/handlers/health_test.go: -------------------------------------------------------------------------------- 1 | package handlers_test 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/http/httptest" 9 | "testing" 10 | 11 | "github.com/base-org/pessimism/internal/api/models" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | const ( 16 | testAddress = "http://abc.xyz" 17 | ) 18 | 19 | func Test_HealthCheck(t *testing.T) { 20 | 21 | var tests = []struct { 22 | name string 23 | description string 24 | function string 25 | 26 | constructionLogic func() testSuite 27 | testLogic func(*testing.T, testSuite) 28 | }{ 29 | { 30 | name: "Successful Health Check", 31 | description: "When GetHealth is called provided a healthy application, a healthy check should be rendered", 32 | function: "GetHealth", 33 | 34 | constructionLogic: func() testSuite { 35 | ts := createTestSuite(t) 36 | ts.mockSvc.EXPECT(). 37 | CheckHealth(). 38 | Return(&models.HealthCheck{ 39 | Healthy: true, 40 | ChainConnectionStatus: models.ChainConnectionStatus{ 41 | IsL1Healthy: true, 42 | IsL2Healthy: true, 43 | }}). 44 | Times(1) 45 | 46 | return ts 47 | }, 48 | 49 | testLogic: func(t *testing.T, ts testSuite) { 50 | w := httptest.NewRecorder() 51 | r := httptest.NewRequest(http.MethodGet, testAddress, nil) 52 | 53 | ts.testHandler.HealthCheck(w, r) 54 | res := w.Result() 55 | 56 | data, err := io.ReadAll(res.Body) 57 | if err != nil { 58 | t.Errorf("Error: %v", err) 59 | } 60 | 61 | actualHc := &models.HealthCheck{} 62 | err = json.Unmarshal(data, actualHc) 63 | 64 | assert.NoError(t, err) 65 | assert.True(t, actualHc.Healthy) 66 | assert.True(t, actualHc.ChainConnectionStatus.IsL1Healthy) 67 | assert.True(t, actualHc.ChainConnectionStatus.IsL2Healthy) 68 | }, 69 | }, 70 | { 71 | name: "Failed Health Check", 72 | description: "When GetHealth is called provided a unhealthy application, an unhealthy check should be rendered", 73 | function: "GetHealth", 74 | 75 | constructionLogic: func() testSuite { 76 | ts := createTestSuite(t) 77 | ts.mockSvc.EXPECT(). 78 | CheckHealth(). 79 | Return(&models.HealthCheck{ 80 | Healthy: false, 81 | ChainConnectionStatus: models.ChainConnectionStatus{ 82 | IsL1Healthy: false, 83 | IsL2Healthy: true, 84 | }}). 85 | Times(1) 86 | 87 | return ts 88 | }, 89 | 90 | testLogic: func(t *testing.T, ts testSuite) { 91 | w := httptest.NewRecorder() 92 | r := httptest.NewRequest(http.MethodGet, testAddress, nil) 93 | 94 | ts.testHandler.HealthCheck(w, r) 95 | res := w.Result() 96 | 97 | data, err := io.ReadAll(res.Body) 98 | if err != nil { 99 | t.Errorf("Error: %v", err) 100 | } 101 | 102 | actualHc := &models.HealthCheck{} 103 | err = json.Unmarshal(data, actualHc) 104 | 105 | assert.NoError(t, err) 106 | assert.False(t, actualHc.Healthy) 107 | assert.False(t, actualHc.ChainConnectionStatus.IsL1Healthy) 108 | assert.True(t, actualHc.ChainConnectionStatus.IsL2Healthy) 109 | }, 110 | }, 111 | } 112 | 113 | for i, tc := range tests { 114 | t.Run(fmt.Sprintf("%d-%s-%s", i, tc.name, tc.function), func(t *testing.T) { 115 | testMeta := tc.constructionLogic() 116 | tc.testLogic(t, testMeta) 117 | }) 118 | 119 | } 120 | 121 | } 122 | -------------------------------------------------------------------------------- /internal/api/handlers/invariant.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | 7 | "github.com/base-org/pessimism/internal/api/models" 8 | "github.com/base-org/pessimism/internal/logging" 9 | "github.com/go-chi/render" 10 | "go.uber.org/zap" 11 | ) 12 | 13 | func renderInvariantResponse(w http.ResponseWriter, r *http.Request, 14 | ir *models.InvResponse) { 15 | w.WriteHeader(ir.Code) 16 | render.JSON(w, r, ir) 17 | } 18 | 19 | // RunInvariant ... Handle invariant run request 20 | func (ph *PessimismHandler) RunInvariant(w http.ResponseWriter, r *http.Request) { 21 | var body models.InvRequestBody 22 | 23 | if err := json.NewDecoder(r.Body).Decode(&body); err != nil { 24 | logging.WithContext(ph.ctx). 25 | Error("Could not unmarshal request", zap.Error(err)) 26 | 27 | renderInvariantResponse(w, r, 28 | models.NewInvUnmarshalErrResp()) 29 | return 30 | } 31 | 32 | sUUID, err := ph.service.ProcessInvariantRequest(body) 33 | if err != nil { 34 | logging.WithContext(ph.ctx). 35 | Error("Could not process invariant request", zap.Error(err)) 36 | 37 | renderInvariantResponse(w, r, models.NewInvNoProcessInvResp()) 38 | return 39 | } 40 | 41 | renderInvariantResponse(w, r, models.NewInvAcceptedResp(sUUID)) 42 | } 43 | -------------------------------------------------------------------------------- /internal/api/handlers/invariant_test.go: -------------------------------------------------------------------------------- 1 | package handlers_test 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "net/http/httptest" 10 | "testing" 11 | 12 | "github.com/base-org/pessimism/internal/api/models" 13 | "github.com/base-org/pessimism/internal/core" 14 | "github.com/golang/mock/gomock" 15 | "github.com/stretchr/testify/assert" 16 | ) 17 | 18 | func Test_ProcessInvariantRequest(t *testing.T) { 19 | 20 | var tests = []struct { 21 | name string 22 | description string 23 | function string 24 | 25 | constructionLogic func() testSuite 26 | testLogic func(*testing.T, testSuite) 27 | }{ 28 | { 29 | name: "Get Invariant Failure", 30 | description: "When provided a malformed request body, a failed decoding response should be returned", 31 | function: "RunInvariant", 32 | 33 | constructionLogic: func() testSuite { 34 | ts := createTestSuite(t) 35 | return ts 36 | }, 37 | 38 | testLogic: func(t *testing.T, ts testSuite) { 39 | w := httptest.NewRecorder() 40 | 41 | testBody := bytes.NewBuffer([]byte{0x42}) 42 | r := httptest.NewRequest(http.MethodGet, testAddress, testBody) 43 | 44 | ts.testHandler.RunInvariant(w, r) 45 | res := w.Result() 46 | 47 | data, err := io.ReadAll(res.Body) 48 | if err != nil { 49 | t.Errorf("Error: %v", err) 50 | } 51 | 52 | actualResp := &models.InvResponse{} 53 | err = json.Unmarshal(data, actualResp) 54 | 55 | assert.NoError(t, err) 56 | assert.Equal(t, models.NewInvUnmarshalErrResp(), actualResp) 57 | }, 58 | }, 59 | { 60 | name: "Process Invariant Failure", 61 | description: "When provided an internal error occurs, a failed processing response should be returned", 62 | function: "RunInvariant", 63 | 64 | constructionLogic: func() testSuite { 65 | ts := createTestSuite(t) 66 | 67 | ts.mockSvc.EXPECT(). 68 | ProcessInvariantRequest(gomock.Any()). 69 | Return(core.NilSUUID(), testError1()). 70 | Times(1) 71 | 72 | return ts 73 | }, 74 | 75 | testLogic: func(t *testing.T, ts testSuite) { 76 | w := httptest.NewRecorder() 77 | 78 | testBody, _ := json.Marshal(models.InvRequestBody{Method: "run"}) 79 | 80 | testBytes := bytes.NewBuffer(testBody) 81 | r := httptest.NewRequest(http.MethodGet, testAddress, testBytes) 82 | 83 | ts.testHandler.RunInvariant(w, r) 84 | res := w.Result() 85 | 86 | data, err := io.ReadAll(res.Body) 87 | if err != nil { 88 | t.Errorf("Error: %v", err) 89 | } 90 | 91 | actualResp := &models.InvResponse{} 92 | err = json.Unmarshal(data, actualResp) 93 | 94 | assert.NoError(t, err) 95 | assert.Equal(t, models.NewInvNoProcessInvResp(), actualResp) 96 | }, 97 | }, 98 | { 99 | name: "Process Invariant Success", 100 | description: "When an invariant is successfully processed, a suuid should be rendered", 101 | function: "RunInvariant", 102 | 103 | constructionLogic: func() testSuite { 104 | ts := createTestSuite(t) 105 | 106 | ts.mockSvc.EXPECT(). 107 | ProcessInvariantRequest(gomock.Any()). 108 | Return(testSUUID1(), nil). 109 | Times(1) 110 | 111 | return ts 112 | }, 113 | 114 | testLogic: func(t *testing.T, ts testSuite) { 115 | w := httptest.NewRecorder() 116 | 117 | testBody, _ := json.Marshal(models.InvRequestBody{Method: "run"}) 118 | 119 | testBytes := bytes.NewBuffer(testBody) 120 | r := httptest.NewRequest(http.MethodGet, testAddress, testBytes) 121 | 122 | ts.testHandler.RunInvariant(w, r) 123 | res := w.Result() 124 | 125 | data, err := io.ReadAll(res.Body) 126 | if err != nil { 127 | t.Errorf("Error: %v", err) 128 | } 129 | 130 | actualResp := &models.InvResponse{} 131 | err = json.Unmarshal(data, actualResp) 132 | 133 | assert.NoError(t, err) 134 | 135 | assert.Equal(t, actualResp.Status, models.OK) 136 | assert.Equal(t, actualResp.Code, http.StatusAccepted) 137 | assert.Contains(t, actualResp.Result[core.SUUIDKey], testSUUID1().PID.String()) 138 | }, 139 | }, 140 | } 141 | 142 | for i, tc := range tests { 143 | t.Run(fmt.Sprintf("%d-%s-%s", i, tc.name, tc.function), func(t *testing.T) { 144 | testMeta := tc.constructionLogic() 145 | tc.testLogic(t, testMeta) 146 | }) 147 | 148 | } 149 | 150 | } 151 | -------------------------------------------------------------------------------- /internal/api/handlers/middleware/middleware.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "go.uber.org/zap" 8 | ) 9 | 10 | type HeaderParam = string 11 | 12 | const ( 13 | ContentLength HeaderParam = "Content-Length" 14 | UserAgent HeaderParam = "User-Agent" 15 | Host HeaderParam = "Host" 16 | ContentType HeaderParam = "Content-Type" 17 | ) 18 | 19 | // InjectedLogging uses logging middleware 20 | func InjectedLogging(logger *zap.Logger) func(http.Handler) http.Handler { 21 | return func(next http.Handler) http.Handler { 22 | fn := func(w http.ResponseWriter, r *http.Request) { 23 | defer func() { 24 | if err := recover(); err != nil { 25 | w.WriteHeader(http.StatusInternalServerError) 26 | logger.Error("Failure occurred during request processing") 27 | } 28 | }() 29 | 30 | userAgent := r.Header.Get(UserAgent) 31 | if len(userAgent) == 0 { 32 | userAgent = "-" 33 | } 34 | 35 | contentLength := r.Header.Get(ContentLength) 36 | if len(contentLength) == 0 { 37 | contentLength = "-" 38 | } 39 | 40 | host := r.Header.Get(Host) 41 | if host == "" { 42 | host = r.RemoteAddr 43 | } 44 | 45 | contentType := r.Header.Get(ContentType) 46 | if contentType == "" { 47 | contentType = "-" 48 | } 49 | 50 | start := time.Now() 51 | next.ServeHTTP(w, r) 52 | 53 | logger.Info("HTTP request received", 54 | zap.String("method", r.Method), zap.String("path", r.URL.EscapedPath()), 55 | zap.Duration("duration", time.Since(start)), zap.String("content_type", contentType), 56 | zap.String("content_length", contentLength), 57 | zap.String("user_agent", userAgent), zap.String("host", host), 58 | ) 59 | } 60 | 61 | return http.HandlerFunc(fn) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /internal/api/models/health.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // HealthCheck ... Returns health status of server 8 | // Currently just returns True 9 | type HealthCheck struct { 10 | Timestamp time.Time 11 | Healthy bool 12 | ChainConnectionStatus ChainConnectionStatus 13 | } 14 | 15 | // ChainConnectionStatus ... Used to display health status of each node connection 16 | type ChainConnectionStatus struct { 17 | IsL1Healthy bool 18 | IsL2Healthy bool 19 | } 20 | -------------------------------------------------------------------------------- /internal/api/models/invariant.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "math/big" 5 | "time" 6 | 7 | "github.com/base-org/pessimism/internal/core" 8 | ) 9 | 10 | // InvariantMethod ... Represents the invariant operation method 11 | type InvariantMethod int 12 | 13 | const ( 14 | Run InvariantMethod = iota 15 | // NOTE - Update is not implemented yet 16 | Update 17 | // NOTE - Stop is not implemented yet 18 | Stop 19 | ) 20 | 21 | func StringToInvariantMethod(s string) InvariantMethod { 22 | switch s { 23 | case "run": 24 | return Run 25 | case "update": 26 | return Update 27 | case "stop": 28 | return Stop 29 | default: 30 | return Run 31 | } 32 | } 33 | 34 | // InvResponseStatus ... Represents the invariant operation response status 35 | type InvResponseStatus string 36 | 37 | const ( 38 | OK InvResponseStatus = "OK" 39 | NotOK InvResponseStatus = "NOTOK" 40 | ) 41 | 42 | // InvRequestParams ... Request params for invariant operation 43 | type InvRequestParams struct { 44 | Network string `json:"network"` 45 | PType string `json:"pipeline_type"` 46 | InvType string `json:"type"` 47 | 48 | StartHeight *big.Int `json:"start_height"` 49 | EndHeight *big.Int `json:"end_height"` 50 | 51 | SessionParams map[string]interface{} `json:"invariant_params"` 52 | // TODO(#81): No Support for Multiple Alerting Destinations for an Invariant Session 53 | AlertingDest string `json:"alert_destination"` 54 | } 55 | 56 | // AlertingDestType ... Returns the alerting destination type 57 | func (irp *InvRequestParams) AlertingDestType() core.AlertDestination { 58 | return core.StringToAlertingDestType(irp.AlertingDest) 59 | } 60 | 61 | // NetworkType ... Returns the network type 62 | func (irp *InvRequestParams) NetworkType() core.Network { 63 | return core.StringToNetwork(irp.Network) 64 | } 65 | 66 | // PiplineType ... Returns the pipeline type 67 | func (irp *InvRequestParams) PiplineType() core.PipelineType { 68 | return core.StringToPipelineType(irp.PType) 69 | } 70 | 71 | // InvariantType ... Returns the invariant type 72 | func (irp *InvRequestParams) InvariantType() core.InvariantType { 73 | return core.StringToInvariantType(irp.InvType) 74 | } 75 | 76 | // GeneratePipelineConfig ... Generates a pipeline config using the request params 77 | func (irp *InvRequestParams) GeneratePipelineConfig(endpoint string, pollInterval time.Duration, 78 | regType core.RegisterType) *core.PipelineConfig { 79 | return &core.PipelineConfig{ 80 | Network: irp.NetworkType(), 81 | DataType: regType, 82 | PipelineType: irp.PiplineType(), 83 | ClientConfig: &core.ClientConfig{ 84 | RPCEndpoint: endpoint, 85 | PollInterval: pollInterval, 86 | StartHeight: irp.StartHeight, 87 | EndHeight: irp.EndHeight, 88 | }, 89 | } 90 | } 91 | 92 | // SessionConfig ... Generates a session config using the request params 93 | func (irp *InvRequestParams) SessionConfig() *core.SessionConfig { 94 | return &core.SessionConfig{ 95 | AlertDest: irp.AlertingDestType(), 96 | Type: irp.InvariantType(), 97 | Params: irp.SessionParams, 98 | } 99 | } 100 | 101 | // InvRequestBody ... Request body for invariant operation request 102 | type InvRequestBody struct { 103 | Method string `json:"method"` 104 | Params InvRequestParams `json:"params"` 105 | } 106 | 107 | // MethodType ... Returns the invariant method type 108 | func (irb *InvRequestBody) MethodType() InvariantMethod { 109 | return StringToInvariantMethod(irb.Method) 110 | } 111 | 112 | // InvResult ... Result of invariant operation 113 | type InvResult = map[string]string 114 | 115 | // InvResponse ... Response for invariant operation request 116 | type InvResponse struct { 117 | Code int `json:"status_code"` 118 | Status InvResponseStatus `json:"status"` 119 | 120 | Result InvResult `json:"result"` 121 | Error string `json:"error"` 122 | } 123 | -------------------------------------------------------------------------------- /internal/api/models/models.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | ) 8 | 9 | // NewInvAcceptedResp ...Returns an invariant response with status accepted 10 | func NewInvAcceptedResp(id core.SUUID) *InvResponse { 11 | return &InvResponse{ 12 | Status: OK, 13 | Code: http.StatusAccepted, 14 | Result: InvResult{core.SUUIDKey: id.String()}, 15 | } 16 | } 17 | 18 | // NewInvUnmarshalErrResp ... New unmarshal error response construction 19 | func NewInvUnmarshalErrResp() *InvResponse { 20 | return &InvResponse{ 21 | Status: NotOK, 22 | Code: http.StatusBadRequest, 23 | Error: "could not unmarshal request body", 24 | } 25 | } 26 | 27 | // NewInvNoProcessInvResp ... New internal processing response error 28 | func NewInvNoProcessInvResp() *InvResponse { 29 | return &InvResponse{ 30 | Status: NotOK, 31 | Code: http.StatusInternalServerError, 32 | Error: "error processing invariant request", 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /internal/api/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "time" 8 | 9 | "github.com/base-org/pessimism/internal/api/handlers" 10 | "github.com/base-org/pessimism/internal/logging" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | // Config ... Server configuration options 15 | type Config struct { 16 | Host string 17 | Port int 18 | KeepAlive int 19 | ReadTimeout int 20 | WriteTimeout int 21 | ShutdownTimeout int 22 | } 23 | 24 | // Server ... Server representation struct 25 | type Server struct { 26 | Cfg *Config 27 | serverHTTP *http.Server 28 | } 29 | 30 | // New ... Initializer 31 | func New(ctx context.Context, cfg *Config, apiHandlers handlers.Handlers) (*Server, func(), error) { 32 | restServer := initializeServer(cfg, apiHandlers) 33 | 34 | stop := func() { 35 | logging.WithContext(ctx).Info("starting to shutdown REST API HTTP server") 36 | 37 | ctx, cancel := context.WithTimeout(ctx, time.Duration(cfg.ShutdownTimeout)*time.Second) 38 | if err := restServer.serverHTTP.Shutdown(ctx); err != nil { 39 | logging.WithContext(ctx).Error("failed to shutdown REST API HTTP server") 40 | panic(err) 41 | } 42 | 43 | defer cancel() 44 | } 45 | 46 | return restServer, stop, nil 47 | } 48 | 49 | // spawnServer ... Starts a listen and serve API routine 50 | func spawnServer(server *Server) { 51 | logging.NoContext().Info("Starting REST API HTTP server", 52 | zap.String("address", server.serverHTTP.Addr)) 53 | 54 | if err := server.serverHTTP.ListenAndServe(); err != http.ErrServerClosed { 55 | logging.NoContext().Error("failed to run REST API HTTP server", zap.String("address", server.serverHTTP.Addr)) 56 | panic(err) 57 | } 58 | } 59 | 60 | func (s *Server) Start() { 61 | go spawnServer(s) 62 | } 63 | 64 | // initializeServer ... Initializes server struct object 65 | func initializeServer(config *Config, handler http.Handler) *Server { 66 | return &Server{ 67 | Cfg: config, 68 | serverHTTP: &http.Server{ 69 | Addr: fmt.Sprintf("%s:%d", config.Host, config.Port), 70 | Handler: handler, 71 | IdleTimeout: time.Duration(config.KeepAlive) * time.Second, 72 | ReadTimeout: time.Duration(config.ReadTimeout) * time.Second, 73 | WriteTimeout: time.Duration(config.WriteTimeout) * time.Second, 74 | }, 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /internal/api/service/health.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/base-org/pessimism/internal/api/models" 7 | "github.com/base-org/pessimism/internal/logging" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | // CheckHealth ... Returns health check for server 12 | func (svc *PessimismService) CheckHealth() *models.HealthCheck { 13 | hc := &models.ChainConnectionStatus{ 14 | IsL1Healthy: svc.CheckETHRPCHealth(svc.cfg.L1RpcEndpoint), 15 | IsL2Healthy: svc.CheckETHRPCHealth(svc.cfg.L2RpcEndpoint), 16 | } 17 | 18 | healthy := hc.IsL1Healthy && hc.IsL2Healthy 19 | 20 | return &models.HealthCheck{ 21 | Timestamp: time.Now(), 22 | Healthy: healthy, 23 | ChainConnectionStatus: *hc, 24 | } 25 | } 26 | 27 | func (svc *PessimismService) CheckETHRPCHealth(url string) bool { 28 | logger := logging.WithContext(svc.ctx) 29 | 30 | err := svc.ethClient.DialContext(svc.ctx, url) 31 | if err != nil { 32 | logger.Error("error conntecting to %s", zap.String("url", url)) 33 | return false 34 | } 35 | 36 | _, err = svc.ethClient.HeaderByNumber(svc.ctx, nil) 37 | if err != nil { 38 | logger.Error("error connecting to url", zap.String("url", url)) 39 | return false 40 | } 41 | 42 | logger.Debug("successfully connected", zap.String("url", url)) 43 | return true 44 | } 45 | -------------------------------------------------------------------------------- /internal/api/service/health_test.go: -------------------------------------------------------------------------------- 1 | package service_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | svc "github.com/base-org/pessimism/internal/api/service" 9 | "github.com/golang/mock/gomock" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func Test_GetHealth(t *testing.T) { 14 | ctrl := gomock.NewController(t) 15 | 16 | var tests = []struct { 17 | name string 18 | description string 19 | function string 20 | 21 | constructionLogic func() testSuite 22 | testLogic func(*testing.T, testSuite) 23 | }{ 24 | { 25 | name: "Get Health Success", 26 | description: "", 27 | function: "ProcessInvariantRequest", 28 | 29 | constructionLogic: func() testSuite { 30 | cfg := svc.Config{} 31 | ts := createTestSuite(ctrl, cfg) 32 | 33 | ts.mockEthClientInterface.EXPECT(). 34 | DialContext(context.Background(), gomock.Any()). 35 | Return(nil). 36 | AnyTimes() 37 | 38 | ts.mockService.EXPECT(). 39 | CheckETHRPCHealth(gomock.Any()). 40 | Return(true). 41 | AnyTimes() 42 | 43 | ts.mockEthClientInterface.EXPECT(). 44 | HeaderByNumber(gomock.Any(), gomock.Any()). 45 | Return(nil, nil). 46 | AnyTimes() 47 | 48 | return ts 49 | }, 50 | 51 | testLogic: func(t *testing.T, ts testSuite) { 52 | hc := ts.apiSvc.CheckHealth() 53 | 54 | assert.True(t, hc.Healthy) 55 | assert.True(t, hc.ChainConnectionStatus.IsL2Healthy) 56 | assert.True(t, hc.ChainConnectionStatus.IsL1Healthy) 57 | 58 | }, 59 | }, 60 | { 61 | name: "Get Unhealthy Response", 62 | description: "Emulates unhealthy rpc endpoints", 63 | function: "ProcessInvariantRequest", 64 | 65 | constructionLogic: func() testSuite { 66 | cfg := svc.Config{} 67 | ts := createTestSuite(ctrl, cfg) 68 | 69 | ts.mockEthClientInterface.EXPECT(). 70 | DialContext(gomock.Any(), gomock.Any()). 71 | Return(testErr1()). 72 | AnyTimes() 73 | 74 | ts.mockService.EXPECT(). 75 | CheckETHRPCHealth(gomock.Any()). 76 | Return(false). 77 | AnyTimes() 78 | 79 | ts.mockEthClientInterface.EXPECT(). 80 | HeaderByNumber(gomock.Any(), gomock.Any()). 81 | Return(nil, nil). 82 | AnyTimes() 83 | 84 | return ts 85 | }, 86 | 87 | testLogic: func(t *testing.T, ts testSuite) { 88 | hc := ts.apiSvc.CheckHealth() 89 | assert.False(t, hc.Healthy) 90 | assert.False(t, hc.ChainConnectionStatus.IsL2Healthy) 91 | assert.False(t, hc.ChainConnectionStatus.IsL1Healthy) 92 | }, 93 | }, 94 | } 95 | 96 | for i, tc := range tests { 97 | t.Run(fmt.Sprintf("%d-%s", i, tc.name), func(t *testing.T) { 98 | testMeta := tc.constructionLogic() 99 | tc.testLogic(t, testMeta) 100 | }) 101 | 102 | } 103 | 104 | } 105 | -------------------------------------------------------------------------------- /internal/api/service/invariant.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "github.com/base-org/pessimism/internal/api/models" 5 | "github.com/base-org/pessimism/internal/core" 6 | "github.com/base-org/pessimism/internal/engine/registry" 7 | ) 8 | 9 | // ProcessInvariantRequest ... Processes an invariant request type 10 | func (svc *PessimismService) ProcessInvariantRequest(ir models.InvRequestBody) (core.SUUID, error) { 11 | if ir.MethodType() == models.Run { // Deploy invariant session 12 | return svc.RunInvariantSession(ir.Params) 13 | } 14 | // TODO - Add support for other method types (ie. delete. update) 15 | 16 | return core.NilSUUID(), nil 17 | } 18 | 19 | // runInvariantSession ... Runs an invariant session provided 20 | func (svc *PessimismService) RunInvariantSession(params models.InvRequestParams) (core.SUUID, error) { 21 | inv, err := registry.GetInvariant(params.InvariantType(), params.SessionParams) 22 | if err != nil { 23 | return core.NilSUUID(), err 24 | } 25 | 26 | // TODO(#53): API Request Validation Submodule 27 | endpoint, err := svc.cfg.GetEndpointForNetwork(params.NetworkType()) 28 | if err != nil { 29 | return core.NilSUUID(), err 30 | } 31 | 32 | pollInterval, err := svc.cfg.GetPollIntervalForNetwork(params.NetworkType()) 33 | if err != nil { 34 | return core.NilSUUID(), err 35 | } 36 | 37 | pConfig := params.GeneratePipelineConfig(endpoint, pollInterval, inv.InputType()) 38 | sConfig := params.SessionConfig() 39 | 40 | sUUID, err := svc.m.StartInvSession(pConfig, sConfig) 41 | if err != nil { 42 | return core.NilSUUID(), err 43 | } 44 | 45 | return sUUID, nil 46 | } 47 | -------------------------------------------------------------------------------- /internal/api/service/service.go: -------------------------------------------------------------------------------- 1 | //go:generate mockgen -package mocks --destination ../../mocks/api_service.go . Service 2 | 3 | package service 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | "time" 9 | 10 | "github.com/base-org/pessimism/internal/api/models" 11 | "github.com/base-org/pessimism/internal/client" 12 | "github.com/base-org/pessimism/internal/core" 13 | "github.com/base-org/pessimism/internal/subsystem" 14 | ) 15 | 16 | // Config ... Used to store necessary API service config values 17 | type Config struct { 18 | L1RpcEndpoint string 19 | L2RpcEndpoint string 20 | L1PollInterval int 21 | L2PollInterval int 22 | } 23 | 24 | // GetEndpointForNetwork ... Returns config endpoint for network type 25 | func (cfg *Config) GetEndpointForNetwork(n core.Network) (string, error) { 26 | switch n { 27 | case core.Layer1: 28 | return cfg.L1RpcEndpoint, nil 29 | 30 | case core.Layer2: 31 | return cfg.L2RpcEndpoint, nil 32 | 33 | default: 34 | return "", fmt.Errorf("could not find endpoint for network %s", n.String()) 35 | } 36 | } 37 | 38 | // GetEndpointForNetwork ... Returns config poll-interval for network type 39 | func (cfg *Config) GetPollIntervalForNetwork(n core.Network) (time.Duration, error) { 40 | switch n { 41 | case core.Layer1: 42 | return time.Duration(cfg.L1PollInterval), nil 43 | 44 | case core.Layer2: 45 | return time.Duration(cfg.L2PollInterval), nil 46 | 47 | default: 48 | return 0, fmt.Errorf("could not find endpoint for network %s", n.String()) 49 | } 50 | } 51 | 52 | // Service ... Interface for API service 53 | type Service interface { 54 | ProcessInvariantRequest(ir models.InvRequestBody) (core.SUUID, error) 55 | RunInvariantSession(params models.InvRequestParams) (core.SUUID, error) 56 | 57 | CheckHealth() *models.HealthCheck 58 | CheckETHRPCHealth(url string) bool 59 | } 60 | 61 | // PessimismService ... API service 62 | type PessimismService struct { 63 | ctx context.Context 64 | cfg *Config 65 | ethClient client.EthClientInterface 66 | 67 | m subsystem.Manager 68 | } 69 | 70 | // New ... Initializer 71 | func New(ctx context.Context, cfg *Config, m subsystem.Manager, ethClient client.EthClientInterface) *PessimismService { 72 | return &PessimismService{ 73 | ctx: ctx, 74 | cfg: cfg, 75 | ethClient: ethClient, 76 | 77 | m: m, 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /internal/api/service/service_test.go: -------------------------------------------------------------------------------- 1 | package service_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | svc "github.com/base-org/pessimism/internal/api/service" 8 | "github.com/base-org/pessimism/internal/subsystem" 9 | 10 | "github.com/base-org/pessimism/internal/core" 11 | "github.com/base-org/pessimism/internal/mocks" 12 | "github.com/golang/mock/gomock" 13 | ) 14 | 15 | const ( 16 | testErrMsg1 = "69" 17 | testErrMsg2 = "420" 18 | testErrMsg3 = "666" 19 | ) 20 | 21 | type testSuite struct { 22 | testCfg svc.Config 23 | 24 | mockAlertMan *mocks.AlertManager 25 | mockEngineMan *mocks.EngineManager 26 | mockEtlMan *mocks.EtlManager 27 | mockService *mocks.MockService 28 | mockEthClientInterface *mocks.MockEthClientInterface 29 | 30 | apiSvc svc.Service 31 | mockCtrl *gomock.Controller 32 | } 33 | 34 | func testErr1() error { 35 | return fmt.Errorf(testErrMsg1) 36 | } 37 | func testErr2() error { 38 | return fmt.Errorf(testErrMsg2) 39 | } 40 | func testErr3() error { 41 | return fmt.Errorf(testErrMsg3) 42 | } 43 | 44 | func testSUUID1() core.SUUID { 45 | return core.MakeSUUID(1, 1, 1) 46 | } 47 | 48 | func createTestSuite(ctrl *gomock.Controller, cfg svc.Config) testSuite { 49 | engineManager := mocks.NewEngineManager(ctrl) 50 | etlManager := mocks.NewEtlManager(ctrl) 51 | alertManager := mocks.NewAlertManager(ctrl) 52 | serviceManager := mocks.NewMockService(ctrl) 53 | ethClientManager := mocks.NewMockEthClientInterface(ctrl) 54 | 55 | // NOTE - These tests should be migrated to the subsystem manager package 56 | // TODO(#76): No Subsystem Manager Tests 57 | m := subsystem.NewManager(context.Background(), etlManager, engineManager, alertManager) 58 | 59 | service := svc.New(context.Background(), &cfg, m, ethClientManager) 60 | return testSuite{ 61 | testCfg: cfg, 62 | 63 | mockAlertMan: alertManager, 64 | mockEngineMan: engineManager, 65 | mockEtlMan: etlManager, 66 | mockService: serviceManager, 67 | mockEthClientInterface: ethClientManager, 68 | 69 | apiSvc: service, 70 | mockCtrl: ctrl, 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /internal/app/app.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | 9 | "github.com/base-org/pessimism/internal/api/models" 10 | "github.com/base-org/pessimism/internal/api/server" 11 | "github.com/base-org/pessimism/internal/config" 12 | "github.com/base-org/pessimism/internal/core" 13 | "github.com/base-org/pessimism/internal/engine/registry" 14 | "github.com/base-org/pessimism/internal/logging" 15 | "github.com/base-org/pessimism/internal/subsystem" 16 | "go.uber.org/zap" 17 | ) 18 | 19 | // BootSession ... Application wrapper for InvRequestParams 20 | type BootSession = models.InvRequestParams 21 | 22 | // Application ... Pessimism app struct 23 | type Application struct { 24 | cfg *config.Config 25 | ctx context.Context 26 | 27 | sub subsystem.Manager 28 | server *server.Server 29 | } 30 | 31 | // New ... Initializer 32 | func New(ctx context.Context, cfg *config.Config, 33 | sub subsystem.Manager, server *server.Server) *Application { 34 | return &Application{ 35 | ctx: ctx, 36 | cfg: cfg, 37 | sub: sub, 38 | server: server, 39 | } 40 | } 41 | 42 | // Start ... Starts the application 43 | func (a *Application) Start() error { 44 | // Spawn subsystem event loop routines 45 | a.sub.StartEventRoutines(a.ctx) 46 | 47 | // Start the API server 48 | a.server.Start() 49 | return nil 50 | } 51 | 52 | // ListenForShutdown ... Handles and listens for shutdown 53 | func (a *Application) ListenForShutdown(stop func()) { 54 | done := <-a.End() // Blocks until an OS signal is received 55 | 56 | logging.WithContext(a.ctx). 57 | Info("Received shutdown OS signal", zap.String("signal", done.String())) 58 | stop() 59 | } 60 | 61 | // End ... Returns a channel that will receive an OS signal 62 | func (a *Application) End() <-chan os.Signal { 63 | sigs := make(chan os.Signal, 1) 64 | signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) 65 | return sigs 66 | } 67 | 68 | // BootStrap ... Bootstraps the application 69 | func (a *Application) BootStrap(sessions []BootSession) error { 70 | logger := logging.WithContext(a.ctx) 71 | 72 | for _, session := range sessions { 73 | inv, err := registry.GetInvariant(session.InvariantType(), session.SessionParams) 74 | if err != nil { 75 | return err 76 | } 77 | 78 | endpoint, err := a.cfg.SvcConfig.GetEndpointForNetwork(session.NetworkType()) 79 | if err != nil { 80 | return err 81 | } 82 | 83 | pollInterval, err := a.cfg.SvcConfig.GetPollIntervalForNetwork(session.NetworkType()) 84 | if err != nil { 85 | return err 86 | } 87 | 88 | pConfig := session.GeneratePipelineConfig(endpoint, pollInterval, inv.InputType()) 89 | sConfig := session.SessionConfig() 90 | 91 | sUUID, err := a.sub.StartInvSession(pConfig, sConfig) 92 | if err != nil { 93 | return err 94 | } 95 | 96 | logger.Info("invariant session started", 97 | zap.String(core.SUUIDKey, sUUID.String())) 98 | } 99 | return nil 100 | } 101 | -------------------------------------------------------------------------------- /internal/client/eth_client.go: -------------------------------------------------------------------------------- 1 | //go:generate mockgen -package mocks --destination ../mocks/eth_client.go . EthClientInterface 2 | 3 | package client 4 | 5 | /* 6 | NOTE 7 | geth client docs: https://pkg.go.dev/github.com/ethereum/go-ethereum/ethclient 8 | geth api docs: https://geth.ethereum.org/docs/rpc/server 9 | */ 10 | 11 | import ( 12 | "context" 13 | "math/big" 14 | 15 | "github.com/ethereum/go-ethereum" 16 | "github.com/ethereum/go-ethereum/common" 17 | "github.com/ethereum/go-ethereum/core/types" 18 | "github.com/ethereum/go-ethereum/ethclient" 19 | ) 20 | 21 | // TODO (#20) : Introduce optional Retry-able EthClient 22 | type EthClient struct { 23 | client *ethclient.Client 24 | } 25 | 26 | // EthClientInterface ... Provides interface wrapper for ethClient functions 27 | // Useful for mocking go-etheruem node client logic 28 | type EthClientInterface interface { 29 | DialContext(ctx context.Context, rawURL string) error 30 | HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) 31 | BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) 32 | 33 | BalanceAt(ctx context.Context, account common.Address, number *big.Int) (*big.Int, error) 34 | FilterLogs(ctx context.Context, query ethereum.FilterQuery) ([]types.Log, error) 35 | } 36 | 37 | // NewEthClient ... Initializer 38 | func NewEthClient() EthClientInterface { 39 | return &EthClient{ 40 | client: ðclient.Client{}, 41 | } 42 | } 43 | 44 | // DialContext ... Wraps go-etheruem node dialContext RPC creation 45 | func (ec *EthClient) DialContext(ctx context.Context, rawURL string) error { 46 | client, err := ethclient.DialContext(ctx, rawURL) 47 | 48 | if err != nil { 49 | return err 50 | } 51 | 52 | ec.client = client 53 | return nil 54 | } 55 | 56 | // HeaderByNumber ... Wraps go-ethereum node headerByNumber RPC call 57 | func (ec *EthClient) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) { 58 | return ec.client.HeaderByNumber(ctx, number) 59 | } 60 | 61 | // BlockByNumber ... Wraps go-ethereum node blockByNumber RPC call 62 | func (ec *EthClient) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { 63 | return ec.client.BlockByNumber(ctx, number) 64 | } 65 | 66 | // BalanceAt ... Wraps go-ethereum node balanceAt RPC call 67 | func (ec *EthClient) BalanceAt(ctx context.Context, account common.Address, number *big.Int) (*big.Int, error) { 68 | return ec.client.BalanceAt(ctx, account, number) 69 | } 70 | 71 | // FilterLogs ... Wraps go-ethereum node balanceAt RPC call 72 | func (ec *EthClient) FilterLogs(ctx context.Context, query ethereum.FilterQuery) ([]types.Log, error) { 73 | return ec.client.FilterLogs(ctx, query) 74 | } 75 | -------------------------------------------------------------------------------- /internal/client/slack_client.go: -------------------------------------------------------------------------------- 1 | //go:generate mockgen -package mocks --destination ../mocks/slack_client.go . SlackClient 2 | 3 | package client 4 | 5 | // NOTE - API endpoint specifications for slack client 6 | // can be found here - https://api.slack.com/methods/chat.postMessage 7 | 8 | import ( 9 | "bytes" 10 | "context" 11 | "encoding/json" 12 | "io" 13 | "net/http" 14 | 15 | "github.com/base-org/pessimism/internal/logging" 16 | ) 17 | 18 | // SlackClient ... Interface for slack client 19 | type SlackClient interface { 20 | PostData(context.Context, string) (*SlackAPIResponse, error) 21 | } 22 | 23 | // slackClient ... Slack client 24 | type slackClient struct { 25 | url string 26 | client *http.Client 27 | } 28 | 29 | // NewSlackClient ... Initializer 30 | func NewSlackClient(url string) SlackClient { 31 | if url == "" { 32 | logging.NoContext().Warn("No Slack webhook URL not provided") 33 | } 34 | 35 | return slackClient{ 36 | url: url, 37 | // NOTE - This is a default client, we can add more configuration to it 38 | // when necessary 39 | client: &http.Client{}, 40 | } 41 | } 42 | 43 | // slackPayload represents the structure of a slack alert 44 | type slackPayload struct { 45 | Text interface{} `json:"text"` 46 | } 47 | 48 | // newSlackPayload ... initializes a new slack payload 49 | func newSlackPayload(text interface{}) *slackPayload { 50 | return &slackPayload{Text: text} 51 | } 52 | 53 | // marshal ... marshals the slack payload 54 | func (sp *slackPayload) marshal() ([]byte, error) { 55 | bytes, err := json.Marshal(sp) 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | return bytes, nil 61 | } 62 | 63 | // SlackAPIResponse ... represents the structure of a slack API response 64 | type SlackAPIResponse struct { 65 | Ok bool `json:"ok"` 66 | Err string `json:"error"` 67 | } 68 | 69 | // PostAlert ... handles posting data to slack 70 | func (sc slackClient) PostData(ctx context.Context, str string) (*SlackAPIResponse, error) { 71 | // make & marshal payload 72 | payload, err := newSlackPayload(str).marshal() 73 | if err != nil { 74 | return nil, err 75 | } 76 | 77 | req, err := http.NewRequestWithContext(ctx, 78 | http.MethodPost, sc.url, bytes.NewReader(payload)) 79 | if err != nil { 80 | return nil, err 81 | } 82 | req.Header.Set("Content-Type", "application/json") 83 | 84 | // make request 85 | resp, err := sc.client.Do(req) 86 | if err != nil { 87 | return nil, err 88 | } 89 | defer resp.Body.Close() 90 | 91 | // read response 92 | bytes, err := io.ReadAll(resp.Body) 93 | if err != nil { 94 | return nil, err 95 | } 96 | 97 | var apiResp *SlackAPIResponse 98 | if err := json.Unmarshal(bytes, &apiResp); err != nil { 99 | return nil, err 100 | } 101 | 102 | return apiResp, err 103 | } 104 | -------------------------------------------------------------------------------- /internal/common/common.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "math/big" 5 | 6 | "github.com/ethereum/go-ethereum/common" 7 | "github.com/ethereum/go-ethereum/params" 8 | ) 9 | 10 | // WeiToEther ... Converts wei to ether 11 | func WeiToEther(wei *big.Int) *big.Float { 12 | return new(big.Float).Quo(new(big.Float).SetInt(wei), big.NewFloat(params.Ether)) 13 | } 14 | 15 | // SliceToAddresses ... Converts a slice of strings to a slice of addresses 16 | func SliceToAddresses(slice []string) []common.Address { 17 | var addresses []common.Address 18 | for _, addr := range slice { 19 | addresses = append(addresses, common.HexToAddress(addr)) 20 | } 21 | 22 | return addresses 23 | } 24 | -------------------------------------------------------------------------------- /internal/common/common_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "math/big" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | const ( 11 | weiPerETH = 1000000000000000000 12 | ) 13 | 14 | // Test_WeiToEth ... Tests wei to ether conversion 15 | func Test_WeiToEth(t *testing.T) { 16 | ether := WeiToEther(big.NewInt(weiPerETH)) 17 | etherFloat, _ := ether.Float64() 18 | 19 | assert.Equal(t, etherFloat, float64(1), "should be equal") 20 | } 21 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "log" 5 | "strconv" 6 | "strings" 7 | 8 | "github.com/base-org/pessimism/internal/api/server" 9 | "github.com/base-org/pessimism/internal/api/service" 10 | 11 | "github.com/base-org/pessimism/internal/logging" 12 | "github.com/joho/godotenv" 13 | 14 | "os" 15 | ) 16 | 17 | type FilePath string 18 | 19 | type Env string 20 | 21 | const ( 22 | Development Env = "development" 23 | Production Env = "production" 24 | Local Env = "local" 25 | 26 | // trueEnvVal ... Represents the encoded string value for true (ie. 1) 27 | trueEnvVal = "1" 28 | ) 29 | 30 | // Config ... Application level configuration defined by `FilePath` value 31 | // TODO - Consider renaming to "environment config" 32 | type Config struct { 33 | Environment Env 34 | BootStrapPath string 35 | 36 | // TODO - Consider moving this URL to a more appropriate location 37 | SlackURL string 38 | 39 | SvcConfig *service.Config 40 | ServerConfig *server.Config 41 | 42 | LoggerConfig *logging.Config 43 | } 44 | 45 | // NewConfig ... Initializer 46 | func NewConfig(fileName FilePath) *Config { 47 | if err := godotenv.Load(string(fileName)); err != nil { 48 | log.Fatalf("config file not found for file: %s", fileName) 49 | } 50 | 51 | config := &Config{ 52 | BootStrapPath: getEnvStrWithDefault("BOOTSTRAP_PATH", ""), 53 | Environment: Env(getEnvStr("ENV")), 54 | SlackURL: getEnvStrWithDefault("SLACK_URL", ""), 55 | 56 | SvcConfig: &service.Config{ 57 | L1RpcEndpoint: getEnvStr("L1_RPC_ENDPOINT"), 58 | L2RpcEndpoint: getEnvStr("L2_RPC_ENDPOINT"), 59 | L1PollInterval: getEnvInt("L1_POLL_INTERVAL"), 60 | L2PollInterval: getEnvInt("L2_POLL_INTERVAL"), 61 | }, 62 | 63 | LoggerConfig: &logging.Config{ 64 | UseCustom: getEnvBool("LOGGER_USE_CUSTOM"), 65 | Level: getEnvInt("LOGGER_LEVEL"), 66 | DisableCaller: getEnvBool("LOGGER_DISABLE_CALLER"), 67 | DisableStacktrace: getEnvBool("LOGGER_DISABLE_STACKTRACE"), 68 | Encoding: getEnvStr("LOGGER_ENCODING"), 69 | OutputPaths: getEnvSlice("LOGGER_OUTPUT_PATHS"), 70 | ErrorOutputPaths: getEnvSlice("LOGGER_ERROR_OUTPUT_PATHS"), 71 | }, 72 | 73 | ServerConfig: &server.Config{ 74 | Host: getEnvStr("SERVER_HOST"), 75 | Port: getEnvInt("SERVER_PORT"), 76 | KeepAlive: getEnvInt("SERVER_KEEP_ALIVE_TIME"), 77 | ReadTimeout: getEnvInt("SERVER_READ_TIMEOUT"), 78 | WriteTimeout: getEnvInt("SERVER_WRITE_TIMEOUT"), 79 | }, 80 | } 81 | 82 | return config 83 | } 84 | 85 | // IsProduction ... Returns true if the env is production 86 | func (cfg *Config) IsProduction() bool { 87 | return cfg.Environment == Production 88 | } 89 | 90 | // IsDevelopment ... Returns true if the env is development 91 | func (cfg *Config) IsDevelopment() bool { 92 | return cfg.Environment == Development 93 | } 94 | 95 | // IsLocal ... Returns true if the env is local 96 | func (cfg *Config) IsLocal() bool { 97 | return cfg.Environment == Local 98 | } 99 | 100 | // IsBootstrap ... Returns true if a state bootstrap is required 101 | func (cfg *Config) IsBootstrap() bool { 102 | return cfg.BootStrapPath != "" 103 | } 104 | 105 | // getEnvStr ... Reads env var from process environment, panics if not found 106 | func getEnvStr(key string) string { 107 | envVar, ok := os.LookupEnv(key) 108 | 109 | // Not found 110 | if !ok { 111 | log.Fatalf("could not find env var given key: %s", key) 112 | } 113 | 114 | return envVar 115 | } 116 | 117 | // getEnvStrWithDefault ... Reads env var from process environment, returns default if not found 118 | func getEnvStrWithDefault(key string, defaultValue string) string { 119 | envVar, ok := os.LookupEnv(key) 120 | 121 | // Not found 122 | if !ok { 123 | return defaultValue 124 | } 125 | 126 | return envVar 127 | } 128 | 129 | // getEnvBool ... Reads env vars and converts to booleans 130 | func getEnvBool(key string) bool { 131 | return getEnvStr(key) == trueEnvVal 132 | } 133 | 134 | // getEnvSlice ... Reads env vars and converts to string slice 135 | func getEnvSlice(key string) []string { 136 | return strings.Split(getEnvStr(key), ",") 137 | } 138 | 139 | // getEnvInt ... Reads env vars and converts to int 140 | func getEnvInt(key string) int { 141 | val := getEnvStr(key) 142 | intRep, err := strconv.Atoi(val) 143 | if err != nil { 144 | log.Fatalf("env val is not int; got: %s=%s; err: %s", key, val, err.Error()) 145 | } 146 | return intRep 147 | } 148 | -------------------------------------------------------------------------------- /internal/core/alert.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import "time" 4 | 5 | // AlertingPolicy ... The alerting policy for an invariant session 6 | // NOTE - This could be extended to support additional 7 | // policy metadata like criticality, etc. 8 | type AlertingPolicy struct { 9 | Destination AlertDestination 10 | } 11 | 12 | // Alert ... An alert 13 | type Alert struct { 14 | Dest AlertDestination 15 | PUUID PUUID 16 | SUUID SUUID 17 | Timestamp time.Time 18 | Ptype PipelineType 19 | 20 | Content string 21 | } 22 | -------------------------------------------------------------------------------- /internal/core/config.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "math/big" 5 | "time" 6 | ) 7 | 8 | // ClientConfig ... Configuration passed through to an oracle component constructor 9 | type ClientConfig struct { 10 | RPCEndpoint string 11 | PollInterval time.Duration 12 | NumOfRetries int 13 | StartHeight *big.Int 14 | EndHeight *big.Int 15 | } 16 | 17 | // SessionConfig ... Configuration passed through to a session constructor 18 | type SessionConfig struct { 19 | AlertDest AlertDestination 20 | Type InvariantType 21 | Params InvSessionParams 22 | } 23 | 24 | // PipelineConfig ... Configuration passed through to a pipeline constructor 25 | type PipelineConfig struct { 26 | Network Network 27 | DataType RegisterType 28 | PipelineType PipelineType 29 | ClientConfig *ClientConfig 30 | } 31 | 32 | // Backfill ... Returns true if the oracle is configured to backfill 33 | func (oc *ClientConfig) Backfill() bool { 34 | return oc.StartHeight != nil 35 | } 36 | 37 | // Backtest ... Returns true if the oracle is configured to backtest 38 | func (oc *ClientConfig) Backtest() bool { 39 | return oc.EndHeight != nil 40 | } 41 | -------------------------------------------------------------------------------- /internal/core/constants.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import "github.com/base-org/pessimism/internal/logging" 4 | 5 | // Network ... Represents the network for which a pipeline's oracle 6 | // is subscribed to. 7 | type Network uint8 8 | 9 | const ( 10 | Layer1 = iota + 1 11 | Layer2 12 | 13 | UnknownNetwork 14 | ) 15 | 16 | const ( 17 | UnknownType = "unknown" 18 | ) 19 | 20 | // String ... Converts a network to a string 21 | func (n Network) String() string { 22 | switch n { 23 | case Layer1: 24 | return "layer1" 25 | 26 | case Layer2: 27 | return "layer2" 28 | } 29 | 30 | return UnknownType 31 | } 32 | 33 | // StringToNetwork ... Converts a string to a network 34 | func StringToNetwork(stringType string) Network { 35 | switch stringType { 36 | case "layer1": 37 | return Layer1 38 | 39 | case "layer2": 40 | return Layer2 41 | } 42 | 43 | return UnknownNetwork 44 | } 45 | 46 | type FetchType int 47 | 48 | const ( 49 | FetchHeader FetchType = 0 50 | FetchBlock FetchType = 1 51 | ) 52 | 53 | type Timeouts int 54 | 55 | const ( 56 | EthClientTimeout Timeouts = 20 // in seconds 57 | ) 58 | 59 | // InvariantType ... Represents the type of invariant 60 | type InvariantType uint8 61 | 62 | const ( 63 | ExampleInv = iota + 1 64 | TxCaller 65 | BalanceEnforcement 66 | ContractEvent 67 | ) 68 | 69 | // String ... Converts an invariant type to a string 70 | func (it InvariantType) String() string { 71 | switch it { 72 | case BalanceEnforcement: 73 | return "balance_enforcement" 74 | 75 | case ContractEvent: 76 | return "contract_event" 77 | 78 | default: 79 | return "unknown" 80 | } 81 | } 82 | 83 | // StringToInvariantType ... Converts a string to an invariant type 84 | func StringToInvariantType(stringType string) InvariantType { 85 | switch stringType { 86 | case "balance_enforcement": 87 | return BalanceEnforcement 88 | 89 | case "contract_event": 90 | return ContractEvent 91 | 92 | default: 93 | return InvariantType(0) 94 | } 95 | } 96 | 97 | // AlertDestination ... The destination for an alert 98 | type AlertDestination uint8 99 | 100 | const ( 101 | Slack AlertDestination = iota + 1 102 | ThirdParty // 2 103 | ) 104 | 105 | // String ... Converts an alerting destination type to a string 106 | func (ad AlertDestination) String() string { 107 | switch ad { 108 | case Slack: 109 | return "slack" 110 | case ThirdParty: 111 | return "third_party" 112 | default: 113 | return "unknown" 114 | } 115 | } 116 | 117 | // StringToAlertingDestType ... Converts a string to an alerting destination type 118 | func StringToAlertingDestType(stringType string) AlertDestination { 119 | switch stringType { 120 | case "slack": 121 | return Slack 122 | 123 | case "third_party": 124 | return ThirdParty 125 | } 126 | 127 | return AlertDestination(0) 128 | } 129 | 130 | // ID keys used for logging 131 | const ( 132 | AddrKey logging.LogKey = "address" 133 | 134 | CUUIDKey logging.LogKey = "cuuid" 135 | PUUIDKey logging.LogKey = "puuid" 136 | SUUIDKey logging.LogKey = "suuid" 137 | ) 138 | -------------------------------------------------------------------------------- /internal/core/core.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/ethereum/go-ethereum/common" 7 | ) 8 | 9 | // TransitOption ... Option used to initialize transit data 10 | type TransitOption = func(*TransitData) 11 | 12 | // WithAddress ... Injects address to transit data 13 | func WithAddress(address common.Address) TransitOption { 14 | return func(td *TransitData) { 15 | td.Address = address 16 | } 17 | } 18 | 19 | // TransitData ... Standardized type used for data inter-communication 20 | // between all ETL components and Risk Engine 21 | type TransitData struct { 22 | Timestamp time.Time 23 | 24 | Network Network 25 | Type RegisterType 26 | 27 | Address common.Address 28 | Value any 29 | } 30 | 31 | // NewTransitData ... Initializes transit data with supplied options 32 | // NOTE - transit data is used as a standard data representation 33 | // for commmunication between all ETL components and the risk engine 34 | func NewTransitData(rt RegisterType, val any, opts ...TransitOption) TransitData { 35 | td := TransitData{ 36 | Timestamp: time.Now(), 37 | Type: rt, 38 | Value: val, 39 | } 40 | 41 | for _, opt := range opts { // Apply options 42 | opt(&td) 43 | } 44 | 45 | return td 46 | } 47 | 48 | // Addressed ... Indicates whether the transit data has an 49 | // associated address field 50 | func (td *TransitData) Addressed() bool { 51 | return td.Address != common.Address{0} 52 | } 53 | 54 | // NewTransitChannel ... Builds new tranit channel 55 | func NewTransitChannel() chan TransitData { 56 | return make(chan TransitData) 57 | } 58 | 59 | // InvariantInput ... Standardized type used to supply 60 | // the Risk Engine 61 | type InvariantInput struct { 62 | PUUID PUUID 63 | Input TransitData 64 | } 65 | 66 | // EngineInputRelay ... Represents a inter-subsystem 67 | // relay used to bind final ETL pipeline outputs to risk engine inputs 68 | type EngineInputRelay struct { 69 | pUUID PUUID 70 | outChan chan InvariantInput 71 | } 72 | 73 | // NewEngineRelay ... Initializer 74 | func NewEngineRelay(pUUID PUUID, outChan chan InvariantInput) *EngineInputRelay { 75 | return &EngineInputRelay{ 76 | pUUID: pUUID, 77 | outChan: outChan, 78 | } 79 | } 80 | 81 | // RelayTransitData ... Creates invariant input from transit data to send to risk engine 82 | func (eir *EngineInputRelay) RelayTransitData(td TransitData) error { 83 | invInput := InvariantInput{ 84 | PUUID: eir.pUUID, 85 | Input: td, 86 | } 87 | 88 | eir.outChan <- invInput 89 | return nil 90 | } 91 | 92 | const ( 93 | AddressKey = "address" 94 | NestedArgs = "args" 95 | ) 96 | 97 | // InvSessionParams ... Parameters used to initialize an invariant session 98 | type InvSessionParams map[string]interface{} 99 | 100 | // Address ... Returns the address from the invariant session params 101 | func (sp *InvSessionParams) Address() string { 102 | rawAddr, found := (*sp)[AddressKey] 103 | if !found { 104 | return "" 105 | } 106 | 107 | addr, success := rawAddr.(string) 108 | if !success { 109 | return "" 110 | } 111 | 112 | return addr 113 | } 114 | 115 | // Address ... Returns the address from the invariant session params 116 | func (sp *InvSessionParams) NestedArgs() []string { 117 | rawArgs, found := (*sp)[NestedArgs] 118 | if !found { 119 | return []string{} 120 | } 121 | 122 | args, success := rawArgs.([]interface{}) 123 | if !success { 124 | return []string{} 125 | } 126 | 127 | var strArgs []string 128 | for _, arg := range args { 129 | strArgs = append(strArgs, arg.(string)) 130 | } 131 | 132 | return strArgs 133 | } 134 | 135 | // InvalOutcome ... Represents an invalidation outcome 136 | type InvalOutcome struct { 137 | TimeStamp time.Time 138 | Message string 139 | } 140 | 141 | // Subsystem ... Represents a subsystem 142 | type Subsystem interface { 143 | EventLoop() error 144 | Shutdown() error 145 | } 146 | -------------------------------------------------------------------------------- /internal/core/etl.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | // ComponentType ... 4 | type ComponentType uint8 5 | 6 | const ( 7 | Oracle ComponentType = iota + 1 8 | Pipe 9 | Aggregator 10 | ) 11 | 12 | func (ct ComponentType) String() string { 13 | switch ct { 14 | case Oracle: 15 | return "oracle" 16 | 17 | case Pipe: 18 | return "pipe" 19 | 20 | case Aggregator: 21 | return "aggregator" 22 | } 23 | 24 | return UnknownType 25 | } 26 | 27 | // PipelineType ... 28 | type PipelineType uint8 29 | 30 | const ( 31 | Backtest PipelineType = iota + 1 32 | Live 33 | MockTest 34 | ) 35 | 36 | func StringToPipelineType(stringType string) PipelineType { 37 | switch stringType { 38 | case "backtest": 39 | return Backtest 40 | 41 | case "live": 42 | return Live 43 | 44 | case "mocktest": 45 | return MockTest 46 | } 47 | 48 | return PipelineType(0) 49 | } 50 | 51 | func (pt PipelineType) String() string { 52 | switch pt { 53 | case Backtest: 54 | return "backtest" 55 | 56 | case Live: 57 | return "live" 58 | 59 | case MockTest: 60 | return "mocktest" 61 | } 62 | 63 | return UnknownType 64 | } 65 | -------------------------------------------------------------------------------- /internal/core/id_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func Test_Component_ID(t *testing.T) { 10 | 11 | expectedPID := ComponentPID([4]byte{1, 1, 1, 1}) 12 | actualID := MakeCUUID(1, 1, 1, 1) 13 | 14 | assert.Equal(t, expectedPID, actualID.PID) 15 | 16 | expectedStr := "layer1:backtest:oracle:account_balance" 17 | actualStr := actualID.PID.String() 18 | 19 | assert.Equal(t, expectedStr, actualStr) 20 | } 21 | 22 | func Test_Pipeline_ID(t *testing.T) { 23 | expectedID := PipelinePID([9]byte{1, 1, 1, 1, 1, 1, 1, 1, 1}) 24 | actualID := MakePUUID(1, 25 | MakeCUUID(1, 1, 1, 1), 26 | MakeCUUID(1, 1, 1, 1)) 27 | 28 | assert.Equal(t, expectedID, actualID.PID) 29 | 30 | expectedStr := "backtest::layer1:backtest:oracle:account_balance::layer1:backtest:oracle:account_balance" 31 | actualStr := actualID.PID.String() 32 | 33 | assert.Equal(t, expectedStr, actualStr) 34 | } 35 | 36 | func Test_InvSession_ID(t *testing.T) { 37 | expectedID := InvSessionPID([3]byte{1, 2, 1}) 38 | actualID := MakeSUUID(1, 2, 1) 39 | 40 | assert.Equal(t, expectedID, actualID.PID) 41 | 42 | expectedStr := "layer1:live:unknown" 43 | actualStr := actualID.PID.String() 44 | 45 | assert.Equal(t, expectedStr, actualStr) 46 | } 47 | -------------------------------------------------------------------------------- /internal/core/register.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | // RegisterType ... One byte register type enum 4 | type RegisterType uint8 5 | 6 | const ( 7 | AccountBalance RegisterType = iota + 1 8 | GethBlock 9 | EventLog 10 | ) 11 | 12 | // String ... Returns string representation of a 13 | // register enum 14 | func (rt RegisterType) String() string { 15 | switch rt { 16 | case AccountBalance: 17 | return "account_balance" 18 | 19 | case GethBlock: 20 | return "geth_block" 21 | 22 | case EventLog: 23 | return "event_log" 24 | } 25 | 26 | return UnknownType 27 | } 28 | 29 | // DataRegister ... Represents an ETL subsytem data type that 30 | // can be produced and consumed by heterogenous components 31 | type DataRegister struct { 32 | Addressing bool 33 | Sk *StateKey 34 | 35 | DataType RegisterType 36 | ComponentType ComponentType 37 | ComponentConstructor interface{} 38 | Dependencies []RegisterType 39 | } 40 | 41 | // StateKey ... Returns a cloned state key for a data register 42 | func (dr *DataRegister) StateKey() *StateKey { 43 | return dr.Sk.Clone() 44 | } 45 | 46 | // Stateful ... Indicates whether the data register has statefulness 47 | func (dr *DataRegister) Stateful() bool { 48 | return dr.Sk != nil 49 | } 50 | 51 | // RegisterDependencyPath ... Represents an inclusive acyclic sequential 52 | // path of data register dependencies 53 | type RegisterDependencyPath struct { 54 | Path []*DataRegister 55 | } 56 | 57 | // GeneratePUUID ... Generates a PUUID for an existing dependency path 58 | // provided an enumerated pipeline and network type 59 | func (rdp RegisterDependencyPath) GeneratePUUID(pt PipelineType, n Network) PUUID { 60 | firstComp, lastComp := rdp.Path[0], rdp.Path[len(rdp.Path)-1] 61 | firstUUID := MakeCUUID(pt, firstComp.ComponentType, firstComp.DataType, n) 62 | lastUUID := MakeCUUID(pt, lastComp.ComponentType, lastComp.DataType, n) 63 | 64 | return MakePUUID(pt, firstUUID, lastUUID) 65 | } 66 | -------------------------------------------------------------------------------- /internal/core/state.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // StateKey ... Represents a key in the state store 8 | type StateKey struct { 9 | Nesting bool 10 | Prefix RegisterType 11 | ID string 12 | 13 | PUUID *PUUID 14 | } 15 | 16 | // Clone ... Returns a copy of the state key 17 | func (sk *StateKey) Clone() *StateKey { 18 | return &StateKey{ 19 | Nesting: sk.Nesting, 20 | Prefix: sk.Prefix, 21 | ID: sk.ID, 22 | PUUID: sk.PUUID, 23 | } 24 | } 25 | 26 | // MakeStateKey ... Builds a minimal state key using 27 | // a prefix and key 28 | func MakeStateKey(pre RegisterType, id string, nest bool) *StateKey { 29 | return &StateKey{ 30 | Nesting: nest, 31 | Prefix: pre, 32 | ID: id, 33 | } 34 | } 35 | 36 | // IsNested ... Indicates whether the state key is nested 37 | // NOTE - This is used to determine if the state key maps 38 | // to a value slice of state keys in the state store (ie. nested) 39 | func (sk *StateKey) IsNested() bool { 40 | return sk.Nesting 41 | } 42 | 43 | // SetPUUID ... Adds a pipeline UUID to the state key prefix and returns a new state key 44 | func (sk *StateKey) SetPUUID(pUUID PUUID) error { 45 | if sk.PUUID != nil { 46 | return fmt.Errorf("state key already has a pipeline UUID %s", sk.PUUID.String()) 47 | } 48 | 49 | sk.PUUID = &pUUID 50 | return nil 51 | } 52 | 53 | const ( 54 | AddressPrefix = iota + 1 55 | NestedPrefix 56 | ) 57 | 58 | // String ... Returns a string representation of the state key 59 | func (sk StateKey) String() string { 60 | pUUID := "" 61 | 62 | if sk.PUUID != nil { 63 | pUUID = sk.PUUID.String() 64 | } 65 | 66 | return fmt.Sprintf("%s-%s-%s", pUUID, sk.Prefix, sk.ID) 67 | } 68 | -------------------------------------------------------------------------------- /internal/engine/addressing.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | "github.com/ethereum/go-ethereum/common" 8 | ) 9 | 10 | // AddressingMap ... Interface for mapping addresses to session UUIDs 11 | type AddressingMap interface { 12 | GetSessionUUIDByPair(address common.Address, pUUID core.PUUID) (core.SUUID, error) 13 | Insert(pUUID core.PUUID, sUUID core.SUUID, address common.Address) error 14 | } 15 | 16 | // addressEntry ... Entry for the addressing map 17 | type addressEntry struct { 18 | address common.Address 19 | sUUID core.SUUID 20 | pUUID core.PUUID 21 | } 22 | 23 | // addressingMap ... Implementation of AddressingMap 24 | type addressingMap struct { 25 | m map[common.Address][]*addressEntry 26 | } 27 | 28 | // GetSessionUUIDByPair ... Gets the session UUID by the pair of address and pipeline UUID 29 | func (am *addressingMap) GetSessionUUIDByPair(address common.Address, 30 | pUUID core.PUUID) (core.SUUID, error) { 31 | if _, found := am.m[address]; !found { 32 | return core.NilSUUID(), fmt.Errorf("address provided is not tracked %s", address.String()) 33 | } 34 | 35 | // Now we know it's entry has been seen 36 | for _, entry := range am.m[address] { 37 | if entry.pUUID == pUUID { // Found 38 | return entry.sUUID, nil 39 | } 40 | } 41 | 42 | return core.NilSUUID(), fmt.Errorf("could not find matching pUUID %s", pUUID.String()) 43 | } 44 | 45 | // Insert ... Inserts a new entry into the addressing map 46 | func (am *addressingMap) Insert(pUUID core.PUUID, 47 | sUUID core.SUUID, address common.Address) error { 48 | newEntry := &addressEntry{ 49 | address: address, 50 | sUUID: sUUID, 51 | pUUID: pUUID} 52 | 53 | if _, found := am.m[address]; !found { 54 | am.m[address] = []*addressEntry{newEntry} 55 | return nil 56 | } 57 | 58 | // Now we know it's entry has been seen 59 | for _, entry := range am.m[address] { 60 | if entry.pUUID == pUUID { 61 | return fmt.Errorf("%s already exists for suuid %s", address, sUUID.String()) 62 | } 63 | } 64 | 65 | am.m[address] = append(am.m[address], newEntry) 66 | return nil 67 | } 68 | 69 | // NewAddressingMap ... Initializer 70 | func NewAddressingMap() AddressingMap { 71 | return &addressingMap{ 72 | m: make(map[common.Address][]*addressEntry), 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /internal/engine/addressing_test.go: -------------------------------------------------------------------------------- 1 | package engine_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | "github.com/base-org/pessimism/internal/engine" 8 | "github.com/ethereum/go-ethereum/common" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | var ( 13 | testPUUID = core.MakePUUID(0, 14 | core.MakeCUUID(core.Live, 0, 0, 0), 15 | core.MakeCUUID(core.Live, 0, 0, 0)) 16 | ) 17 | 18 | func Test_GetSessionUUIDByPair(t *testing.T) { 19 | am := engine.NewAddressingMap() 20 | 21 | pUUID := core.NilPUUID() 22 | sUUID := core.NilSUUID() 23 | address := common.HexToAddress("0x24") 24 | 25 | err := am.Insert(pUUID, sUUID, address) 26 | assert.NoError(t, err, "should not error") 27 | 28 | // Test for found 29 | sUUID, err = am.GetSessionUUIDByPair(address, pUUID) 30 | assert.NoError(t, err, "should not error") 31 | assert.Equal(t, core.NilSUUID(), sUUID, "should be equal") 32 | 33 | } 34 | 35 | func Test_Insert(t *testing.T) { 36 | am := engine.NewAddressingMap() 37 | 38 | pUUID := core.NilPUUID() 39 | sUUID := core.NilSUUID() 40 | address := common.HexToAddress("0x24") 41 | 42 | err := am.Insert(pUUID, sUUID, address) 43 | assert.NoError(t, err, "should not error") 44 | 45 | // Test for found 46 | sUUID, err = am.GetSessionUUIDByPair(address, pUUID) 47 | assert.NoError(t, err, "should not error") 48 | assert.Equal(t, core.NilSUUID(), sUUID, "should be equal") 49 | 50 | // Test for not found 51 | sUUID, err = am.GetSessionUUIDByPair(address, testPUUID) 52 | assert.Error(t, err, "should error") 53 | assert.Equal(t, core.NilSUUID(), sUUID, "should be equal") 54 | } 55 | -------------------------------------------------------------------------------- /internal/engine/engine.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | "github.com/base-org/pessimism/internal/engine/invariant" 8 | "github.com/base-org/pessimism/internal/logging" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | type Type int 13 | 14 | const ( 15 | HardCoded Type = iota 16 | Dynamic 17 | ) 18 | 19 | // RiskEngine ... Execution engine interface 20 | type RiskEngine interface { 21 | Type() Type 22 | Execute(context.Context, core.TransitData, 23 | invariant.Invariant) (*core.InvalOutcome, bool) 24 | } 25 | 26 | // hardCodedEngine ... Hard coded execution engine 27 | // IE: native application code for invariant implementation 28 | type hardCodedEngine struct { 29 | // TODO: Add any engine specific fields here 30 | } 31 | 32 | // NewHardCodedEngine ... Initializer 33 | func NewHardCodedEngine() RiskEngine { 34 | return &hardCodedEngine{} 35 | } 36 | 37 | // Type ... Returns the engine type 38 | func (e *hardCodedEngine) Type() Type { 39 | return HardCoded 40 | } 41 | 42 | // Execute ... Executes the invariant 43 | func (e *hardCodedEngine) Execute(ctx context.Context, data core.TransitData, 44 | inv invariant.Invariant) (*core.InvalOutcome, bool) { 45 | logger := logging.WithContext(ctx) 46 | 47 | logger.Debug("Performing invariant invalidation", 48 | zap.String("suuid", inv.SUUID().String())) 49 | outcome, invalid, err := inv.Invalidate(data) 50 | if err != nil { 51 | logger.Error("Failed to perform invalidation option for invariant", zap.Error(err)) 52 | return nil, false 53 | } 54 | 55 | return outcome, invalid 56 | } 57 | -------------------------------------------------------------------------------- /internal/engine/invariant/config.go: -------------------------------------------------------------------------------- 1 | package invariant 2 | 3 | import "github.com/base-org/pessimism/internal/core" 4 | 5 | // DeployConfig ... Configuration for deploying an invariant session 6 | type DeployConfig struct { 7 | Network core.Network 8 | PUUID core.PUUID 9 | InvType core.InvariantType 10 | InvParams core.InvSessionParams 11 | Register *core.DataRegister 12 | } 13 | -------------------------------------------------------------------------------- /internal/engine/invariant/invariant.go: -------------------------------------------------------------------------------- 1 | package invariant 2 | 3 | import ( 4 | "github.com/base-org/pessimism/internal/core" 5 | ) 6 | 7 | // ExecutionType ... Enum for execution type 8 | type ExecutionType int 9 | 10 | const ( 11 | // HardCoded ... Hard coded execution type (ie native application code) 12 | HardCoded ExecutionType = iota 13 | ) 14 | 15 | // Invariant ... Interface that all invariant implementations must adhere to 16 | type Invariant interface { 17 | InputType() core.RegisterType 18 | Invalidate(core.TransitData) (*core.InvalOutcome, bool, error) 19 | SUUID() core.SUUID 20 | SetSUUID(core.SUUID) 21 | } 22 | 23 | // BaseInvariantOpt ... Functional option for BaseInvariant 24 | type BaseInvariantOpt = func(bi *BaseInvariant) *BaseInvariant 25 | 26 | // WithAddressing ... Toggles addressing property for invariant 27 | func WithAddressing() BaseInvariantOpt { 28 | return func(bi *BaseInvariant) *BaseInvariant { 29 | bi.addressing = true 30 | return bi 31 | } 32 | } 33 | 34 | // BaseInvariant ... Base invariant implementation 35 | type BaseInvariant struct { 36 | addressing bool 37 | sUUID core.SUUID 38 | inType core.RegisterType 39 | } 40 | 41 | // NewBaseInvariant ... Initializer 42 | func NewBaseInvariant(inType core.RegisterType, 43 | opts ...BaseInvariantOpt) Invariant { 44 | bi := &BaseInvariant{ 45 | inType: inType, 46 | } 47 | 48 | for _, opt := range opts { 49 | opt(bi) 50 | } 51 | 52 | return bi 53 | } 54 | 55 | // SetSUUID ... Sets the invariant session UUID 56 | func (bi *BaseInvariant) SetSUUID(sUUID core.SUUID) { 57 | bi.sUUID = sUUID 58 | } 59 | 60 | // SUUID ... Returns the invariant session UUID 61 | func (bi *BaseInvariant) SUUID() core.SUUID { 62 | return bi.sUUID 63 | } 64 | 65 | // InputType ... Returns the input type for the invariant 66 | func (bi *BaseInvariant) InputType() core.RegisterType { 67 | return bi.inType 68 | } 69 | 70 | // Invalidate ... Invalidates the invariant; defaults to no-op 71 | func (bi *BaseInvariant) Invalidate(core.TransitData) (*core.InvalOutcome, bool, error) { 72 | return nil, false, nil 73 | } 74 | -------------------------------------------------------------------------------- /internal/engine/registry/balance.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/base-org/pessimism/internal/core" 8 | "github.com/base-org/pessimism/internal/engine/invariant" 9 | "github.com/base-org/pessimism/internal/logging" 10 | "go.uber.org/zap" 11 | ) 12 | 13 | // BalanceInvConfig ... Configuration for the balance invariant 14 | type BalanceInvConfig struct { 15 | Address string `json:"address"` 16 | UpperBound *float64 `json:"upper"` 17 | LowerBound *float64 `json:"lower"` 18 | } 19 | 20 | // BalanceInvariant ... 21 | type BalanceInvariant struct { 22 | cfg *BalanceInvConfig 23 | 24 | invariant.Invariant 25 | } 26 | 27 | // reportMsg ... Message to be sent to the alerting system 28 | const reportMsg = ` 29 | Current value: %3f 30 | Upper bound: %s 31 | Lower bound: %s 32 | 33 | Session UUID: %s 34 | Session Address: %s 35 | ` 36 | 37 | // NewBalanceInvariant ... Initializer 38 | func NewBalanceInvariant(cfg *BalanceInvConfig) invariant.Invariant { 39 | return &BalanceInvariant{ 40 | cfg: cfg, 41 | 42 | Invariant: invariant.NewBaseInvariant(core.AccountBalance, invariant.WithAddressing()), 43 | } 44 | } 45 | 46 | // Invalidate ... Checks if the balance is within the bounds 47 | // specified in the config 48 | func (bi *BalanceInvariant) Invalidate(td core.TransitData) (*core.InvalOutcome, bool, error) { 49 | logging.NoContext().Debug("Checking invalidation for balance invariant", zap.String("data", fmt.Sprintf("%v", td))) 50 | 51 | if td.Type != bi.InputType() { 52 | return nil, false, fmt.Errorf("invalid type supplied") 53 | } 54 | 55 | balance, ok := td.Value.(float64) 56 | if !ok { 57 | return nil, false, fmt.Errorf("could not cast transit data value to float type") 58 | } 59 | 60 | invalidated := false 61 | 62 | // balance > upper bound 63 | if bi.cfg.UpperBound != nil && 64 | *bi.cfg.UpperBound < balance { 65 | invalidated = true 66 | } 67 | 68 | // balance < lower bound 69 | if bi.cfg.LowerBound != nil && 70 | *bi.cfg.LowerBound > balance { 71 | invalidated = true 72 | } 73 | 74 | if invalidated { 75 | var upper, lower string 76 | 77 | if bi.cfg.UpperBound != nil { 78 | upper = fmt.Sprintf("%2f", *bi.cfg.UpperBound) 79 | } else { 80 | upper = "∞" 81 | } 82 | 83 | if bi.cfg.LowerBound != nil { 84 | lower = fmt.Sprintf("%2f", *bi.cfg.LowerBound) 85 | } else { 86 | lower = "-∞" 87 | } 88 | 89 | return &core.InvalOutcome{ 90 | TimeStamp: time.Now(), 91 | Message: fmt.Sprintf(reportMsg, balance, 92 | upper, lower, 93 | bi.SUUID(), bi.cfg.Address), 94 | }, true, nil 95 | } 96 | 97 | // No invalidation 98 | return nil, false, nil 99 | } 100 | -------------------------------------------------------------------------------- /internal/engine/registry/balance_test.go: -------------------------------------------------------------------------------- 1 | package registry_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | "github.com/base-org/pessimism/internal/engine/registry" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func Test_Invalidate(t *testing.T) { 12 | upper := float64(5) 13 | lower := float64(1) 14 | 15 | bi := registry.NewBalanceInvariant(®istry.BalanceInvConfig{ 16 | Address: "0x123", 17 | UpperBound: &upper, 18 | LowerBound: &lower, 19 | }) 20 | 21 | // No invalidation 22 | 23 | testData1 := core.TransitData{ 24 | Type: core.AccountBalance, 25 | Value: float64(3), 26 | } 27 | 28 | _, inval, err := bi.Invalidate(testData1) 29 | assert.NoError(t, err) 30 | assert.False(t, inval) 31 | 32 | // Upper bound invalidation 33 | testData2 := core.TransitData{ 34 | Type: core.AccountBalance, 35 | Value: float64(6), 36 | } 37 | 38 | _, inval, err = bi.Invalidate(testData2) 39 | assert.NoError(t, err) 40 | assert.True(t, inval) 41 | 42 | // Lower bound invalidation 43 | testData3 := core.TransitData{ 44 | Type: core.AccountBalance, 45 | Value: float64(0.1), 46 | } 47 | 48 | _, inval, err = bi.Invalidate(testData3) 49 | assert.NoError(t, err) 50 | assert.True(t, inval) 51 | } 52 | -------------------------------------------------------------------------------- /internal/engine/registry/event_log.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | "github.com/base-org/pessimism/internal/engine/invariant" 8 | "github.com/ethereum/go-ethereum/common" 9 | "github.com/ethereum/go-ethereum/core/types" 10 | "github.com/ethereum/go-ethereum/crypto" 11 | ) 12 | 13 | // EventInvConfig ... Configuration for the event invariant 14 | type EventInvConfig struct { 15 | ContractName string `json:"contract_name"` 16 | Address string `json:"address"` 17 | Sigs []string `json:"args"` 18 | } 19 | 20 | // EventInvariant ... 21 | type EventInvariant struct { 22 | cfg *EventInvConfig 23 | sigs []common.Hash 24 | 25 | invariant.Invariant 26 | } 27 | 28 | // eventReportMsg ... Message to be sent to the alerting system 29 | const eventReportMsg = ` 30 | _Monitored Event Triggered_ 31 | 32 | Contract Name: %s 33 | Contract Address: %s 34 | Transaction Hash: %s 35 | Event: %s 36 | ` 37 | 38 | // NewEventInvariant ... Initializer 39 | func NewEventInvariant(cfg *EventInvConfig) invariant.Invariant { 40 | var sigs []common.Hash 41 | for _, sig := range cfg.Sigs { 42 | sigs = append(sigs, crypto.Keccak256Hash([]byte(sig))) 43 | } 44 | 45 | return &EventInvariant{ 46 | cfg: cfg, 47 | sigs: sigs, 48 | 49 | Invariant: invariant.NewBaseInvariant(core.EventLog, 50 | invariant.WithAddressing()), 51 | } 52 | } 53 | 54 | // Invalidate ... Checks if the balance is within the bounds 55 | // specified in the config 56 | func (ei *EventInvariant) Invalidate(td core.TransitData) (*core.InvalOutcome, bool, error) { 57 | if td.Type != ei.InputType() { 58 | return nil, false, fmt.Errorf("invalid type supplied") 59 | } 60 | 61 | if td.Address.String() != ei.cfg.Address { 62 | return nil, false, fmt.Errorf("invalid address supplied") 63 | } 64 | 65 | log, success := td.Value.(types.Log) 66 | if !success { 67 | return nil, false, fmt.Errorf("could not convert transit data to log") 68 | } 69 | 70 | var invalidated = false 71 | 72 | for _, sig := range ei.sigs { 73 | if log.Topics[0] == sig { 74 | invalidated = true 75 | break 76 | } 77 | } 78 | 79 | if !invalidated { 80 | return nil, false, nil 81 | } 82 | 83 | return &core.InvalOutcome{ 84 | Message: fmt.Sprintf(eventReportMsg, ei.cfg.ContractName, log.Address, log.TxHash.Hex(), ei.cfg.Sigs[0]), 85 | }, true, nil 86 | } 87 | -------------------------------------------------------------------------------- /internal/engine/registry/event_log_test.go: -------------------------------------------------------------------------------- 1 | package registry_test 2 | 3 | // TODO 4 | -------------------------------------------------------------------------------- /internal/engine/registry/registry.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | "github.com/base-org/pessimism/internal/core" 8 | "github.com/base-org/pessimism/internal/engine/invariant" 9 | ) 10 | 11 | // GetInvariant ... Returns an invariant based on the invariant type provided 12 | // a general config 13 | func GetInvariant(it core.InvariantType, cfg any) (invariant.Invariant, error) { 14 | var inv invariant.Invariant 15 | 16 | switch it { 17 | case core.BalanceEnforcement: 18 | cfg, err := json.Marshal(cfg) 19 | if err != nil { 20 | return nil, err 21 | } 22 | // convert json to struct 23 | invConfg := BalanceInvConfig{} 24 | err = json.Unmarshal(cfg, &invConfg) 25 | if err != nil { 26 | return nil, err 27 | } 28 | 29 | inv = NewBalanceInvariant(&invConfg) 30 | 31 | case core.ContractEvent: 32 | cfg, err := json.Marshal(cfg) 33 | if err != nil { 34 | return nil, err 35 | } 36 | // convert json to struct 37 | invConfg := EventInvConfig{} 38 | err = json.Unmarshal(cfg, &invConfg) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | inv = NewEventInvariant(&invConfg) 44 | 45 | default: 46 | return nil, fmt.Errorf("could not find implementation for type %s", it.String()) 47 | } 48 | 49 | return inv, nil 50 | } 51 | -------------------------------------------------------------------------------- /internal/engine/store.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | "github.com/base-org/pessimism/internal/engine/invariant" 8 | ) 9 | 10 | // SessionStore ... 11 | type SessionStore interface { 12 | AddInvSession(sUUID core.SUUID, pID core.PUUID, inv invariant.Invariant) error 13 | GetInvSessionByUUID(sUUID core.SUUID) (invariant.Invariant, error) 14 | GetInvariantsByUUIDs(sUUIDs ...core.SUUID) ([]invariant.Invariant, error) 15 | GetInvSessionsForPipeline(pUUID core.PUUID) ([]core.SUUID, error) 16 | } 17 | 18 | // sessionStore ... 19 | type sessionStore struct { 20 | sessionPipelineMap map[core.PUUID][]core.SUUID 21 | invSessionMap map[core.SUUID]invariant.Invariant // no duplicates 22 | } 23 | 24 | // NewSessionStore ... Initializer 25 | func NewSessionStore() SessionStore { 26 | return &sessionStore{ 27 | invSessionMap: make(map[core.SUUID]invariant.Invariant), 28 | sessionPipelineMap: make(map[core.PUUID][]core.SUUID), 29 | } 30 | } 31 | 32 | // GetInvariantsByUUIDs ... Fetches in-order all invariants associated with a set of session UUIDs 33 | func (ss *sessionStore) GetInvariantsByUUIDs(sUUIDs ...core.SUUID) ([]invariant.Invariant, error) { 34 | invariants := make([]invariant.Invariant, len(sUUIDs)) 35 | 36 | for i, uuid := range sUUIDs { 37 | session, err := ss.GetInvSessionByUUID(uuid) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | invariants[i] = session 43 | } 44 | 45 | return invariants, nil 46 | } 47 | 48 | // GetInvSessionByUUID .... Fetches invariant session by UUID 49 | func (ss *sessionStore) GetInvSessionByUUID(sUUID core.SUUID) (invariant.Invariant, error) { 50 | if entry, found := ss.invSessionMap[sUUID]; found { 51 | return entry, nil 52 | } 53 | return nil, fmt.Errorf("invariant UUID doesn't exists in store inv mapping") 54 | } 55 | 56 | // GetInvSessionsForPipeline ... Returns all invariant session ids associated with pipeline 57 | func (ss *sessionStore) GetInvSessionsForPipeline(pUUID core.PUUID) ([]core.SUUID, error) { 58 | if sessionIDs, found := ss.sessionPipelineMap[pUUID]; found { 59 | return sessionIDs, nil 60 | } 61 | return nil, fmt.Errorf("pipeline UUID doesn't exists in store inv mapping") 62 | } 63 | 64 | // AddInvSession ... Adds an invariant session to the store 65 | func (ss *sessionStore) AddInvSession(sUUID core.SUUID, 66 | pUUID core.PUUID, inv invariant.Invariant) error { 67 | if _, found := ss.invSessionMap[sUUID]; found { 68 | return fmt.Errorf("invariant UUID already exists in store pid mapping") 69 | } 70 | 71 | if _, found := ss.sessionPipelineMap[pUUID]; !found { // 72 | ss.sessionPipelineMap[pUUID] = make([]core.SUUID, 0) 73 | } 74 | ss.invSessionMap[sUUID] = inv 75 | ss.sessionPipelineMap[pUUID] = append(ss.sessionPipelineMap[pUUID], sUUID) 76 | return nil 77 | } 78 | 79 | // RemoveInvSession ... Removes an existing invariant session from the store 80 | func (ss *sessionStore) RemoveInvSession(_ core.SUUID, 81 | _ core.PUUID, _ invariant.Invariant) error { 82 | return nil 83 | } 84 | -------------------------------------------------------------------------------- /internal/engine/store_test.go: -------------------------------------------------------------------------------- 1 | package engine_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | "github.com/base-org/pessimism/internal/engine" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestSessionStore(t *testing.T) { 12 | // Setup 13 | ss := engine.NewSessionStore() 14 | 15 | // Test GetInvSessionByUUID 16 | _, err := ss.GetInvSessionByUUID(core.NilSUUID()) 17 | assert.Error(t, err, "should error") 18 | } 19 | -------------------------------------------------------------------------------- /internal/etl/component/aggregator.go: -------------------------------------------------------------------------------- 1 | package component 2 | 3 | // TODO(#12): No Aggregation Component Support 4 | type Aggregator struct{} 5 | -------------------------------------------------------------------------------- /internal/etl/component/component.go: -------------------------------------------------------------------------------- 1 | package component 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | ) 8 | 9 | const ( 10 | killSig = 0 11 | ) 12 | 13 | // Component ... Generalized interface that all pipeline components must adhere to 14 | type Component interface { 15 | /* 16 | NOTE - Storing the PUUID assumes that one component 17 | can only be a part of one pipeline at a time. This could be 18 | problematic if we want to have a component be a part of multiple 19 | pipelines at once. In that case, we would need to store a slice 20 | of PUUIDs instead. 21 | */ 22 | // PUUID ... Returns component's PUUID 23 | PUUID() core.PUUID 24 | 25 | // UUID ... 26 | UUID() core.CUUID 27 | // Type ... Returns component enum type 28 | Type() core.ComponentType 29 | 30 | // AddRelay ... Adds an engine relay to component egress routing 31 | AddRelay(relay *core.EngineInputRelay) error 32 | 33 | // AddEgress ... 34 | AddEgress(core.CUUID, chan core.TransitData) error 35 | // RemoveEgress ... 36 | RemoveEgress(core.CUUID) error 37 | 38 | // Close ... Signifies a component to stop operating 39 | Close() error 40 | 41 | // EventLoop ... Component driver function; spun up as separate go routine 42 | EventLoop() error 43 | 44 | // GetIngress ... Returns component ingress channel for some register type value 45 | GetIngress(rt core.RegisterType) (chan core.TransitData, error) 46 | 47 | // OutputType ... Returns component output data type 48 | OutputType() core.RegisterType 49 | 50 | StateKey() *core.StateKey 51 | 52 | // TODO(#24): Add Internal Component Activity State Tracking 53 | ActivityState() ActivityState 54 | } 55 | 56 | // metaData ... Component agnostic struct that stores component metadata and routing state 57 | type metaData struct { 58 | id core.CUUID 59 | pUUID core.PUUID 60 | 61 | cType core.ComponentType 62 | output core.RegisterType 63 | state ActivityState 64 | 65 | inTypes []core.RegisterType 66 | 67 | closeChan chan int 68 | stateChan chan StateChange 69 | sk *core.StateKey 70 | 71 | *ingressHandler 72 | *egressHandler 73 | 74 | *sync.RWMutex 75 | } 76 | 77 | // newMetaData ... Initializer 78 | func newMetaData(ct core.ComponentType, ot core.RegisterType) *metaData { 79 | return &metaData{ 80 | id: core.NilCUUID(), 81 | pUUID: core.NilPUUID(), 82 | 83 | cType: ct, 84 | egressHandler: newEgressHandler(), 85 | ingressHandler: newIngressHandler(), 86 | state: Inactive, 87 | closeChan: make(chan int), 88 | stateChan: make(chan StateChange), 89 | output: ot, 90 | RWMutex: &sync.RWMutex{}, 91 | } 92 | } 93 | 94 | // ActivityState ... Returns component current activity state 95 | func (meta *metaData) ActivityState() ActivityState { 96 | return meta.state 97 | } 98 | 99 | // StateKey ... Returns component's state key 100 | func (meta *metaData) StateKey() *core.StateKey { 101 | return meta.sk 102 | } 103 | 104 | // UUID ... Returns component's CUUID 105 | func (meta *metaData) UUID() core.CUUID { 106 | return meta.id 107 | } 108 | 109 | // UUID ... Returns component's PUUID 110 | // NOTE - This currently assumes that component collisions are impossible 111 | func (meta *metaData) PUUID() core.PUUID { 112 | return meta.pUUID 113 | } 114 | 115 | // Type ... Returns component's type 116 | func (meta *metaData) Type() core.ComponentType { 117 | return meta.cType 118 | } 119 | 120 | // OutputType ... Returns component's data output type 121 | func (meta *metaData) OutputType() core.RegisterType { 122 | return meta.output 123 | } 124 | 125 | // emitStateChange ... Emits a stateChange event to stateChan 126 | func (meta *metaData) emitStateChange(as ActivityState) { 127 | event := StateChange{ 128 | ID: meta.id, 129 | From: meta.state, 130 | To: as, 131 | } 132 | 133 | meta.state = as 134 | meta.stateChan <- event // Send to upstream consumers 135 | } 136 | 137 | // Option ... Component type agnostic option 138 | type Option = func(*metaData) 139 | 140 | // WithCUUID ... Passes component UUID to component metadata field 141 | func WithCUUID(id core.CUUID) Option { 142 | return func(meta *metaData) { 143 | meta.id = id 144 | } 145 | } 146 | 147 | // WithPUUID ... Passes component PUUID to component metadata field 148 | func WithPUUID(pUUID core.PUUID) Option { 149 | return func(meta *metaData) { 150 | meta.pUUID = pUUID 151 | } 152 | } 153 | 154 | // WithEventChan ... Passes state channel to component metadata field 155 | func WithEventChan(sc chan StateChange) Option { 156 | return func(md *metaData) { 157 | md.stateChan = sc 158 | } 159 | } 160 | 161 | // WithInTypes ... Passes input types to component metadata field 162 | func WithInTypes(its []core.RegisterType) Option { 163 | return func(md *metaData) { 164 | md.inTypes = its 165 | } 166 | } 167 | 168 | // WithStateKey ... Passes state key to component metadata field 169 | func WithStateKey(key *core.StateKey) Option { 170 | return func(md *metaData) { 171 | md.sk = key 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /internal/etl/component/component_test.go: -------------------------------------------------------------------------------- 1 | package component 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/base-org/pessimism/internal/core" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func Test_MetaData(t *testing.T) { 12 | var tests = []struct { 13 | name string 14 | description string 15 | function string 16 | 17 | constructionLogic func() *metaData 18 | testLogic func(*testing.T, *metaData) 19 | }{ 20 | { 21 | name: "Test State Change Emit", 22 | description: "When emitStateChange is called, a new state should be state for metadata and sent in channel", 23 | function: "emitStateChange", 24 | 25 | constructionLogic: func() *metaData { 26 | return newMetaData(0, 0) 27 | }, 28 | 29 | testLogic: func(t *testing.T, md *metaData) { 30 | 31 | go func() { 32 | // Simulate a component ending itself 33 | md.emitStateChange(Terminated) 34 | 35 | }() 36 | 37 | sChange := <-md.stateChan 38 | 39 | assert.Equal(t, sChange.From, Inactive) 40 | assert.Equal(t, sChange.To, Terminated) 41 | assert.Equal(t, sChange.ID, core.NilCUUID()) 42 | }, 43 | }, 44 | } 45 | for i, tc := range tests { 46 | t.Run(fmt.Sprintf("%d-%s", i, tc.name), func(t *testing.T) { 47 | testMeta := tc.constructionLogic() 48 | tc.testLogic(t, testMeta) 49 | }) 50 | 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /internal/etl/component/egress.go: -------------------------------------------------------------------------------- 1 | package component 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | ) 8 | 9 | // egressHandler ... Used to route transit data from a component to it's respective edge components. 10 | // Also used to manage egresses or "edge routes" for some component. 11 | type egressHandler struct { 12 | egresses map[core.ComponentPID]chan core.TransitData 13 | 14 | relay *core.EngineInputRelay 15 | } 16 | 17 | // newEgress ... Initializer 18 | func newEgressHandler() *egressHandler { 19 | return &egressHandler{ 20 | egresses: make(map[core.ComponentPID]chan core.TransitData), 21 | relay: nil, 22 | } 23 | } 24 | 25 | // Send ... Sends single piece of transitData to all innner mapping value channels 26 | func (eh *egressHandler) Send(td core.TransitData) error { 27 | if len(eh.egresses) == 0 && !eh.HasEngineRelay() { 28 | return fmt.Errorf(egressNotExistErr) 29 | } 30 | 31 | if eh.HasEngineRelay() { 32 | if err := eh.relay.RelayTransitData(td); err != nil { 33 | return err 34 | } 35 | } 36 | 37 | // NOTE - Consider introducing a fail safe timeout to ensure that freezing on clogged chanel buffers is recognized 38 | for _, channel := range eh.egresses { 39 | channel <- td 40 | } 41 | 42 | return nil 43 | } 44 | 45 | // SendBatch ... Sends slice of transitData to all innner mapping value channels 46 | func (eh *egressHandler) SendBatch(dataSlice []core.TransitData) error { 47 | // NOTE - Consider introducing a fail safe timeout to ensure that freezing on clogged chanel buffers is recognized 48 | for _, data := range dataSlice { 49 | // NOTE - Does it make sense to fail loudly here? 50 | 51 | if err := eh.Send(data); err != nil { 52 | return err 53 | } 54 | } 55 | 56 | return nil 57 | } 58 | 59 | // AddEgress ... Inserts a new egress given an ID and channel; fail on key collision 60 | func (eh *egressHandler) AddEgress(componentID core.CUUID, outChan chan core.TransitData) error { 61 | if _, found := eh.egresses[componentID.PID]; found { 62 | return fmt.Errorf(egressAlreadyExistsErr, componentID.String()) 63 | } 64 | 65 | eh.egresses[componentID.PID] = outChan 66 | return nil 67 | } 68 | 69 | // RemoveEgress ... Removes an egress given an ID; fail if no key found 70 | func (eh *egressHandler) RemoveEgress(componentID core.CUUID) error { 71 | if _, found := eh.egresses[componentID.PID]; !found { 72 | return fmt.Errorf(egressNotFoundErr, componentID.PID.String()) 73 | } 74 | 75 | delete(eh.egresses, componentID.PID) 76 | return nil 77 | } 78 | 79 | // HasEngineRelay ... Returns true if engine relay exists, false otherwise 80 | func (eh *egressHandler) HasEngineRelay() bool { 81 | return eh.relay != nil 82 | } 83 | 84 | // AddRelay ... Adds a relay assuming no existings ones 85 | func (eh *egressHandler) AddRelay(relay *core.EngineInputRelay) error { 86 | if eh.HasEngineRelay() { 87 | return fmt.Errorf(engineEgressExistsErr) 88 | } 89 | 90 | eh.relay = relay 91 | return nil 92 | } 93 | -------------------------------------------------------------------------------- /internal/etl/component/ingress.go: -------------------------------------------------------------------------------- 1 | package component 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | ) 8 | 9 | // ingressHandler ... Used to manage ingresses for some component 10 | // NOTE: An edge is only possible between two components (C0, C1) where C0 -> C1 11 | // if C0.outGresses[registerType] ⊆ C1.ingresses 12 | type ingressHandler struct { 13 | ingreses map[core.RegisterType]chan core.TransitData 14 | } 15 | 16 | // newIngressHandler ... Initializer 17 | func newIngressHandler() *ingressHandler { 18 | return &ingressHandler{ 19 | ingreses: make(map[core.RegisterType]chan core.TransitData), 20 | } 21 | } 22 | 23 | // GetIngress ... Fetches ingress channel for some register type 24 | func (ih *ingressHandler) GetIngress(rt core.RegisterType) (chan core.TransitData, error) { 25 | val, found := ih.ingreses[rt] 26 | if !found { 27 | return nil, fmt.Errorf(ingressNotFoundErr, rt.String()) 28 | } 29 | 30 | return val, nil 31 | } 32 | 33 | // createIngress ... Creates ingress channel for some register type 34 | func (ih *ingressHandler) createIngress(rt core.RegisterType) error { 35 | if _, found := ih.ingreses[rt]; found { 36 | return fmt.Errorf(ingressAlreadyExistsErr, rt.String()) 37 | } 38 | 39 | ih.ingreses[rt] = core.NewTransitChannel() 40 | 41 | return nil 42 | } 43 | -------------------------------------------------------------------------------- /internal/etl/component/ingress_test.go: -------------------------------------------------------------------------------- 1 | package component 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/base-org/pessimism/internal/core" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func Test_Add_Remove_Ingress(t *testing.T) { 12 | var tests = []struct { 13 | name string 14 | description string 15 | 16 | constructionLogic func() *ingressHandler 17 | testLogic func(*testing.T, *ingressHandler) 18 | }{ 19 | { 20 | name: "Successful Add Test", 21 | description: "When a register type is passed to AddIngress function, it should be successfully added to handler ingress mapping", 22 | 23 | constructionLogic: func() *ingressHandler { 24 | handler := newIngressHandler() 25 | return handler 26 | }, 27 | 28 | testLogic: func(t *testing.T, ih *ingressHandler) { 29 | 30 | err := ih.createIngress(core.GethBlock) 31 | assert.NoError(t, err, "geth.block register should added as an egress") 32 | 33 | }, 34 | }, 35 | { 36 | name: "Failed Add Test", 37 | description: "When the same register type is added twice to AddIngress function, the second add should fail with key collisions", 38 | 39 | constructionLogic: func() *ingressHandler { 40 | handler := newIngressHandler() 41 | if err := handler.createIngress(core.GethBlock); err != nil { 42 | panic(err) 43 | } 44 | 45 | return handler 46 | }, 47 | 48 | testLogic: func(t *testing.T, ih *ingressHandler) { 49 | err := ih.createIngress(core.GethBlock) 50 | 51 | assert.Error(t, err, "geth.block register should fail to be added") 52 | assert.Equal(t, err.Error(), fmt.Sprintf(ingressAlreadyExistsErr, core.GethBlock.String())) 53 | 54 | }, 55 | }, 56 | } 57 | 58 | for i, tc := range tests { 59 | t.Run(fmt.Sprintf("%d-%s", i, tc.name), func(t *testing.T) { 60 | testIngress := tc.constructionLogic() 61 | tc.testLogic(t, testIngress) 62 | }) 63 | 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /internal/etl/component/oracle.go: -------------------------------------------------------------------------------- 1 | package component 2 | 3 | import ( 4 | "context" 5 | "math/big" 6 | "sync" 7 | 8 | "github.com/base-org/pessimism/internal/core" 9 | "github.com/base-org/pessimism/internal/logging" 10 | "go.uber.org/zap" 11 | ) 12 | 13 | // OracleDefinition ... Provides a generalized interface for developers to bind their own functionality to 14 | type OracleDefinition interface { 15 | ConfigureRoutine(pUUID core.PUUID) error 16 | BackTestRoutine(ctx context.Context, componentChan chan core.TransitData, 17 | startHeight *big.Int, endHeight *big.Int) error 18 | ReadRoutine(ctx context.Context, componentChan chan core.TransitData) error 19 | } 20 | 21 | // Oracle ... Component used to represent a data source reader; E.g, Eth block indexing, interval API polling 22 | type Oracle struct { 23 | ctx context.Context 24 | 25 | definition OracleDefinition 26 | oracleChannel chan core.TransitData 27 | 28 | wg *sync.WaitGroup 29 | 30 | *metaData 31 | } 32 | 33 | // NewOracle ... Initializer 34 | func NewOracle(ctx context.Context, outType core.RegisterType, 35 | od OracleDefinition, opts ...Option) (Component, error) { 36 | o := &Oracle{ 37 | ctx: ctx, 38 | definition: od, 39 | oracleChannel: core.NewTransitChannel(), 40 | wg: &sync.WaitGroup{}, 41 | 42 | metaData: newMetaData(core.Oracle, outType), 43 | } 44 | 45 | for _, opt := range opts { 46 | opt(o.metaData) 47 | } 48 | 49 | logging.WithContext(ctx).Info("Constructed component", 50 | zap.String(core.CUUIDKey, o.metaData.id.String())) 51 | 52 | return o, nil 53 | } 54 | 55 | // Close ... This function is called at the end when processes related to oracle need to shut down 56 | func (o *Oracle) Close() error { 57 | logging.WithContext(o.ctx). 58 | Info("Waiting for oracle definition go routines to finish", 59 | zap.String(core.CUUIDKey, o.id.String())) 60 | o.closeChan <- killSig 61 | 62 | o.wg.Wait() 63 | logging.WithContext(o.ctx).Info("Oracle definition go routines have exited", 64 | zap.String(core.CUUIDKey, o.id.String())) 65 | return nil 66 | } 67 | 68 | // EventLoop ... Component loop that actively waits and transits register data 69 | // from a channel that the definition's read routine writes to 70 | func (o *Oracle) EventLoop() error { 71 | // TODO(#24) - Add Internal Component Activity State Tracking 72 | 73 | err := o.definition.ConfigureRoutine(o.pUUID) 74 | if err != nil { 75 | return err 76 | } 77 | 78 | logger := logging.WithContext(o.ctx) 79 | 80 | logger.Debug("Starting component event loop", 81 | zap.String(core.CUUIDKey, o.id.String())) 82 | 83 | o.wg.Add(1) 84 | 85 | routineCtx, cancel := context.WithCancel(o.ctx) 86 | // o.emitStateChange(Live) 87 | 88 | // Spawn definition read routine 89 | go func() { 90 | defer o.wg.Done() 91 | if err := o.definition.ReadRoutine(routineCtx, o.oracleChannel); err != nil { 92 | logger.Error("Received error from read routine", 93 | zap.String(core.CUUIDKey, o.id.String()), 94 | zap.Error(err)) 95 | } 96 | }() 97 | 98 | for { 99 | select { 100 | case registerData := <-o.oracleChannel: 101 | logger.Debug("Sending data", 102 | zap.String(core.CUUIDKey, o.id.String())) 103 | 104 | if err := o.egressHandler.Send(registerData); err != nil { 105 | logger.Error(transitErr, zap.String("ID", o.id.String())) 106 | } 107 | 108 | case <-o.closeChan: 109 | logger.Debug("Received component shutdown signal", 110 | zap.String(core.CUUIDKey, o.id.String())) 111 | 112 | // o.emitStateChange(Terminated) 113 | logger.Debug("Closing component channel and context", 114 | zap.String(core.CUUIDKey, o.id.String())) 115 | close(o.oracleChannel) 116 | cancel() // End definition routine 117 | 118 | logger.Debug("Component shutdown success", 119 | zap.String(core.CUUIDKey, o.id.String())) 120 | return nil 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /internal/etl/component/pipe.go: -------------------------------------------------------------------------------- 1 | package component 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | "github.com/base-org/pessimism/internal/logging" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | type PipeDefinition interface { 12 | Transform(ctx context.Context, data core.TransitData) ([]core.TransitData, error) 13 | ConfigureRoutine(pUUID core.PUUID) error 14 | } 15 | 16 | // Pipe ... Component used to represent any arbitrary computation; pipes must can read from all component types 17 | // E.G. (ORACLE || CONVEYOR || PIPE) -> PIPE 18 | 19 | type Pipe struct { 20 | ctx context.Context 21 | inType core.RegisterType 22 | 23 | def PipeDefinition 24 | 25 | *metaData 26 | } 27 | 28 | // NewPipe ... Initializer 29 | func NewPipe(ctx context.Context, pd PipeDefinition, inType core.RegisterType, 30 | outType core.RegisterType, opts ...Option) (Component, error) { 31 | // TODO - Validate inTypes size 32 | 33 | pipe := &Pipe{ 34 | ctx: ctx, 35 | def: pd, 36 | inType: inType, 37 | 38 | metaData: newMetaData(core.Pipe, outType), 39 | } 40 | 41 | if err := pipe.createIngress(inType); err != nil { 42 | return nil, err 43 | } 44 | 45 | for _, opt := range opts { 46 | opt(pipe.metaData) 47 | } 48 | 49 | return pipe, nil 50 | } 51 | 52 | // Close ... Shuts down component by emitting a kill signal to a close channel 53 | func (p *Pipe) Close() error { 54 | p.closeChan <- killSig 55 | 56 | return nil 57 | } 58 | 59 | // EventLoop ... Driver loop for component that actively subscribes 60 | // to an input channel where transit data is read, transformed, and transitte 61 | // to downstream components 62 | func (p *Pipe) EventLoop() error { 63 | logger := logging.WithContext(p.ctx) 64 | 65 | logger.Info("Starting event loop", 66 | zap.String("ID", p.id.String()), 67 | ) 68 | 69 | inChan, err := p.GetIngress(p.inType) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | if err = p.def.ConfigureRoutine(p.pUUID); err != nil { 75 | return err 76 | } 77 | 78 | for { 79 | select { 80 | case inputData := <-inChan: 81 | outputData, err := p.def.Transform(p.ctx, inputData) 82 | if err != nil { 83 | // TODO - Introduce metrics service (`prometheus`) call 84 | logger.Error(err.Error(), zap.String("ID", p.id.String())) 85 | continue 86 | } 87 | 88 | if length := len(outputData); length > 0 { 89 | logger.Debug("Received tranformation output data", 90 | zap.String("ID", p.id.String()), 91 | zap.Int("Length", length)) 92 | } else { 93 | logger.Debug("Received output data of length 0", 94 | zap.String("ID", p.id.String())) 95 | continue 96 | } 97 | 98 | logger.Debug("Sending data batch", 99 | zap.String("ID", p.id.String()), 100 | zap.String("Type", p.OutputType().String())) 101 | 102 | if err := p.egressHandler.SendBatch(outputData); err != nil { 103 | logger.Error(transitErr, zap.String("ID", p.id.String())) 104 | } 105 | 106 | // Manager is telling us to shutdown 107 | case <-p.closeChan: 108 | logger.Debug("Received component shutdown signal", 109 | zap.String("ID", p.id.String())) 110 | 111 | // p.emitStateChange(Terminated) 112 | 113 | return nil 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /internal/etl/component/pipe_test.go: -------------------------------------------------------------------------------- 1 | package component_test 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "sync" 7 | "testing" 8 | "time" 9 | 10 | "github.com/base-org/pessimism/internal/core" 11 | "github.com/base-org/pessimism/internal/mocks" 12 | "github.com/ethereum/go-ethereum/common" 13 | "github.com/ethereum/go-ethereum/core/types" 14 | "github.com/ethereum/go-ethereum/rlp" 15 | "github.com/stretchr/testify/assert" 16 | ) 17 | 18 | func Test_Pipe_Event_Flow(t *testing.T) { 19 | ctx, cancel := context.WithCancel(context.Background()) 20 | defer cancel() 21 | 22 | ts := time.Date(1969, time.April, 1, 4, 20, 0, 0, time.Local) 23 | 24 | // Setup component dependencies 25 | testID := core.MakeCUUID(6, 9, 6, 9) 26 | 27 | outputChan := make(chan core.TransitData) 28 | 29 | // Construct test component 30 | testPipe, err := mocks.NewMockPipe(ctx, core.GethBlock, core.EventLog) 31 | assert.NoError(t, err) 32 | 33 | err = testPipe.AddEgress(testID, outputChan) 34 | assert.NoError(t, err) 35 | 36 | // Encoded value taken from https://github.com/ethereum/go-ethereum/blob/master/core/types/block_test.go#L36 37 | blockEnc := common.FromHex("f9030bf901fea083cafc574e1f51ba9dc0568fc617a08ea2429fb384059c972f13b19fa1c8dd55a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a0ef1552a40b7165c3cd773806b9e0c165b75356e0314bf0706f279c729f51e017a05fe50b260da6308036625b850b5d6ced6d0a9f814c0688bc91ffb7b7a3a54b67a0bc37d79753ad738a6dac4921e57392f145d8887476de3f783dfa7edae9283e52b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001832fefd8825208845506eb0780a0bd4472abb6659ebe3ee06ee4d7b72a00a9f4d001caca51342001075469aff49888a13a5a8c8f2bb1c4843b9aca00f90106f85f800a82c35094095e7baea6a6c7c4c2dfeb977efac326af552d870a801ba09bea4c4daac7c7c52e093e6a4c35dbbcf8856f1af7b059ba20253e70848d094fa08a8fae537ce25ed8cb5af9adac3f141af69bd515bd2ba031522df09b97dd72b1b8a302f8a0018080843b9aca008301e24194095e7baea6a6c7c4c2dfeb977efac326af552d878080f838f7940000000000000000000000000000000000000001e1a0000000000000000000000000000000000000000000000000000000000000000080a0fe38ca4e44a30002ac54af7cf922a6ac2ba11b7d22f548e8ecb3f51f41cb31b0a06de6a5cbae13c0c856e33acf021b51819636cfc009d39eafb9f606d546e305a8c0") 38 | 39 | var block types.Block 40 | err = rlp.DecodeBytes(blockEnc, &block) 41 | assert.NoError(t, err) 42 | 43 | // Start component event loop on separate go routine 44 | go func() { 45 | if err := testPipe.EventLoop(); err != nil { 46 | log.Printf("Got error from testPipe event loop %s", err.Error()) 47 | } 48 | }() 49 | 50 | wg := sync.WaitGroup{} 51 | 52 | inputData := core.TransitData{ 53 | Timestamp: ts, 54 | Type: core.GethBlock, 55 | Value: block, 56 | } 57 | var outputData core.TransitData 58 | 59 | // Spawn listener routine that reads for output from testPipe 60 | wg.Add(1) 61 | go func() { 62 | defer wg.Done() 63 | 64 | // Read first value from channel and return 65 | for output := range outputChan { 66 | outputData = output 67 | return 68 | } 69 | 70 | }() 71 | 72 | entryChan, err := testPipe.GetIngress(core.GethBlock) 73 | assert.NoError(t, err) 74 | 75 | entryChan <- inputData 76 | 77 | // Wait for pipe to transform block data into a transaction slice 78 | wg.Wait() 79 | 80 | assert.NotNil(t, outputData) 81 | 82 | _, success := outputData.Value.(types.Block) 83 | assert.True(t, success) 84 | assert.Equal(t, outputData.Timestamp, ts, "Timestamp failed to verify") 85 | 86 | } 87 | -------------------------------------------------------------------------------- /internal/etl/component/types.go: -------------------------------------------------------------------------------- 1 | package component 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | ) 8 | 9 | type ActivityState int 10 | 11 | const ( 12 | Inactive ActivityState = iota 13 | Live 14 | Terminated 15 | ) 16 | 17 | func (as ActivityState) String() string { 18 | switch as { 19 | case Inactive: 20 | return "inactive" 21 | 22 | case Live: 23 | return "live" 24 | 25 | case Terminated: 26 | return "terminated" 27 | } 28 | 29 | return "unknown" 30 | } 31 | 32 | // StateChange ... Represents a component state change event 33 | // that is processed by component management logic to determine 34 | // proper pipeline states and duplicate pipeline merging opportunities 35 | type StateChange struct { 36 | ID core.CUUID 37 | 38 | From ActivityState // S 39 | To ActivityState // S' 40 | } 41 | 42 | // EgressHandler specific errors 43 | const ( 44 | engineEgressExistsErr = "engine egress already exists" 45 | egressAlreadyExistsErr = "%s egress key already exists within component router mapping" 46 | egressNotFoundErr = "no egress key %s exists within component router mapping" 47 | egressNotExistErr = "received transit request with 0 out channels to write to" 48 | 49 | transitErr = "received transit error: %s" 50 | ) 51 | 52 | // IngressHandler specific errors 53 | const ( 54 | ingressAlreadyExistsErr = "ingress already exists for %s" 55 | ingressNotFoundErr = "ingress not found for %s" 56 | ) 57 | 58 | type ( 59 | // OracleConstructorFunc ... Type declaration that a registry oracle component constructor must adhere to 60 | OracleConstructorFunc = func(context.Context, *core.ClientConfig, ...Option) (Component, error) 61 | 62 | // PipeConstructorFunc ... Type declaration that a registry pipe component constructor must adhere to 63 | PipeConstructorFunc = func(context.Context, *core.ClientConfig, ...Option) (Component, error) 64 | ) 65 | 66 | // OracleType ... 67 | type OracleType = string 68 | 69 | const ( 70 | // BackTestOracle ... Represents an oracle used for backtesting some invariant 71 | BacktestOracle OracleType = "backtest" 72 | // LiveOracle ... Represents an oracle used for powering some live invariant 73 | LiveOracle OracleType = "live" 74 | ) 75 | -------------------------------------------------------------------------------- /internal/etl/pipeline/analysis.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | "github.com/base-org/pessimism/internal/etl/component" 8 | "github.com/base-org/pessimism/internal/etl/registry" 9 | "github.com/base-org/pessimism/internal/state" 10 | ) 11 | 12 | // Analyzer ... Interface for analyzing pipelines 13 | type Analyzer interface { 14 | Mergable(p1 Pipeline, p2 Pipeline) bool 15 | MergePipelines(ctx context.Context, p1 Pipeline, p2 Pipeline) (Pipeline, error) 16 | } 17 | 18 | // analyzer ... Implementation of Analyzer 19 | type analyzer struct { 20 | dRegistry registry.Registry 21 | } 22 | 23 | // NewAnalyzer ... Initializer 24 | func NewAnalyzer(dRegistry registry.Registry) Analyzer { 25 | return &analyzer{ 26 | dRegistry: dRegistry, 27 | } 28 | } 29 | 30 | // Mergable ... Returns true if pipelines can be merged or deduped 31 | func (a *analyzer) Mergable(p1 Pipeline, p2 Pipeline) bool { 32 | // Invalid if pipelines are not the same length 33 | if len(p1.Components()) != len(p2.Components()) { 34 | return false 35 | } 36 | 37 | // Invalid if pipelines are not live 38 | if p1.Config().PipelineType != core.Live || 39 | p2.Config().PipelineType != core.Live { 40 | return false 41 | } 42 | 43 | // Invalid if either pipeline requires a backfill 44 | // NOTE - This is a temporary solution to prevent live backfills on two pipelines 45 | // from being merged. 46 | // In the future, this should only check the current state of each pipeline 47 | // to ensure that the backfill has been completed for both. 48 | if p1.Config().ClientConfig.Backfill() || 49 | p2.Config().ClientConfig.Backfill() { 50 | return false 51 | } 52 | 53 | // Invalid if pipelines do not share the same PID 54 | if p1.UUID().PID != p2.UUID().PID { 55 | return false 56 | } 57 | 58 | return true 59 | } 60 | 61 | // MergePipelines ... Merges two pipelines into one (p1 --merge-> p2) 62 | func (a *analyzer) MergePipelines(ctx context.Context, p1 Pipeline, p2 Pipeline) (Pipeline, error) { 63 | for i, compi := range p1.Components() { 64 | compj := p2.Components()[i] 65 | 66 | reg, err := a.dRegistry.GetRegister(compi.OutputType()) 67 | if err != nil { 68 | return nil, err 69 | } 70 | 71 | if reg.Stateful() { // Merge state items from compi into compj 72 | err = a.mergeComponentState(ctx, compi, compj, p1.UUID(), p2.UUID()) 73 | if err != nil { 74 | return nil, err 75 | } 76 | } 77 | } 78 | return p2, nil 79 | } 80 | 81 | // mergeComponentState ... Merges state items from p2 into p1 82 | func (a *analyzer) mergeComponentState(ctx context.Context, compi, compj component.Component, 83 | p1, p2 core.PUUID) error { 84 | ss, err := state.FromContext(ctx) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | items, err := ss.GetSlice(ctx, compi.StateKey()) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | for _, item := range items { 95 | _, err := ss.SetSlice(ctx, compj.StateKey(), item) 96 | if err != nil { 97 | return err 98 | } 99 | } 100 | 101 | if compi.StateKey().IsNested() { 102 | err = a.MergeNestedStateKeys(ctx, compi, compj, p1, p2, ss) 103 | if err != nil { 104 | return err 105 | } 106 | } 107 | 108 | return nil 109 | } 110 | 111 | // MergeNestedStateKeys ... Merges nested state keys from p1 into p2 112 | func (a *analyzer) MergeNestedStateKeys(ctx context.Context, c1, c2 component.Component, 113 | p1, p2 core.PUUID, ss state.Store) error { 114 | items, err := ss.GetSlice(ctx, c1.StateKey()) 115 | if err != nil { 116 | return err 117 | } 118 | 119 | for _, item := range items { 120 | key1 := &core.StateKey{ 121 | Prefix: c1.OutputType(), 122 | ID: item, 123 | PUUID: &p1, 124 | } 125 | 126 | key2 := &core.StateKey{ 127 | Prefix: c2.OutputType(), 128 | ID: item, 129 | PUUID: &p2, 130 | } 131 | 132 | nestedValues, err := ss.GetSlice(ctx, key1) 133 | if err != nil { 134 | return err 135 | } 136 | 137 | for _, value := range nestedValues { 138 | _, err = ss.SetSlice(ctx, key2, value) 139 | if err != nil { 140 | return err 141 | } 142 | } 143 | 144 | err = ss.Remove(ctx, key1) 145 | if err != nil { 146 | return err 147 | } 148 | } 149 | 150 | return nil 151 | } 152 | -------------------------------------------------------------------------------- /internal/etl/pipeline/analysis_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/base-org/pessimism/internal/core" 8 | "github.com/base-org/pessimism/internal/etl/component" 9 | "github.com/base-org/pessimism/internal/etl/pipeline" 10 | "github.com/base-org/pessimism/internal/etl/registry" 11 | "github.com/base-org/pessimism/internal/mocks" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func Test_Mergable(t *testing.T) { 16 | var tests = []struct { 17 | name string 18 | function string 19 | description string 20 | testConstructor func() pipeline.Analyzer 21 | testLogic func(t *testing.T, a pipeline.Analyzer) 22 | }{ 23 | { 24 | name: "Successful Pipeline Merge", 25 | function: "Mergable", 26 | description: "Mergable function should return true if pipelines are mergable", 27 | testConstructor: func() pipeline.Analyzer { 28 | dRegistry := registry.NewRegistry() 29 | return pipeline.NewAnalyzer(dRegistry) 30 | }, 31 | testLogic: func(t *testing.T, a pipeline.Analyzer) { 32 | // Setup test pipelines 33 | mockOracle, err := mocks.NewMockOracle(context.Background(), core.GethBlock) 34 | assert.NoError(t, err) 35 | 36 | comps := []component.Component{mockOracle} 37 | testPUUID := core.MakePUUID(0, core.MakeCUUID(core.Live, 0, 0, 0), core.MakeCUUID(core.Live, 0, 0, 0)) 38 | testPUUID2 := core.MakePUUID(0, core.MakeCUUID(core.Live, 0, 0, 0), core.MakeCUUID(core.Live, 0, 0, 0)) 39 | 40 | testCfg := &core.PipelineConfig{ 41 | PipelineType: core.Live, 42 | ClientConfig: &core.ClientConfig{}, 43 | } 44 | 45 | p1, err := pipeline.NewPipeline(testCfg, testPUUID, comps) 46 | assert.NoError(t, err) 47 | 48 | p2, err := pipeline.NewPipeline(testCfg, testPUUID2, comps) 49 | assert.NoError(t, err) 50 | 51 | assert.True(t, a.Mergable(p1, p2)) 52 | }, 53 | }, 54 | { 55 | name: "Failure Pipeline Merge", 56 | function: "Mergable", 57 | description: "Mergable function should return false when PID's do not match", 58 | testConstructor: func() pipeline.Analyzer { 59 | dRegistry := registry.NewRegistry() 60 | return pipeline.NewAnalyzer(dRegistry) 61 | }, 62 | testLogic: func(t *testing.T, a pipeline.Analyzer) { 63 | // Setup test pipelines 64 | mockOracle, err := mocks.NewMockOracle(context.Background(), core.GethBlock) 65 | assert.NoError(t, err) 66 | 67 | comps := []component.Component{mockOracle} 68 | testPUUID := core.MakePUUID(0, core.MakeCUUID(core.Backtest, 0, 0, 0), core.MakeCUUID(core.Live, 0, 0, 0)) 69 | testPUUID2 := core.MakePUUID(0, core.MakeCUUID(core.Live, 0, 0, 0), core.MakeCUUID(core.Live, 0, 0, 0)) 70 | 71 | testCfg := &core.PipelineConfig{ 72 | PipelineType: core.Live, 73 | ClientConfig: &core.ClientConfig{}, 74 | } 75 | 76 | p1, err := pipeline.NewPipeline(testCfg, testPUUID, comps) 77 | assert.NoError(t, err) 78 | 79 | p2, err := pipeline.NewPipeline(testCfg, testPUUID2, comps) 80 | assert.NoError(t, err) 81 | 82 | assert.False(t, a.Mergable(p1, p2)) 83 | }, 84 | }, 85 | } 86 | 87 | for _, test := range tests { 88 | t.Run(test.name, func(t *testing.T) { 89 | a := test.testConstructor() 90 | test.testLogic(t, a) 91 | }) 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /internal/etl/pipeline/manager_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/base-org/pessimism/internal/core" 9 | "github.com/base-org/pessimism/internal/etl/registry" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | // TODO(#33): No Unit Tests for Pipeline & ETL Manager Logic 14 | func Test_Manager(t *testing.T) { 15 | var tests = []struct { 16 | name string 17 | function string 18 | description string 19 | 20 | constructionLogic func() Manager 21 | testLogic func(t *testing.T, m Manager) 22 | }{ 23 | { 24 | name: "Successful Pipe Component Construction", 25 | function: "inferComponent", 26 | description: "inferComponent function should generate pipe component instance provided valid params", 27 | 28 | constructionLogic: func() Manager { 29 | reg := registry.NewRegistry() 30 | return NewManager(context.Background(), NewAnalyzer(reg), reg, NewEtlStore(), NewComponentGraph(), nil) 31 | }, 32 | 33 | testLogic: func(t *testing.T, m Manager) { 34 | cUUID := core.MakeCUUID(1, 1, 1, 1) 35 | 36 | register, err := registry.NewRegistry().GetRegister(core.GethBlock) 37 | 38 | assert.NoError(t, err) 39 | 40 | cc := &core.ClientConfig{} 41 | c, err := inferComponent(context.Background(), cc, cUUID, core.NilPUUID(), register) 42 | assert.NoError(t, err) 43 | 44 | assert.Equal(t, c.UUID(), cUUID) 45 | assert.Equal(t, c.Type(), register.ComponentType) 46 | assert.Equal(t, c.OutputType(), register.DataType) 47 | 48 | }, 49 | }, 50 | } 51 | 52 | for i, tc := range tests { 53 | t.Run(fmt.Sprintf("%d-%s-%s", i, tc.function, tc.name), func(t *testing.T) { 54 | testPipeline := tc.constructionLogic() 55 | tc.testLogic(t, testPipeline) 56 | }) 57 | 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /internal/etl/pipeline/pipeline.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | "github.com/base-org/pessimism/internal/etl/component" 8 | "github.com/base-org/pessimism/internal/logging" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | // Pipeline ... Pipeline interface 13 | type Pipeline interface { 14 | Config() *core.PipelineConfig 15 | UUID() core.PUUID 16 | Close() error 17 | Components() []component.Component 18 | RunPipeline(wg *sync.WaitGroup) error 19 | 20 | AddEngineRelay(engineChan chan core.InvariantInput) error 21 | } 22 | 23 | // pipeline ... Pipeline implementation 24 | type pipeline struct { 25 | cfg *core.PipelineConfig 26 | uuid core.PUUID 27 | 28 | aState ActivityState 29 | pType core.PipelineType //nolint:unused // will be implemented soon 30 | 31 | components []component.Component 32 | } 33 | 34 | // NewPipeline ... Initializer 35 | func NewPipeline(cfg *core.PipelineConfig, pUUID core.PUUID, comps []component.Component) (Pipeline, error) { 36 | pl := &pipeline{ 37 | cfg: cfg, 38 | uuid: pUUID, 39 | components: comps, 40 | aState: Booting, 41 | } 42 | 43 | return pl, nil 44 | } 45 | 46 | // Config ... Returns pipeline config 47 | func (pl *pipeline) Config() *core.PipelineConfig { 48 | return pl.cfg 49 | } 50 | 51 | // Components ... Returns slice of all constituent components 52 | func (pl *pipeline) Components() []component.Component { 53 | return pl.components 54 | } 55 | 56 | // UUID ... Returns pipeline UUID 57 | func (pl *pipeline) UUID() core.PUUID { 58 | return pl.uuid 59 | } 60 | 61 | // AddEngineRelay ... Adds a relay to the pipeline that forces it to send transformed invariant input 62 | // to a risk engine 63 | func (pl *pipeline) AddEngineRelay(engineChan chan core.InvariantInput) error { 64 | lastComponent := pl.components[0] 65 | eir := core.NewEngineRelay(pl.uuid, engineChan) 66 | 67 | logging.NoContext().Debug("Adding engine relay to pipeline", 68 | zap.String(core.CUUIDKey, lastComponent.UUID().String()), 69 | zap.String(core.PUUIDKey, pl.uuid.String())) 70 | 71 | return lastComponent.AddRelay(eir) 72 | } 73 | 74 | // RunPipeline ... Spawns and manages component event loops 75 | // for some pipeline 76 | func (pl *pipeline) RunPipeline(wg *sync.WaitGroup) error { 77 | for _, comp := range pl.components { 78 | wg.Add(1) 79 | // NOTE - This is a hack and a bit leaky since 80 | // we're teaching callee level absractions about the pipelines 81 | // which they execute within. 82 | 83 | go func(c component.Component, wg *sync.WaitGroup) { 84 | defer wg.Done() 85 | 86 | logging.NoContext(). 87 | Debug("Attempting to start component event loop", 88 | zap.String(core.CUUIDKey, c.UUID().String()), 89 | zap.String(core.PUUIDKey, pl.uuid.String())) 90 | 91 | if err := c.EventLoop(); err != nil { 92 | logging.NoContext().Error("Obtained error from event loop", zap.Error(err), 93 | zap.String(core.CUUIDKey, c.UUID().String()), 94 | zap.String(core.PUUIDKey, pl.uuid.String())) 95 | } 96 | }(comp, wg) 97 | } 98 | 99 | return nil 100 | } 101 | 102 | // Close ... Closes all components in the pipeline 103 | func (pl *pipeline) Close() error { 104 | for _, comp := range pl.components { 105 | if comp.ActivityState() != component.Terminated { 106 | logging.NoContext(). 107 | Debug("Shutting down pipeline component", 108 | zap.String(core.CUUIDKey, comp.UUID().String()), 109 | zap.String(core.PUUIDKey, pl.uuid.String())) 110 | 111 | if err := comp.Close(); err != nil { 112 | return err 113 | } 114 | } 115 | } 116 | return nil 117 | } 118 | -------------------------------------------------------------------------------- /internal/etl/pipeline/pipeline_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | // TODO(#33): No Unit Tests for Pipeline & ETL Manager Logic 9 | 10 | func Test_Pipeline(t *testing.T) { 11 | var tests = []struct { 12 | name string 13 | function string 14 | description string 15 | 16 | constructionLogic func() Pipeline 17 | testLogic func(t *testing.T, pl Pipeline) 18 | }{ 19 | // { 20 | // name: "Successful Add When PID Already Exists", 21 | // function: "addPipeline", 22 | // description: "", 23 | 24 | // constructionLogic: func() Pipeline { 25 | // return getTestPipeLine(context.Background()) 26 | // }, 27 | 28 | // testLogic: func(t *testing.T, pl Pipeline) { 29 | // wg := sync.WaitGroup{} 30 | 31 | // err := pl.RunPipeline(&wg) 32 | // assert.NoError(t, err) 33 | 34 | // err = pl.Close() 35 | // assert.NoError(t, err) 36 | 37 | // }, 38 | // }, 39 | } 40 | 41 | for i, tc := range tests { 42 | t.Run(fmt.Sprintf("%d-%s-%s", i, tc.function, tc.name), func(t *testing.T) { 43 | testPipeline := tc.constructionLogic() 44 | tc.testLogic(t, testPipeline) 45 | }) 46 | 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /internal/etl/pipeline/store.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | ) 8 | 9 | // TODO(#48): Pipeline Analysis Functionality 10 | // EtlStore ... Interface used to define all etl storage based functions 11 | type EtlStore interface { 12 | AddComponentLink(cID core.CUUID, pID core.PUUID) 13 | AddPipeline(id core.PUUID, pl Pipeline) 14 | GetAllPipelines() []Pipeline 15 | GetExistingPipelinesByPID(pPID core.PipelinePID) []core.PUUID 16 | GetPUUIDs(cID core.CUUID) ([]core.PUUID, error) 17 | GetPipelineFromPUUID(pUUID core.PUUID) (Pipeline, error) 18 | } 19 | 20 | // pipelineEntry ... value entry for some 21 | // pipeline with necessary metadata 22 | type pipelineEntry struct { 23 | id core.PUUID 24 | as ActivityState 25 | p Pipeline 26 | } 27 | 28 | type pipelineMap = map[core.PipelinePID][]pipelineEntry 29 | 30 | // etlStore ... Stores critical pipeline information 31 | // 32 | // pipeLines - Mapping used for storing all existing pipelines 33 | // compPipelines - Mapping used for storing all component-->[]PID entries 34 | type etlStore struct { 35 | pipelines pipelineMap 36 | compPipelines map[core.CUUID][]core.PUUID 37 | } 38 | 39 | // NewEtlStore ... Initializer 40 | func NewEtlStore() EtlStore { 41 | return &etlStore{ 42 | compPipelines: make(map[core.CUUID][]core.PUUID), 43 | pipelines: make(pipelineMap), 44 | } 45 | } 46 | 47 | /* 48 | Note - PUUIDs can only conflict 49 | when whenpipeLineType = Live && activityState = Active 50 | */ 51 | 52 | // addComponentLink ... Creates an entry for some new C_UUID:P_UUID mapping 53 | func (store *etlStore) AddComponentLink(cUUID core.CUUID, pUUID core.PUUID) { 54 | // EDGE CASE - C_UUID:P_UUID pair already exists 55 | if _, found := store.compPipelines[cUUID]; !found { // Create slice 56 | store.compPipelines[cUUID] = make([]core.PUUID, 0) 57 | } 58 | 59 | store.compPipelines[cUUID] = append(store.compPipelines[cUUID], pUUID) 60 | } 61 | 62 | // addPipeline ... Creates and stores a new pipeline entry 63 | func (store *etlStore) AddPipeline(pUUID core.PUUID, pl Pipeline) { 64 | entry := pipelineEntry{ 65 | id: pUUID, 66 | as: Booting, 67 | p: pl, 68 | } 69 | 70 | entrySlice, found := store.pipelines[pUUID.PID] 71 | if !found { 72 | entrySlice = make([]pipelineEntry, 0) 73 | } 74 | 75 | entrySlice = append(entrySlice, entry) 76 | 77 | store.pipelines[pUUID.PID] = entrySlice 78 | 79 | for _, comp := range pl.Components() { 80 | store.AddComponentLink(comp.UUID(), pUUID) 81 | } 82 | } 83 | 84 | // GetPUUIDs ... Returns all entried PIDs for some CID 85 | func (store *etlStore) GetPUUIDs(cID core.CUUID) ([]core.PUUID, error) { 86 | pIDs, found := store.compPipelines[cID] 87 | 88 | if !found { 89 | return []core.PUUID{}, fmt.Errorf("could not find key for %s", cID) 90 | } 91 | 92 | return pIDs, nil 93 | } 94 | 95 | // getPipelineByPID ... Returns pipeline storeovided some PID 96 | func (store *etlStore) GetPipelineFromPUUID(pUUID core.PUUID) (Pipeline, error) { 97 | if _, found := store.pipelines[pUUID.PID]; !found { 98 | return nil, fmt.Errorf(pIDNotFoundErr, pUUID.String()) 99 | } 100 | 101 | for _, plEntry := range store.pipelines[pUUID.PID] { 102 | if plEntry.id.UUID == pUUID.UUID { 103 | return plEntry.p, nil 104 | } 105 | } 106 | 107 | return nil, fmt.Errorf(uuidNotFoundErr) 108 | } 109 | 110 | // GetExistingPipelinesByPID ... Returns existing pipelines for some PID value 111 | func (store *etlStore) GetExistingPipelinesByPID(pPID core.PipelinePID) []core.PUUID { 112 | entries, exists := store.pipelines[pPID] 113 | if !exists { 114 | return []core.PUUID{} 115 | } 116 | 117 | pUUIDs := make([]core.PUUID, len(entries)) 118 | 119 | for i, entry := range entries { 120 | pUUIDs[i] = entry.id 121 | } 122 | 123 | return pUUIDs 124 | } 125 | 126 | // GetAllPipelines ... Returns all existing/current pipelines 127 | func (store *etlStore) GetAllPipelines() []Pipeline { 128 | pipeLines := make([]Pipeline, 0) 129 | 130 | for _, pLines := range store.pipelines { 131 | for _, pipeLine := range pLines { 132 | pipeLines = append(pipeLines, pipeLine.p) 133 | } 134 | } 135 | 136 | return pipeLines 137 | } 138 | -------------------------------------------------------------------------------- /internal/etl/pipeline/types.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | type ActivityState uint8 4 | 5 | const ( 6 | Booting ActivityState = iota 7 | Syncing 8 | Active 9 | Crashed 10 | ) 11 | 12 | func (as ActivityState) String() string { 13 | switch as { 14 | case Booting: 15 | return "booting" 16 | 17 | case Syncing: 18 | return "syncing" 19 | 20 | case Active: 21 | return "active" 22 | 23 | case Crashed: 24 | return "crashed" 25 | } 26 | 27 | return "unknown" 28 | } 29 | 30 | const ( 31 | // EtlStore error constants 32 | couldNotCastErr = "could not cast component initializer function to %s constructor type" 33 | pIDNotFoundErr = "could not find pipeline ID for %s" 34 | uuidNotFoundErr = "could not find matching UUID for pipeline entry" 35 | 36 | // ComponentGraph error constants 37 | cUUIDNotFoundErr = "component with ID %s does not exist within component graph" 38 | cUUIDExistsErr = "component with ID %s already exists in component graph" 39 | edgeExistsErr = "edge already exists from (%s) to (%s) in component graph" 40 | 41 | // Manager error constants 42 | unknownCompType = "unknown component type %s provided" 43 | 44 | noAggregatorErr = "aggregator component has yet to be implemented" 45 | ) 46 | -------------------------------------------------------------------------------- /internal/etl/registry/oracle/account_balance.go: -------------------------------------------------------------------------------- 1 | package oracle 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math/big" 7 | "time" 8 | 9 | "github.com/base-org/pessimism/internal/client" 10 | pess_common "github.com/base-org/pessimism/internal/common" 11 | "github.com/base-org/pessimism/internal/core" 12 | "github.com/base-org/pessimism/internal/etl/component" 13 | "github.com/base-org/pessimism/internal/logging" 14 | "github.com/base-org/pessimism/internal/state" 15 | "github.com/ethereum/go-ethereum/common" 16 | 17 | "go.uber.org/zap" 18 | ) 19 | 20 | // TODO(#21): Verify config validity during Oracle construction 21 | // AddressBalanceODef ... Address register oracle definition used to drive oracle component 22 | type AddressBalanceODef struct { 23 | pUUID core.PUUID 24 | cfg *core.ClientConfig 25 | client client.EthClientInterface 26 | currHeight *big.Int 27 | sk *core.StateKey 28 | } 29 | 30 | // NewAddressBalanceODef ... Initializer for address.balance oracle definition 31 | func NewAddressBalanceODef(cfg *core.ClientConfig, client client.EthClientInterface, 32 | h *big.Int) *AddressBalanceODef { 33 | return &AddressBalanceODef{ 34 | cfg: cfg, 35 | client: client, 36 | currHeight: h, 37 | } 38 | } 39 | 40 | // NewAddressBalanceOracle ... Initializer for address.balance oracle component 41 | func NewAddressBalanceOracle(ctx context.Context, cfg *core.ClientConfig, 42 | opts ...component.Option) (component.Component, error) { 43 | client := client.NewEthClient() 44 | 45 | od := NewAddressBalanceODef(cfg, client, nil) 46 | o, err := component.NewOracle(ctx, core.GethBlock, od, opts...) 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | od.sk = o.StateKey().Clone() 52 | return o, nil 53 | } 54 | 55 | // ConfigureRoutine ... Sets up the oracle client connection and persists puuid to definition state 56 | func (oracle *AddressBalanceODef) ConfigureRoutine(pUUID core.PUUID) error { 57 | oracle.pUUID = pUUID 58 | 59 | ctxTimeout, ctxCancel := context.WithTimeout(context.Background(), 60 | time.Second*time.Duration(core.EthClientTimeout)) 61 | defer ctxCancel() 62 | 63 | logging.WithContext(ctxTimeout).Info("Setting up Account Balance client") 64 | 65 | return oracle.client.DialContext(ctxTimeout, oracle.cfg.RPCEndpoint) 66 | } 67 | 68 | // BackTestRoutine ... 69 | // NOTE - This oracle does not support backtesting 70 | // TODO (#59) : Add account balance backtesting support 71 | func (oracle *AddressBalanceODef) BackTestRoutine(_ context.Context, _ chan core.TransitData, 72 | _ *big.Int, _ *big.Int) error { 73 | return fmt.Errorf(noBackTestSupportError) 74 | } 75 | 76 | // ReadRoutine ... Sequentially polls go-ethereum compatible execution 77 | // client for address (EOA, Contract) native balance amounts 78 | func (oracle *AddressBalanceODef) ReadRoutine(ctx context.Context, componentChan chan core.TransitData) error { 79 | stateStore, err := state.FromContext(ctx) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | ticker := time.NewTicker(oracle.cfg.PollInterval * time.Millisecond) //nolint:durationcheck // inapplicable 85 | for { 86 | select { 87 | case <-ticker.C: // Polling 88 | logging.NoContext().Debug("Getting addresess", 89 | zap.String(core.PUUIDKey, oracle.pUUID.String())) 90 | 91 | // Get addresses from shared state store for pipeline uuid 92 | 93 | addresses, err := stateStore.GetSlice(ctx, oracle.sk) 94 | if err != nil { 95 | logging.WithContext(ctx).Error(err.Error()) 96 | continue 97 | } 98 | 99 | for _, address := range addresses { 100 | // Convert to go-ethereum address type 101 | gethAddress := common.HexToAddress(address) 102 | logging.NoContext().Debug("Balance query", 103 | zap.String(core.AddrKey, gethAddress.String())) 104 | 105 | // Get balance using go-ethereum client 106 | weiBalance, err := oracle.client.BalanceAt(ctx, gethAddress, nil) 107 | if err != nil { 108 | logging.WithContext(ctx).Error(err.Error()) 109 | continue 110 | } 111 | 112 | // Convert wei to ether 113 | // NOTE - There is a possibility of precision loss here 114 | // TODO (#58) : Verify precision loss 115 | ethBalance, _ := pess_common.WeiToEther(weiBalance).Float64() 116 | 117 | logging.NoContext().Debug("Balance", 118 | zap.String(core.AddrKey, gethAddress.String()), 119 | zap.Int64("wei balance ", weiBalance.Int64())) 120 | 121 | logging.NoContext().Debug("Balance", 122 | zap.String(core.AddrKey, gethAddress.String()), 123 | zap.Float64("balance", ethBalance)) 124 | 125 | // Send parsed float64 balance value to downstream component channel 126 | componentChan <- core.NewTransitData(core.AccountBalance, ethBalance, 127 | core.WithAddress(gethAddress)) 128 | } 129 | 130 | case <-ctx.Done(): // Shutdown 131 | return nil 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /internal/etl/registry/oracle/types.go: -------------------------------------------------------------------------------- 1 | package oracle 2 | 3 | const ( 4 | noBackTestSupportError = "backtest routine is unimplemented" 5 | ) 6 | -------------------------------------------------------------------------------- /internal/etl/registry/registry.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | "github.com/base-org/pessimism/internal/etl/registry/oracle" 8 | "github.com/base-org/pessimism/internal/etl/registry/pipe" 9 | ) 10 | 11 | const ( 12 | noEntryErr = "could not find entry in registry for encoded register type %s" 13 | ) 14 | 15 | // Registry ... Interface for registry 16 | type Registry interface { 17 | GetDependencyPath(rt core.RegisterType) (core.RegisterDependencyPath, error) 18 | GetRegister(rt core.RegisterType) (*core.DataRegister, error) 19 | } 20 | 21 | // componentRegistry ... Registry implementation 22 | type componentRegistry struct { 23 | registers map[core.RegisterType]*core.DataRegister 24 | } 25 | 26 | // NewRegistry ... Instantiates a new hardcoded registry 27 | // that contains all extractable ETL data types 28 | func NewRegistry() Registry { 29 | registers := map[core.RegisterType]*core.DataRegister{ 30 | core.GethBlock: { 31 | Addressing: false, 32 | DataType: core.GethBlock, 33 | ComponentType: core.Oracle, 34 | ComponentConstructor: oracle.NewGethBlockOracle, 35 | 36 | Dependencies: noDeps(), 37 | Sk: noState(), 38 | }, 39 | core.AccountBalance: { 40 | Addressing: true, 41 | DataType: core.AccountBalance, 42 | ComponentType: core.Oracle, 43 | ComponentConstructor: oracle.NewAddressBalanceOracle, 44 | 45 | Dependencies: noDeps(), 46 | Sk: &core.StateKey{ 47 | Nesting: false, 48 | Prefix: core.AccountBalance, 49 | ID: core.AddressKey, 50 | PUUID: nil, 51 | }, 52 | }, 53 | core.EventLog: { 54 | Addressing: true, 55 | DataType: core.EventLog, 56 | ComponentType: core.Pipe, 57 | ComponentConstructor: pipe.NewEventParserPipe, 58 | 59 | Dependencies: makeDeps(core.GethBlock), 60 | Sk: &core.StateKey{ 61 | Nesting: true, 62 | Prefix: core.EventLog, 63 | ID: core.AddressKey, 64 | PUUID: nil, 65 | }, 66 | }, 67 | } 68 | 69 | return &componentRegistry{registers} 70 | } 71 | 72 | // makeDeps ... Makes dependency slice 73 | func makeDeps(types ...core.RegisterType) []core.RegisterType { 74 | deps := make([]core.RegisterType, len(types)) 75 | copy(deps, types) 76 | 77 | return deps 78 | } 79 | 80 | // noDeps ... Returns empty dependency slice 81 | func noDeps() []core.RegisterType { 82 | return []core.RegisterType{} 83 | } 84 | 85 | // noState ... Returns empty state key, indicating no state dependencies 86 | // for cross subsystem communication (i.e. ETL -> Risk Engine) 87 | func noState() *core.StateKey { 88 | return nil 89 | } 90 | 91 | // GetDependencyPath ... Returns in-order slice of ETL pipeline path 92 | func (cr *componentRegistry) GetDependencyPath(rt core.RegisterType) (core.RegisterDependencyPath, error) { 93 | destRegister, err := cr.GetRegister(rt) 94 | if err != nil { 95 | return core.RegisterDependencyPath{}, err 96 | } 97 | 98 | registers := make([]*core.DataRegister, len(destRegister.Dependencies)+1) 99 | 100 | registers[0] = destRegister 101 | 102 | for i, depType := range destRegister.Dependencies { 103 | depRegister, err := cr.GetRegister(depType) 104 | if err != nil { 105 | return core.RegisterDependencyPath{}, err 106 | } 107 | 108 | registers[i+1] = depRegister 109 | } 110 | 111 | return core.RegisterDependencyPath{Path: registers}, nil 112 | } 113 | 114 | // GetRegister ... Returns a data register provided an enum type 115 | func (cr *componentRegistry) GetRegister(rt core.RegisterType) (*core.DataRegister, error) { 116 | if _, exists := cr.registers[rt]; !exists { 117 | return nil, fmt.Errorf(noEntryErr, rt) 118 | } 119 | 120 | return cr.registers[rt], nil 121 | } 122 | -------------------------------------------------------------------------------- /internal/etl/registry/registry_test.go: -------------------------------------------------------------------------------- 1 | package registry_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/base-org/pessimism/internal/core" 8 | "github.com/base-org/pessimism/internal/etl/registry" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func Test_ComponentRegistry(t *testing.T) { 13 | var tests = []struct { 14 | name string 15 | function string 16 | description string 17 | 18 | constructionLogic func() registry.Registry 19 | testLogic func(*testing.T, registry.Registry) 20 | }{ 21 | { 22 | name: "Fetch Failure", 23 | function: "GetRegister", 24 | description: "When trying to get an invalid register, an error should be returned", 25 | 26 | constructionLogic: registry.NewRegistry, 27 | testLogic: func(t *testing.T, testRegistry registry.Registry) { 28 | 29 | invalidType := core.RegisterType(255) 30 | register, err := testRegistry.GetRegister(invalidType) 31 | 32 | assert.Error(t, err) 33 | assert.Nil(t, register) 34 | }, 35 | }, 36 | { 37 | name: "Fetch Success", 38 | function: "GetRegister", 39 | description: "When trying to get a register provided a valid register type, a register should be returned", 40 | 41 | constructionLogic: registry.NewRegistry, 42 | testLogic: func(t *testing.T, testRegistry registry.Registry) { 43 | 44 | reg, err := testRegistry.GetRegister(core.GethBlock) 45 | 46 | assert.NoError(t, err) 47 | assert.NotNil(t, reg) 48 | assert.Equal(t, reg.DataType, core.GethBlock) 49 | }, 50 | }, 51 | { 52 | name: "Fetch Dependency Path Success", 53 | function: "GetRegister", 54 | description: "When trying to get a register dependency path provided a valid register type, a path should be returned", 55 | 56 | constructionLogic: registry.NewRegistry, 57 | testLogic: func(t *testing.T, testRegistry registry.Registry) { 58 | 59 | path, err := testRegistry.GetDependencyPath(core.EventLog) 60 | 61 | assert.NoError(t, err) 62 | assert.Len(t, path.Path, 2) 63 | 64 | assert.Equal(t, path.Path[1].DataType, core.GethBlock) 65 | assert.Equal(t, path.Path[0].DataType, core.EventLog) 66 | }, 67 | }, 68 | } 69 | 70 | for i, tc := range tests { 71 | t.Run(fmt.Sprintf("%d-%s-%s", i, tc.function, tc.name), func(t *testing.T) { 72 | testRouter := tc.constructionLogic() 73 | tc.testLogic(t, testRouter) 74 | }) 75 | 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /internal/logging/logger.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "context" 5 | 6 | "go.uber.org/zap" 7 | "go.uber.org/zap/zapcore" 8 | ) 9 | 10 | type LogKey = string 11 | 12 | type loggerKeyType int 13 | 14 | const loggerKey loggerKeyType = iota 15 | 16 | // NOTE - Logger is set to Nop as default to avoid redundant testing 17 | var logger *zap.Logger = zap.NewNop() 18 | 19 | // Config ... Configuration passed through to the logging constructor 20 | type Config struct { 21 | UseCustom bool 22 | Level int 23 | DisableCaller bool 24 | DisableStacktrace bool 25 | Encoding string 26 | OutputPaths []string 27 | ErrorOutputPaths []string 28 | } 29 | 30 | // NewLogger ... initializes logging from config 31 | func NewLogger(cfg *Config, isProduction bool) { 32 | var zapCfg zap.Config 33 | 34 | if isProduction { 35 | zapCfg = zap.NewProductionConfig() 36 | } else { 37 | zapCfg = zap.NewDevelopmentConfig() 38 | } 39 | 40 | if cfg != nil && cfg.UseCustom { 41 | zapCfg.Level = zap.NewAtomicLevelAt(zapcore.Level(cfg.Level)) 42 | zapCfg.DisableCaller = cfg.DisableCaller 43 | zapCfg.DisableStacktrace = cfg.DisableStacktrace 44 | // Sampling not defined in cfg 45 | zapCfg.Encoding = cfg.Encoding 46 | // EncoderConfig not defined in cfg 47 | zapCfg.OutputPaths = cfg.OutputPaths 48 | zapCfg.ErrorOutputPaths = cfg.ErrorOutputPaths 49 | // InitialFields not defined in cfg 50 | } 51 | 52 | zapCfg.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder 53 | zapCfg.EncoderConfig.EncodeCaller = zapcore.ShortCallerEncoder 54 | zapCfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder 55 | 56 | var err error 57 | logger, err = zapCfg.Build() 58 | if err != nil { 59 | panic("could not initialize logging") 60 | } 61 | } 62 | 63 | // NewContext ... A helper for middleware to create requestId or other context fields 64 | // and return a context which logger can understand. 65 | func NewContext(ctx context.Context, fields ...zap.Field) context.Context { 66 | return context.WithValue(ctx, loggerKey, WithContext(ctx).With(fields...)) 67 | } 68 | 69 | // WithContext ... Pass in a context containing values to add to each log message 70 | func WithContext(ctx context.Context) *zap.Logger { 71 | if ctx == nil { 72 | return logger 73 | } 74 | 75 | if ctxLogger, ok := ctx.Value(loggerKey).(zap.Logger); ok { 76 | return &ctxLogger 77 | } 78 | return logger 79 | } 80 | 81 | // NoContext ... A log helper to log when there's no context. Rare case usage 82 | func NoContext() *zap.Logger { 83 | return logger 84 | } 85 | -------------------------------------------------------------------------------- /internal/mocks/alert_manager.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/base-org/pessimism/internal/alert (interfaces: Manager) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | reflect "reflect" 9 | 10 | core "github.com/base-org/pessimism/internal/core" 11 | gomock "github.com/golang/mock/gomock" 12 | ) 13 | 14 | // AlertManager is a mock of Manager interface. 15 | type AlertManager struct { 16 | ctrl *gomock.Controller 17 | recorder *AlertManagerMockRecorder 18 | } 19 | 20 | // AlertManagerMockRecorder is the mock recorder for AlertManager. 21 | type AlertManagerMockRecorder struct { 22 | mock *AlertManager 23 | } 24 | 25 | // NewAlertManager creates a new mock instance. 26 | func NewAlertManager(ctrl *gomock.Controller) *AlertManager { 27 | mock := &AlertManager{ctrl: ctrl} 28 | mock.recorder = &AlertManagerMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use. 33 | func (m *AlertManager) EXPECT() *AlertManagerMockRecorder { 34 | return m.recorder 35 | } 36 | 37 | // AddInvariantSession mocks base method. 38 | func (m *AlertManager) AddInvariantSession(arg0 core.SUUID, arg1 core.AlertDestination) error { 39 | m.ctrl.T.Helper() 40 | ret := m.ctrl.Call(m, "AddInvariantSession", arg0, arg1) 41 | ret0, _ := ret[0].(error) 42 | return ret0 43 | } 44 | 45 | // AddInvariantSession indicates an expected call of AddInvariantSession. 46 | func (mr *AlertManagerMockRecorder) AddInvariantSession(arg0, arg1 interface{}) *gomock.Call { 47 | mr.mock.ctrl.T.Helper() 48 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddInvariantSession", reflect.TypeOf((*AlertManager)(nil).AddInvariantSession), arg0, arg1) 49 | } 50 | 51 | // EventLoop mocks base method. 52 | func (m *AlertManager) EventLoop() error { 53 | m.ctrl.T.Helper() 54 | ret := m.ctrl.Call(m, "EventLoop") 55 | ret0, _ := ret[0].(error) 56 | return ret0 57 | } 58 | 59 | // EventLoop indicates an expected call of EventLoop. 60 | func (mr *AlertManagerMockRecorder) EventLoop() *gomock.Call { 61 | mr.mock.ctrl.T.Helper() 62 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EventLoop", reflect.TypeOf((*AlertManager)(nil).EventLoop)) 63 | } 64 | 65 | // Shutdown mocks base method. 66 | func (m *AlertManager) Shutdown() error { 67 | m.ctrl.T.Helper() 68 | ret := m.ctrl.Call(m, "Shutdown") 69 | ret0, _ := ret[0].(error) 70 | return ret0 71 | } 72 | 73 | // Shutdown indicates an expected call of Shutdown. 74 | func (mr *AlertManagerMockRecorder) Shutdown() *gomock.Call { 75 | mr.mock.ctrl.T.Helper() 76 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Shutdown", reflect.TypeOf((*AlertManager)(nil).Shutdown)) 77 | } 78 | 79 | // Transit mocks base method. 80 | func (m *AlertManager) Transit() chan core.Alert { 81 | m.ctrl.T.Helper() 82 | ret := m.ctrl.Call(m, "Transit") 83 | ret0, _ := ret[0].(chan core.Alert) 84 | return ret0 85 | } 86 | 87 | // Transit indicates an expected call of Transit. 88 | func (mr *AlertManagerMockRecorder) Transit() *gomock.Call { 89 | mr.mock.ctrl.T.Helper() 90 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Transit", reflect.TypeOf((*AlertManager)(nil).Transit)) 91 | } 92 | -------------------------------------------------------------------------------- /internal/mocks/api_service.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/base-org/pessimism/internal/api/service (interfaces: Service) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | reflect "reflect" 9 | 10 | models "github.com/base-org/pessimism/internal/api/models" 11 | core "github.com/base-org/pessimism/internal/core" 12 | gomock "github.com/golang/mock/gomock" 13 | ) 14 | 15 | // MockService is a mock of Service interface. 16 | type MockService struct { 17 | ctrl *gomock.Controller 18 | recorder *MockServiceMockRecorder 19 | } 20 | 21 | // MockServiceMockRecorder is the mock recorder for MockService. 22 | type MockServiceMockRecorder struct { 23 | mock *MockService 24 | } 25 | 26 | // NewMockService creates a new mock instance. 27 | func NewMockService(ctrl *gomock.Controller) *MockService { 28 | mock := &MockService{ctrl: ctrl} 29 | mock.recorder = &MockServiceMockRecorder{mock} 30 | return mock 31 | } 32 | 33 | // EXPECT returns an object that allows the caller to indicate expected use. 34 | func (m *MockService) EXPECT() *MockServiceMockRecorder { 35 | return m.recorder 36 | } 37 | 38 | // CheckETHRPCHealth mocks base method. 39 | func (m *MockService) CheckETHRPCHealth(arg0 string) bool { 40 | m.ctrl.T.Helper() 41 | ret := m.ctrl.Call(m, "CheckETHRPCHealth", arg0) 42 | ret0, _ := ret[0].(bool) 43 | return ret0 44 | } 45 | 46 | // CheckETHRPCHealth indicates an expected call of CheckETHRPCHealth. 47 | func (mr *MockServiceMockRecorder) CheckETHRPCHealth(arg0 interface{}) *gomock.Call { 48 | mr.mock.ctrl.T.Helper() 49 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckETHRPCHealth", reflect.TypeOf((*MockService)(nil).CheckETHRPCHealth), arg0) 50 | } 51 | 52 | // CheckHealth mocks base method. 53 | func (m *MockService) CheckHealth() *models.HealthCheck { 54 | m.ctrl.T.Helper() 55 | ret := m.ctrl.Call(m, "CheckHealth") 56 | ret0, _ := ret[0].(*models.HealthCheck) 57 | return ret0 58 | } 59 | 60 | // CheckHealth indicates an expected call of CheckHealth. 61 | func (mr *MockServiceMockRecorder) CheckHealth() *gomock.Call { 62 | mr.mock.ctrl.T.Helper() 63 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckHealth", reflect.TypeOf((*MockService)(nil).CheckHealth)) 64 | } 65 | 66 | // ProcessInvariantRequest mocks base method. 67 | func (m *MockService) ProcessInvariantRequest(arg0 models.InvRequestBody) (core.SUUID, error) { 68 | m.ctrl.T.Helper() 69 | ret := m.ctrl.Call(m, "ProcessInvariantRequest", arg0) 70 | ret0, _ := ret[0].(core.SUUID) 71 | ret1, _ := ret[1].(error) 72 | return ret0, ret1 73 | } 74 | 75 | // ProcessInvariantRequest indicates an expected call of ProcessInvariantRequest. 76 | func (mr *MockServiceMockRecorder) ProcessInvariantRequest(arg0 interface{}) *gomock.Call { 77 | mr.mock.ctrl.T.Helper() 78 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessInvariantRequest", reflect.TypeOf((*MockService)(nil).ProcessInvariantRequest), arg0) 79 | } 80 | 81 | // RunInvariantSession mocks base method. 82 | func (m *MockService) RunInvariantSession(arg0 models.InvRequestParams) (core.SUUID, error) { 83 | m.ctrl.T.Helper() 84 | ret := m.ctrl.Call(m, "RunInvariantSession", arg0) 85 | ret0, _ := ret[0].(core.SUUID) 86 | ret1, _ := ret[1].(error) 87 | return ret0, ret1 88 | } 89 | 90 | // RunInvariantSession indicates an expected call of RunInvariantSession. 91 | func (mr *MockServiceMockRecorder) RunInvariantSession(arg0 interface{}) *gomock.Call { 92 | mr.mock.ctrl.T.Helper() 93 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunInvariantSession", reflect.TypeOf((*MockService)(nil).RunInvariantSession), arg0) 94 | } 95 | -------------------------------------------------------------------------------- /internal/mocks/engine_manager.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/base-org/pessimism/internal/engine (interfaces: Manager) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | reflect "reflect" 9 | 10 | core "github.com/base-org/pessimism/internal/core" 11 | invariant "github.com/base-org/pessimism/internal/engine/invariant" 12 | gomock "github.com/golang/mock/gomock" 13 | ) 14 | 15 | // EngineManager is a mock of Manager interface. 16 | type EngineManager struct { 17 | ctrl *gomock.Controller 18 | recorder *EngineManagerMockRecorder 19 | } 20 | 21 | // EngineManagerMockRecorder is the mock recorder for EngineManager. 22 | type EngineManagerMockRecorder struct { 23 | mock *EngineManager 24 | } 25 | 26 | // NewEngineManager creates a new mock instance. 27 | func NewEngineManager(ctrl *gomock.Controller) *EngineManager { 28 | mock := &EngineManager{ctrl: ctrl} 29 | mock.recorder = &EngineManagerMockRecorder{mock} 30 | return mock 31 | } 32 | 33 | // EXPECT returns an object that allows the caller to indicate expected use. 34 | func (m *EngineManager) EXPECT() *EngineManagerMockRecorder { 35 | return m.recorder 36 | } 37 | 38 | // DeleteInvariantSession mocks base method. 39 | func (m *EngineManager) DeleteInvariantSession(arg0 core.SUUID) (core.SUUID, error) { 40 | m.ctrl.T.Helper() 41 | ret := m.ctrl.Call(m, "DeleteInvariantSession", arg0) 42 | ret0, _ := ret[0].(core.SUUID) 43 | ret1, _ := ret[1].(error) 44 | return ret0, ret1 45 | } 46 | 47 | // DeleteInvariantSession indicates an expected call of DeleteInvariantSession. 48 | func (mr *EngineManagerMockRecorder) DeleteInvariantSession(arg0 interface{}) *gomock.Call { 49 | mr.mock.ctrl.T.Helper() 50 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteInvariantSession", reflect.TypeOf((*EngineManager)(nil).DeleteInvariantSession), arg0) 51 | } 52 | 53 | // DeployInvariantSession mocks base method. 54 | func (m *EngineManager) DeployInvariantSession(arg0 *invariant.DeployConfig) (core.SUUID, error) { 55 | m.ctrl.T.Helper() 56 | ret := m.ctrl.Call(m, "DeployInvariantSession", arg0) 57 | ret0, _ := ret[0].(core.SUUID) 58 | ret1, _ := ret[1].(error) 59 | return ret0, ret1 60 | } 61 | 62 | // DeployInvariantSession indicates an expected call of DeployInvariantSession. 63 | func (mr *EngineManagerMockRecorder) DeployInvariantSession(arg0 interface{}) *gomock.Call { 64 | mr.mock.ctrl.T.Helper() 65 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeployInvariantSession", reflect.TypeOf((*EngineManager)(nil).DeployInvariantSession), arg0) 66 | } 67 | 68 | // EventLoop mocks base method. 69 | func (m *EngineManager) EventLoop() error { 70 | m.ctrl.T.Helper() 71 | ret := m.ctrl.Call(m, "EventLoop") 72 | ret0, _ := ret[0].(error) 73 | return ret0 74 | } 75 | 76 | // EventLoop indicates an expected call of EventLoop. 77 | func (mr *EngineManagerMockRecorder) EventLoop() *gomock.Call { 78 | mr.mock.ctrl.T.Helper() 79 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EventLoop", reflect.TypeOf((*EngineManager)(nil).EventLoop)) 80 | } 81 | 82 | // Shutdown mocks base method. 83 | func (m *EngineManager) Shutdown() error { 84 | m.ctrl.T.Helper() 85 | ret := m.ctrl.Call(m, "Shutdown") 86 | ret0, _ := ret[0].(error) 87 | return ret0 88 | } 89 | 90 | // Shutdown indicates an expected call of Shutdown. 91 | func (mr *EngineManagerMockRecorder) Shutdown() *gomock.Call { 92 | mr.mock.ctrl.T.Helper() 93 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Shutdown", reflect.TypeOf((*EngineManager)(nil).Shutdown)) 94 | } 95 | 96 | // Transit mocks base method. 97 | func (m *EngineManager) Transit() chan core.InvariantInput { 98 | m.ctrl.T.Helper() 99 | ret := m.ctrl.Call(m, "Transit") 100 | ret0, _ := ret[0].(chan core.InvariantInput) 101 | return ret0 102 | } 103 | 104 | // Transit indicates an expected call of Transit. 105 | func (mr *EngineManagerMockRecorder) Transit() *gomock.Call { 106 | mr.mock.ctrl.T.Helper() 107 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Transit", reflect.TypeOf((*EngineManager)(nil).Transit)) 108 | } 109 | -------------------------------------------------------------------------------- /internal/mocks/eth_client.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/base-org/pessimism/internal/client (interfaces: EthClientInterface) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | context "context" 9 | big "math/big" 10 | reflect "reflect" 11 | 12 | ethereum "github.com/ethereum/go-ethereum" 13 | common "github.com/ethereum/go-ethereum/common" 14 | types "github.com/ethereum/go-ethereum/core/types" 15 | gomock "github.com/golang/mock/gomock" 16 | ) 17 | 18 | // MockEthClientInterface is a mock of EthClientInterface interface. 19 | type MockEthClientInterface struct { 20 | ctrl *gomock.Controller 21 | recorder *MockEthClientInterfaceMockRecorder 22 | } 23 | 24 | // MockEthClientInterfaceMockRecorder is the mock recorder for MockEthClientInterface. 25 | type MockEthClientInterfaceMockRecorder struct { 26 | mock *MockEthClientInterface 27 | } 28 | 29 | // NewMockEthClientInterface creates a new mock instance. 30 | func NewMockEthClientInterface(ctrl *gomock.Controller) *MockEthClientInterface { 31 | mock := &MockEthClientInterface{ctrl: ctrl} 32 | mock.recorder = &MockEthClientInterfaceMockRecorder{mock} 33 | return mock 34 | } 35 | 36 | // EXPECT returns an object that allows the caller to indicate expected use. 37 | func (m *MockEthClientInterface) EXPECT() *MockEthClientInterfaceMockRecorder { 38 | return m.recorder 39 | } 40 | 41 | // BalanceAt mocks base method. 42 | func (m *MockEthClientInterface) BalanceAt(arg0 context.Context, arg1 common.Address, arg2 *big.Int) (*big.Int, error) { 43 | m.ctrl.T.Helper() 44 | ret := m.ctrl.Call(m, "BalanceAt", arg0, arg1, arg2) 45 | ret0, _ := ret[0].(*big.Int) 46 | ret1, _ := ret[1].(error) 47 | return ret0, ret1 48 | } 49 | 50 | // BalanceAt indicates an expected call of BalanceAt. 51 | func (mr *MockEthClientInterfaceMockRecorder) BalanceAt(arg0, arg1, arg2 interface{}) *gomock.Call { 52 | mr.mock.ctrl.T.Helper() 53 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BalanceAt", reflect.TypeOf((*MockEthClientInterface)(nil).BalanceAt), arg0, arg1, arg2) 54 | } 55 | 56 | // BlockByNumber mocks base method. 57 | func (m *MockEthClientInterface) BlockByNumber(arg0 context.Context, arg1 *big.Int) (*types.Block, error) { 58 | m.ctrl.T.Helper() 59 | ret := m.ctrl.Call(m, "BlockByNumber", arg0, arg1) 60 | ret0, _ := ret[0].(*types.Block) 61 | ret1, _ := ret[1].(error) 62 | return ret0, ret1 63 | } 64 | 65 | // BlockByNumber indicates an expected call of BlockByNumber. 66 | func (mr *MockEthClientInterfaceMockRecorder) BlockByNumber(arg0, arg1 interface{}) *gomock.Call { 67 | mr.mock.ctrl.T.Helper() 68 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BlockByNumber", reflect.TypeOf((*MockEthClientInterface)(nil).BlockByNumber), arg0, arg1) 69 | } 70 | 71 | // DialContext mocks base method. 72 | func (m *MockEthClientInterface) DialContext(arg0 context.Context, arg1 string) error { 73 | m.ctrl.T.Helper() 74 | ret := m.ctrl.Call(m, "DialContext", arg0, arg1) 75 | ret0, _ := ret[0].(error) 76 | return ret0 77 | } 78 | 79 | // DialContext indicates an expected call of DialContext. 80 | func (mr *MockEthClientInterfaceMockRecorder) DialContext(arg0, arg1 interface{}) *gomock.Call { 81 | mr.mock.ctrl.T.Helper() 82 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DialContext", reflect.TypeOf((*MockEthClientInterface)(nil).DialContext), arg0, arg1) 83 | } 84 | 85 | // FilterLogs mocks base method. 86 | func (m *MockEthClientInterface) FilterLogs(arg0 context.Context, arg1 ethereum.FilterQuery) ([]types.Log, error) { 87 | m.ctrl.T.Helper() 88 | ret := m.ctrl.Call(m, "FilterLogs", arg0, arg1) 89 | ret0, _ := ret[0].([]types.Log) 90 | ret1, _ := ret[1].(error) 91 | return ret0, ret1 92 | } 93 | 94 | // FilterLogs indicates an expected call of FilterLogs. 95 | func (mr *MockEthClientInterfaceMockRecorder) FilterLogs(arg0, arg1 interface{}) *gomock.Call { 96 | mr.mock.ctrl.T.Helper() 97 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FilterLogs", reflect.TypeOf((*MockEthClientInterface)(nil).FilterLogs), arg0, arg1) 98 | } 99 | 100 | // HeaderByNumber mocks base method. 101 | func (m *MockEthClientInterface) HeaderByNumber(arg0 context.Context, arg1 *big.Int) (*types.Header, error) { 102 | m.ctrl.T.Helper() 103 | ret := m.ctrl.Call(m, "HeaderByNumber", arg0, arg1) 104 | ret0, _ := ret[0].(*types.Header) 105 | ret1, _ := ret[1].(error) 106 | return ret0, ret1 107 | } 108 | 109 | // HeaderByNumber indicates an expected call of HeaderByNumber. 110 | func (mr *MockEthClientInterfaceMockRecorder) HeaderByNumber(arg0, arg1 interface{}) *gomock.Call { 111 | mr.mock.ctrl.T.Helper() 112 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeaderByNumber", reflect.TypeOf((*MockEthClientInterface)(nil).HeaderByNumber), arg0, arg1) 113 | } 114 | -------------------------------------------------------------------------------- /internal/mocks/etl_manager.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/base-org/pessimism/internal/etl/pipeline (interfaces: Manager) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | reflect "reflect" 9 | 10 | core "github.com/base-org/pessimism/internal/core" 11 | gomock "github.com/golang/mock/gomock" 12 | ) 13 | 14 | // EtlManager is a mock of Manager interface. 15 | type EtlManager struct { 16 | ctrl *gomock.Controller 17 | recorder *EtlManagerMockRecorder 18 | } 19 | 20 | // EtlManagerMockRecorder is the mock recorder for EtlManager. 21 | type EtlManagerMockRecorder struct { 22 | mock *EtlManager 23 | } 24 | 25 | // NewEtlManager creates a new mock instance. 26 | func NewEtlManager(ctrl *gomock.Controller) *EtlManager { 27 | mock := &EtlManager{ctrl: ctrl} 28 | mock.recorder = &EtlManagerMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use. 33 | func (m *EtlManager) EXPECT() *EtlManagerMockRecorder { 34 | return m.recorder 35 | } 36 | 37 | // CreateDataPipeline mocks base method. 38 | func (m *EtlManager) CreateDataPipeline(arg0 *core.PipelineConfig) (core.PUUID, error) { 39 | m.ctrl.T.Helper() 40 | ret := m.ctrl.Call(m, "CreateDataPipeline", arg0) 41 | ret0, _ := ret[0].(core.PUUID) 42 | ret1, _ := ret[1].(error) 43 | return ret0, ret1 44 | } 45 | 46 | // CreateDataPipeline indicates an expected call of CreateDataPipeline. 47 | func (mr *EtlManagerMockRecorder) CreateDataPipeline(arg0 interface{}) *gomock.Call { 48 | mr.mock.ctrl.T.Helper() 49 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateDataPipeline", reflect.TypeOf((*EtlManager)(nil).CreateDataPipeline), arg0) 50 | } 51 | 52 | // EventLoop mocks base method. 53 | func (m *EtlManager) EventLoop() error { 54 | m.ctrl.T.Helper() 55 | ret := m.ctrl.Call(m, "EventLoop") 56 | ret0, _ := ret[0].(error) 57 | return ret0 58 | } 59 | 60 | // EventLoop indicates an expected call of EventLoop. 61 | func (mr *EtlManagerMockRecorder) EventLoop() *gomock.Call { 62 | mr.mock.ctrl.T.Helper() 63 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EventLoop", reflect.TypeOf((*EtlManager)(nil).EventLoop)) 64 | } 65 | 66 | // GetRegister mocks base method. 67 | func (m *EtlManager) GetRegister(arg0 core.RegisterType) (*core.DataRegister, error) { 68 | m.ctrl.T.Helper() 69 | ret := m.ctrl.Call(m, "GetRegister", arg0) 70 | ret0, _ := ret[0].(*core.DataRegister) 71 | ret1, _ := ret[1].(error) 72 | return ret0, ret1 73 | } 74 | 75 | // GetRegister indicates an expected call of GetRegister. 76 | func (mr *EtlManagerMockRecorder) GetRegister(arg0 interface{}) *gomock.Call { 77 | mr.mock.ctrl.T.Helper() 78 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRegister", reflect.TypeOf((*EtlManager)(nil).GetRegister), arg0) 79 | } 80 | 81 | // RunPipeline mocks base method. 82 | func (m *EtlManager) RunPipeline(arg0 core.PUUID) error { 83 | m.ctrl.T.Helper() 84 | ret := m.ctrl.Call(m, "RunPipeline", arg0) 85 | ret0, _ := ret[0].(error) 86 | return ret0 87 | } 88 | 89 | // RunPipeline indicates an expected call of RunPipeline. 90 | func (mr *EtlManagerMockRecorder) RunPipeline(arg0 interface{}) *gomock.Call { 91 | mr.mock.ctrl.T.Helper() 92 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunPipeline", reflect.TypeOf((*EtlManager)(nil).RunPipeline), arg0) 93 | } 94 | 95 | // Shutdown mocks base method. 96 | func (m *EtlManager) Shutdown() error { 97 | m.ctrl.T.Helper() 98 | ret := m.ctrl.Call(m, "Shutdown") 99 | ret0, _ := ret[0].(error) 100 | return ret0 101 | } 102 | 103 | // Shutdown indicates an expected call of Shutdown. 104 | func (mr *EtlManagerMockRecorder) Shutdown() *gomock.Call { 105 | mr.mock.ctrl.T.Helper() 106 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Shutdown", reflect.TypeOf((*EtlManager)(nil).Shutdown)) 107 | } 108 | -------------------------------------------------------------------------------- /internal/mocks/oracle.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "context" 5 | "math/big" 6 | 7 | "github.com/base-org/pessimism/internal/core" 8 | "github.com/base-org/pessimism/internal/etl/component" 9 | ) 10 | 11 | type mockOracleDefinition struct { 12 | } 13 | 14 | func (md *mockOracleDefinition) ConfigureRoutine(core.PUUID) error { 15 | return nil 16 | } 17 | 18 | func (md *mockOracleDefinition) BackTestRoutine(_ context.Context, _ chan core.TransitData, 19 | _ *big.Int, _ *big.Int) error { 20 | return nil 21 | } 22 | 23 | func (md *mockOracleDefinition) ReadRoutine(_ context.Context, _ chan core.TransitData) error { 24 | return nil 25 | } 26 | 27 | // NewMockOracle ... Takes in a register type that specifies the mocked output type 28 | // Useful for testing inter-component connectivity and higher level component management abstractions 29 | func NewMockOracle(ctx context.Context, ot core.RegisterType) (component.Component, error) { 30 | od := &mockOracleDefinition{} 31 | 32 | return component.NewOracle(ctx, ot, od) 33 | } 34 | -------------------------------------------------------------------------------- /internal/mocks/pipe.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/base-org/pessimism/internal/core" 7 | "github.com/base-org/pessimism/internal/etl/component" 8 | ) 9 | 10 | // mockPipeDefinition ... Mocked pipe definition struct 11 | type mockPipeDefinition struct { 12 | } 13 | 14 | // ConfigureRoutine ... Mocked configure routine function that returns nil 15 | func (md *mockPipeDefinition) ConfigureRoutine(core.PUUID) error { 16 | return nil 17 | } 18 | 19 | // Transform ... Mocked transform function that returns an empty slice 20 | func (md *mockPipeDefinition) Transform(_ context.Context, td core.TransitData) ([]core.TransitData, error) { 21 | return []core.TransitData{td}, nil 22 | } 23 | 24 | // NewMockPipe ... Takes in a register type that specifies the mocked output type 25 | // Useful for testing inter-component connectivity and higher level component management abstractions 26 | func NewMockPipe(ctx context.Context, it core.RegisterType, ot core.RegisterType) (component.Component, error) { 27 | od := &mockPipeDefinition{} 28 | 29 | return component.NewPipe(ctx, od, it, ot) 30 | } 31 | -------------------------------------------------------------------------------- /internal/mocks/slack_client.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/base-org/pessimism/internal/client (interfaces: SlackClient) 3 | 4 | // Package mocks is a generated GoMock package. 5 | package mocks 6 | 7 | import ( 8 | context "context" 9 | reflect "reflect" 10 | 11 | client "github.com/base-org/pessimism/internal/client" 12 | gomock "github.com/golang/mock/gomock" 13 | ) 14 | 15 | // MockSlackClient is a mock of SlackClient interface. 16 | type MockSlackClient struct { 17 | ctrl *gomock.Controller 18 | recorder *MockSlackClientMockRecorder 19 | } 20 | 21 | // MockSlackClientMockRecorder is the mock recorder for MockSlackClient. 22 | type MockSlackClientMockRecorder struct { 23 | mock *MockSlackClient 24 | } 25 | 26 | // NewMockSlackClient creates a new mock instance. 27 | func NewMockSlackClient(ctrl *gomock.Controller) *MockSlackClient { 28 | mock := &MockSlackClient{ctrl: ctrl} 29 | mock.recorder = &MockSlackClientMockRecorder{mock} 30 | return mock 31 | } 32 | 33 | // EXPECT returns an object that allows the caller to indicate expected use. 34 | func (m *MockSlackClient) EXPECT() *MockSlackClientMockRecorder { 35 | return m.recorder 36 | } 37 | 38 | // PostData mocks base method. 39 | func (m *MockSlackClient) PostData(arg0 context.Context, arg1 string) (*client.SlackAPIResponse, error) { 40 | m.ctrl.T.Helper() 41 | ret := m.ctrl.Call(m, "PostData", arg0, arg1) 42 | ret0, _ := ret[0].(*client.SlackAPIResponse) 43 | ret1, _ := ret[1].(error) 44 | return ret0, ret1 45 | } 46 | 47 | // PostData indicates an expected call of PostData. 48 | func (mr *MockSlackClientMockRecorder) PostData(arg0, arg1 interface{}) *gomock.Call { 49 | mr.mock.ctrl.T.Helper() 50 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PostData", reflect.TypeOf((*MockSlackClient)(nil).PostData), arg0, arg1) 51 | } 52 | -------------------------------------------------------------------------------- /internal/state/memory.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/base-org/pessimism/internal/core" 9 | ) 10 | 11 | /* 12 | NOTE - This is a temporary implementation of the state store. 13 | */ 14 | 15 | // stateStore ... In memory state store 16 | type stateStore struct { 17 | // NOTE - This is a temporary implementation of the state store. 18 | // Using a map of string to string slices to represent the state 19 | // store is not a scalable solution and will be rather expensive 20 | // in both memory and time complexity. This will be replaced with 21 | // a more optimal in-memory solution in the future. 22 | sliceStore map[string][]string 23 | 24 | sync.RWMutex 25 | } 26 | 27 | // NewMemState ... Initializer 28 | func NewMemState() Store { 29 | return &stateStore{ 30 | sliceStore: make(map[string][]string, 0), 31 | RWMutex: sync.RWMutex{}, 32 | } 33 | } 34 | 35 | // Get ... Fetches a string value slice from the store 36 | func (ss *stateStore) GetSlice(_ context.Context, key *core.StateKey) ([]string, error) { 37 | ss.RLock() 38 | defer ss.RUnlock() 39 | 40 | val, exists := ss.sliceStore[key.String()] 41 | if !exists { 42 | return []string{}, fmt.Errorf("could not find state store value for key %s", key) 43 | } 44 | 45 | return val, nil 46 | } 47 | 48 | // SetSlice ... Appends a value to the store slice 49 | func (ss *stateStore) SetSlice(_ context.Context, key *core.StateKey, value string) (string, error) { 50 | ss.Lock() 51 | defer ss.Unlock() 52 | 53 | ss.sliceStore[key.String()] = append(ss.sliceStore[key.String()], value) 54 | 55 | return value, nil 56 | } 57 | 58 | // Remove ... Removes a key entry from the store 59 | func (ss *stateStore) Remove(_ context.Context, key *core.StateKey) error { 60 | ss.Lock() 61 | defer ss.Unlock() 62 | 63 | delete(ss.sliceStore, key.String()) 64 | return nil 65 | } 66 | 67 | // GetNestedSubset ... Fetches a subset of a nested slice provided a nested 68 | // key/value pair (ie. filters the state object into a subset object that 69 | // contains only the values that match the nested key/value pair) 70 | func (ss *stateStore) GetNestedSubset(_ context.Context, 71 | key *core.StateKey) (map[string][]string, error) { 72 | ss.RLock() 73 | defer ss.RUnlock() 74 | 75 | values, exists := ss.sliceStore[key.String()] 76 | if !exists { 77 | return map[string][]string{}, fmt.Errorf("could not find state store value for key %s", key) 78 | } 79 | 80 | var nestedMap = make(map[string][]string, 0) 81 | for _, val := range values { 82 | if _, exists := ss.sliceStore[val]; !exists { 83 | return map[string][]string{}, fmt.Errorf("could not find state store value for key %s", key) 84 | } 85 | 86 | nestedValues := ss.sliceStore[val] 87 | nestedMap[val] = nestedValues 88 | } 89 | 90 | return nestedMap, nil 91 | } 92 | -------------------------------------------------------------------------------- /internal/state/memory_test.go: -------------------------------------------------------------------------------- 1 | package state_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/base-org/pessimism/internal/core" 9 | "github.com/base-org/pessimism/internal/state" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func Test_MemState(t *testing.T) { 14 | 15 | testKey := &core.StateKey{false, 1, "test", nil} 16 | testValue := "0xabc" 17 | testValue2 := "0xdef" 18 | 19 | innerTestKey := &core.StateKey{false, 1, "best", nil} 20 | var tests = []struct { 21 | name string 22 | description string 23 | function string 24 | 25 | construction func() state.Store 26 | testLogic func(t *testing.T, ss state.Store) 27 | }{ 28 | { 29 | name: "Test_Set_Success", 30 | description: "Test set", 31 | function: "Set", 32 | construction: state.NewMemState, 33 | testLogic: func(t *testing.T, ss state.Store) { 34 | _, err := ss.SetSlice(context.Background(), testKey, testValue) 35 | assert.NoError(t, err) 36 | 37 | val, err := ss.GetSlice(context.Background(), testKey) 38 | 39 | assert.NoError(t, err) 40 | assert.Equal(t, []string{testValue}, val) 41 | }, 42 | }, 43 | { 44 | name: "Test_Get_Fail", 45 | description: "Test failed get when key doens't exist", 46 | function: "Get", 47 | construction: state.NewMemState, 48 | testLogic: func(t *testing.T, ss state.Store) { 49 | _, err := ss.GetSlice(context.Background(), testKey) 50 | assert.Error(t, err) 51 | }, 52 | }, 53 | { 54 | name: "Test_Remove", 55 | description: "Test remove when value is prepopulated", 56 | function: "Remove", 57 | construction: func() state.Store { 58 | ss := state.NewMemState() 59 | _, err := ss.SetSlice(context.Background(), testKey, testValue) 60 | if err != nil { 61 | panic(err) 62 | } 63 | 64 | return ss 65 | }, 66 | testLogic: func(t *testing.T, ss state.Store) { 67 | err := ss.Remove(context.Background(), testKey) 68 | assert.NoError(t, err, "should not error") 69 | }, 70 | }, 71 | { 72 | name: "Test_GetNestedSubset_Success", 73 | description: "Test get nested subset", 74 | function: "GetNestedSubset", 75 | construction: func() state.Store { 76 | ss := state.NewMemState() 77 | _, err := ss.SetSlice(context.Background(), testKey, innerTestKey.String()) 78 | if err != nil { 79 | panic(err) 80 | } 81 | 82 | _, err = ss.SetSlice(context.Background(), innerTestKey, testValue2) 83 | if err != nil { 84 | panic(err) 85 | } 86 | return ss 87 | }, 88 | testLogic: func(t *testing.T, ss state.Store) { 89 | subGraph, err := ss.GetNestedSubset(context.Background(), testKey) 90 | assert.NoError(t, err, "should not error") 91 | 92 | assert.Contains(t, subGraph, innerTestKey.String(), "should contain inner key") 93 | }, 94 | }, 95 | } 96 | 97 | // TODO - Consider making generic test helpers for this 98 | for i, test := range tests { 99 | t.Run(fmt.Sprintf("%d-%s-%s", i, test.name, test.function), func(t *testing.T) { 100 | testState := test.construction() 101 | test.testLogic(t, testState) 102 | }) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /internal/state/state.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/base-org/pessimism/internal/core" 8 | ) 9 | 10 | type CtxKey uint8 11 | 12 | const ( 13 | Default CtxKey = iota 14 | ) 15 | 16 | // Store ... Interface for a state store 17 | // TODO() - Add optional redis store implementation 18 | type Store interface { 19 | GetSlice(context.Context, *core.StateKey) ([]string, error) 20 | GetNestedSubset(ctx context.Context, key *core.StateKey) (map[string][]string, error) 21 | 22 | SetSlice(context.Context, *core.StateKey, string) (string, error) 23 | Remove(context.Context, *core.StateKey) error 24 | } 25 | 26 | // FromContext ... Fetches a state store from context 27 | func FromContext(ctx context.Context) (Store, error) { 28 | if store, ok := ctx.Value(Default).(Store); ok { 29 | return store, nil 30 | } 31 | 32 | return nil, fmt.Errorf("could not load state object from context") 33 | } 34 | -------------------------------------------------------------------------------- /internal/subsystem/manager.go: -------------------------------------------------------------------------------- 1 | package subsystem 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/base-org/pessimism/internal/alert" 8 | "github.com/base-org/pessimism/internal/core" 9 | "github.com/base-org/pessimism/internal/engine" 10 | "github.com/base-org/pessimism/internal/engine/invariant" 11 | "github.com/base-org/pessimism/internal/etl/pipeline" 12 | "github.com/base-org/pessimism/internal/logging" 13 | "go.uber.org/zap" 14 | ) 15 | 16 | // Manager ... Subsystem manager interface 17 | type Manager interface { 18 | StartEventRoutines(ctx context.Context) 19 | StartInvSession(cfg *core.PipelineConfig, invCfg *core.SessionConfig) (core.SUUID, error) 20 | Shutdown() error 21 | } 22 | 23 | // manager ... Subsystem manager struct 24 | type manager struct { 25 | ctx context.Context 26 | 27 | etl pipeline.Manager 28 | eng engine.Manager 29 | alrt alert.Manager 30 | 31 | *sync.WaitGroup 32 | } 33 | 34 | // NewManager ... Initializer for the subsystem manager 35 | func NewManager(ctx context.Context, etl pipeline.Manager, eng engine.Manager, 36 | alrt alert.Manager, 37 | ) Manager { 38 | return &manager{ 39 | ctx: ctx, 40 | etl: etl, 41 | eng: eng, 42 | alrt: alrt, 43 | WaitGroup: &sync.WaitGroup{}, 44 | } 45 | } 46 | 47 | // Shutdown ... Shuts down all subsystems in primary data flow order 48 | // Ie. ETL -> Engine -> Alert 49 | func (m *manager) Shutdown() error { 50 | if err := m.etl.Shutdown(); err != nil { 51 | return err 52 | } 53 | 54 | if err := m.eng.Shutdown(); err != nil { 55 | return err 56 | } 57 | 58 | return m.alrt.Shutdown() 59 | } 60 | 61 | // StartEventRoutines ... Starts the event loop routines for the subsystems 62 | func (m *manager) StartEventRoutines(ctx context.Context) { 63 | logger := logging.WithContext(ctx) 64 | 65 | m.Add(1) 66 | go func() { // EngineManager driver thread 67 | defer m.Done() 68 | 69 | if err := m.eng.EventLoop(); err != nil { 70 | logger.Error("engine manager event loop error", zap.Error(err)) 71 | } 72 | }() 73 | 74 | m.Add(1) 75 | go func() { // AlertManager driver thread 76 | defer m.Done() 77 | 78 | if err := m.alrt.EventLoop(); err != nil { 79 | logger.Error("alert manager event loop error", zap.Error(err)) 80 | } 81 | }() 82 | 83 | m.Add(1) 84 | go func() { // ETL driver thread 85 | defer m.Done() 86 | 87 | if err := m.alrt.EventLoop(); err != nil { 88 | logger.Error("ETL manager event loop error", zap.Error(err)) 89 | } 90 | }() 91 | } 92 | 93 | // StartInvSession ... Deploys an invariant session 94 | func (m *manager) StartInvSession(cfg *core.PipelineConfig, invCfg *core.SessionConfig) (core.SUUID, error) { 95 | logger := logging.WithContext(m.ctx) 96 | 97 | pUUID, err := m.etl.CreateDataPipeline(cfg) 98 | if err != nil { 99 | return core.NilSUUID(), err 100 | } 101 | 102 | reg, err := m.etl.GetRegister(cfg.DataType) 103 | if err != nil { 104 | return core.NilSUUID(), err 105 | } 106 | 107 | logger.Info("Created etl pipeline", 108 | zap.String(core.PUUIDKey, pUUID.String())) 109 | 110 | deployCfg := &invariant.DeployConfig{ 111 | PUUID: pUUID, 112 | InvType: invCfg.Type, 113 | InvParams: invCfg.Params, 114 | Network: cfg.Network, 115 | Register: reg, 116 | } 117 | 118 | sUUID, err := m.eng.DeployInvariantSession(deployCfg) 119 | if err != nil { 120 | return core.NilSUUID(), err 121 | } 122 | logger.Info("Deployed invariant session", zap.String(core.SUUIDKey, sUUID.String())) 123 | 124 | err = m.alrt.AddInvariantSession(sUUID, invCfg.AlertDest) 125 | if err != nil { 126 | return core.NilSUUID(), err 127 | } 128 | 129 | if err = m.etl.RunPipeline(pUUID); err != nil { 130 | return core.NilSUUID(), err 131 | } 132 | 133 | return sUUID, nil 134 | } 135 | -------------------------------------------------------------------------------- /internal/subsystem/manager_test.go: -------------------------------------------------------------------------------- 1 | package subsystem_test 2 | 3 | // TODO(#76) : No Subsystem Manager Tests 4 | -------------------------------------------------------------------------------- /pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Fixes Issue 4 | 5 | 6 | Fixes # 7 | ## Changes proposed 8 | 9 | 10 | 11 | 12 | 13 | ### Screenshots (Optional) 14 | 15 | ## Note to reviewers 16 | 17 | 18 | --------------------------------------------------------------------------------