├── .github └── workflows │ ├── build.yaml │ └── deploy-testnet.yaml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── Makefile ├── README.md ├── cairo_programs ├── Account.json ├── account_without_validation.json └── contracts.json ├── narwhal-abci ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── README.md ├── assets │ └── architecture.png ├── demo │ ├── .gitignore │ ├── README.md │ ├── benchmark │ │ ├── __init__ │ │ ├── __pycache__ │ │ │ ├── logs.cpython-310.pyc │ │ │ └── utils.cpython-310.pyc │ │ ├── aggregate.py │ │ ├── commands.py │ │ ├── config.py │ │ ├── instance.py │ │ ├── local.py │ │ ├── logs.py │ │ ├── plot.py │ │ ├── remote.py │ │ ├── settings.py │ │ └── utils.py │ ├── cleanup-logs.sh │ ├── fabfile.py │ ├── node_params.json │ ├── poetry.lock │ └── pyproject.toml ├── narwhal-abci │ ├── Cargo.toml │ └── src │ │ ├── abci_server.rs │ │ ├── engine.rs │ │ └── lib.rs ├── node │ ├── Cargo.toml │ └── src │ │ └── main.rs └── starknet-abci │ ├── Cargo.toml │ ├── programs │ └── fibonacci.json │ └── src │ ├── app.rs │ ├── bin │ ├── client.rs │ └── starknet-app.rs │ ├── lib.rs │ ├── transaction.rs │ └── types.rs └── sequencer ├── .gitignore ├── Cargo.toml ├── LICENSE ├── bench ├── cmd │ └── load_test │ │ └── main.go ├── go.mod ├── go.sum └── pkg │ └── abci │ └── client.go ├── cairo_programs └── contracts.json ├── docker-compose.yml ├── playbooks └── deploy.yaml ├── programs ├── factorial.cairo ├── factorial.json ├── fibonacci.cairo └── fibonacci.json ├── prometheus.yml ├── src ├── abci │ ├── application.rs │ └── main.rs ├── bench │ └── main.rs ├── cli │ ├── main.rs │ └── tendermint.rs └── lib │ └── mod.rs └── tests └── client.rs /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: Lambdaworks build checks 2 | on: 3 | push: 4 | branches: [ main ] 5 | pull_request: 6 | branches: [ '*' ] 7 | 8 | concurrency: 9 | group: ${{ github.head_ref || github.run_id }} 10 | cancel-in-progress: true 11 | 12 | jobs: 13 | check: 14 | name: Check 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout sources 18 | uses: actions/checkout@v3 19 | 20 | - name: Install stable toolchain 21 | uses: actions-rs/toolchain@v1 22 | with: 23 | profile: minimal 24 | toolchain: stable 25 | override: true 26 | 27 | - name: Run cargo check 28 | uses: actions-rs/cargo@v1 29 | with: 30 | command: check 31 | 32 | fmt: 33 | name: Lints 34 | runs-on: ubuntu-latest 35 | steps: 36 | - name: Checkout sources 37 | uses: actions/checkout@v3 38 | 39 | - name: Install stable toolchain 40 | uses: actions-rs/toolchain@v1 41 | with: 42 | profile: minimal 43 | toolchain: stable 44 | override: true 45 | components: rustfmt, clippy 46 | 47 | - name: Run cargo fmt 48 | uses: actions-rs/cargo@v1 49 | with: 50 | command: fmt 51 | args: --all -- --check 52 | 53 | clippy: 54 | runs-on: ubuntu-latest 55 | steps: 56 | - uses: actions/checkout@v3 57 | - uses: actions-rs/toolchain@v1 58 | with: 59 | toolchain: stable 60 | components: clippy 61 | - run: make clippy 62 | -------------------------------------------------------------------------------- /.github/workflows/deploy-testnet.yaml: -------------------------------------------------------------------------------- 1 | name: Reset and deploy tendermint testnet 2 | on: workflow_dispatch 3 | jobs: 4 | 5 | deploy: 6 | name: Reset and deploy tendermint testnet 7 | environment: 8 | name: development 9 | runs-on: ubuntu-latest 10 | steps: 11 | 12 | - name: Checkout 13 | uses: actions/checkout@v3.3.0 14 | 15 | - name: Run deploy on all nodes 16 | uses: dawidd6/action-ansible-playbook@v2 17 | with: 18 | # Required, playbook filepath 19 | playbook: deploy.yaml 20 | # Optional, directory where playbooks live 21 | directory: ./ansible/playbooks/ 22 | # Optional, SSH private key 23 | key: ${{ secrets.SSH_PRIVATE_KEY }} 24 | # Optional, literal inventory file contents 25 | inventory: | 26 | tendermint-nodes: 27 | hosts: 28 | starknet-0: 29 | ansible_host: "5.9.57.45" 30 | ansible_user: root 31 | ansible_python_interpreter: /usr/bin/python3 32 | ansible_ssh_common_args: '-o StrictHostKeyChecking=no' 33 | starknet-1: 34 | ansible_host: "5.9.57.44" 35 | ansible_user: root 36 | ansible_python_interpreter: /usr/bin/python3 37 | ansible_ssh_common_args: '-o StrictHostKeyChecking=no' 38 | starknet-2: 39 | ansible_host: "5.9.57.89" 40 | ansible_user: root 41 | ansible_python_interpreter: /usr/bin/python3 42 | ansible_ssh_common_args: '-o StrictHostKeyChecking=no' 43 | options: | 44 | --verbose 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /bin/ 2 | /rollkit-node/rollkit-node 3 | /rollkit-node-bitcoin/rollkit-node-bitcoin 4 | /target 5 | .DS_Store 6 | 7 | abci.height 8 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["sequencer"] 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: reset abci cli consensus_config consensus_install rollkit_celestia bitcoin celestia 2 | 3 | OS := $(shell uname | tr '[:upper:]' '[:lower:]') 4 | 5 | ifeq ($(shell uname -p), arm) 6 | ARCH=arm64 7 | else 8 | ARCH=amd64 9 | endif 10 | 11 | # By default consensus protocol is CometBFT. Can be overriden with Tendermint 12 | CONSENSUS=cometbft 13 | 14 | ifeq ($(CONSENSUS), tendermint) 15 | CONSENSUS_VERSION=0.34.22 16 | CONSENSUS_HOME=~/.tendermint 17 | else 18 | CONSENSUS=cometbft 19 | CONSENSUS_VERSION=0.37.0 20 | CONSENSUS_HOME=~/.cometbft 21 | endif 22 | 23 | test_make: 24 | echo "CONSENSUS = $(CONSENSUS) version=$(CONSENSUS_VERSION) home=$(CONSENSUS_HOME)" 25 | 26 | # Build the client program and put it in bin/aleo 27 | cli: 28 | mkdir -p bin && cargo build --release && cp target/release/cli bin/cli 29 | 30 | # Installs tendermint for current OS and puts it in bin/ 31 | bin/tendermint: 32 | make consensus_install CONSENSUS=tendermint 33 | 34 | # Installs cometbft for current OS and puts it in bin/ 35 | bin/cometbft: 36 | make consensus_install CONSENSUS=cometbft 37 | 38 | # Internal phony target to install tendermint/cometbft for an arbitrary OS 39 | consensus_install: 40 | mkdir -p $(CONSENSUS)-install bin && cd $(CONSENSUS)-install &&\ 41 | wget https://github.com/$(CONSENSUS)/$(CONSENSUS)/releases/download/v$(CONSENSUS_VERSION)/$(CONSENSUS)_$(CONSENSUS_VERSION)_$(OS)_$(ARCH).tar.gz &&\ 42 | tar -xzvf $(CONSENSUS)_$(CONSENSUS_VERSION)_$(OS)_$(ARCH).tar.gz 43 | mv $(CONSENSUS)-install/$(CONSENSUS) bin/ && rm -rf $(CONSENSUS)-install 44 | bin/$(CONSENSUS) init 45 | 46 | # Run a consensus node, installing it if necessary 47 | node: bin/$(CONSENSUS) consensus_config 48 | bin/$(CONSENSUS) node --consensus.create_empty_blocks_interval="30s" 49 | 50 | # Override a tendermint/cometbft node's default configuration. NOTE: we should do something more declarative if we need to update more settings. 51 | consensus_config: 52 | sed -i.bak 's/max_body_bytes = 1000000/max_body_bytes = 12000000/g' $(CONSENSUS_HOME)/config/config.toml 53 | sed -i.bak 's/max_tx_bytes = 1048576/max_tx_bytes = 10485770/g' $(CONSENSUS_HOME)/config/config.toml 54 | sed -i.bak 's#laddr = "tcp://127.0.0.1:26657"#laddr = "tcp://0.0.0.0:26657"#g' $(CONSENSUS_HOME)/config/config.toml 55 | sed -i.bak 's/prometheus = false/prometheus = true/g' $(CONSENSUS_HOME)/config/config.toml 56 | 57 | # remove the blockchain data 58 | reset: bin/$(CONSENSUS) 59 | bin/$(CONSENSUS) unsafe_reset_all 60 | rm -rf abci.height 61 | 62 | # run the Cairo abci application 63 | abci: 64 | cargo run --release --bin abci 65 | 66 | # run tests on release mode (default VM backend) to ensure there is no extra printing to stdout 67 | test: 68 | RUST_BACKTRACE=full cargo test --release -- --nocapture --test-threads=4 69 | 70 | # Initialize the consensus configuration for a localnet of the given amount of validators 71 | localnet: VALIDATORS:=4 72 | localnet: ADDRESS:=127.0.0.1 73 | localnet: HOMEDIR:=localnet 74 | localnet: bin/consensus cli 75 | rm -rf $(HOMEDIR)/ 76 | bin/$(CONSENSUS) testnet --v $(VALIDATORS) --o ./$(HOMEDIR) --starting-ip-address $(ADDRESS) 77 | for n in $$(seq 0 $$(($(VALIDATORS)-1))) ; do \ 78 | make localnet_config CONSENSUS_HOME=$(HOMEDIR)/node$$n NODE=$$n VALIDATORS=$(VALIDATORS); \ 79 | mkdir $(HOMEDIR)/node$$n/abci ; \ 80 | done 81 | .PHONY: localnet 82 | # cargo run --bin genesis --release -- $(HOMEDIR)/* 83 | 84 | # run both the abci application and the consensus node 85 | # assumes config for each node has been done previously 86 | localnet_start: NODE:=0 87 | localnet_start: HOMEDIR:=localnet 88 | localnet_start: 89 | bin/$(CONSENSUS) node --home ./$(HOMEDIR)/node$(NODE) & 90 | cd ./$(HOMEDIR)/node$(NODE)/abci; cargo run --release --bin abci -- --port 26$(NODE)58 91 | .PHONY: localnet_start 92 | 93 | 94 | localnet_config: 95 | sed -i.bak 's/max_body_bytes = 1000000/max_body_bytes = 12000000/g' $(CONSENSUS_HOME)/config/config.toml 96 | sed -i.bak 's/max_tx_bytes = 1048576/max_tx_bytes = 10485770/g' $(CONSENSUS_HOME)/config/config.toml 97 | sed -i.bak 's/prometheus = false/prometheus = true/g' $(CONSENSUS_HOME)/config/config.toml 98 | for n in $$(seq 0 $$(($(VALIDATORS)-1))) ; do \ 99 | eval "sed -i.bak 's/127.0.0.$$(($${n}+1)):26656/127.0.0.1:26$${n}56/g' $(CONSENSUS_HOME)/config/config.toml" ;\ 100 | done 101 | sed -i.bak 's#laddr = "tcp://0.0.0.0:26656"#laddr = "tcp://0.0.0.0:26$(NODE)56"#g' $(CONSENSUS_HOME)/config/config.toml 102 | sed -i.bak 's#laddr = "tcp://127.0.0.1:26657"#laddr = "tcp://0.0.0.0:26$(NODE)57"#g' $(CONSENSUS_HOME)/config/config.toml 103 | sed -i.bak 's#proxy_app = "tcp://127.0.0.1:26658"#proxy_app = "tcp://127.0.0.1:26$(NODE)58"#g' $(CONSENSUS_HOME)/config/config.toml 104 | .PHONY: localnet_config 105 | 106 | 107 | localnet_reset: 108 | bin/$(CONSENSUS) unsafe_reset_all 109 | rm -rf localnet/node*/abci/abci.height; 110 | .PHONY: localnet_reset 111 | 112 | clippy: 113 | cargo clippy --all-targets --all-features -- -D warnings 114 | .PHONY: clippy 115 | 116 | celestia: 117 | (cd local-da; docker compose -f ./docker/test-docker-compose.yml up) 118 | 119 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | _Note: `main` branch contains the development version of the sequencer, for rollups integration please check branch [`rollups`](https://github.com/lambdaclass/starknet_tendermint_sequencer/tree/rollups)_ 2 | # Tendermint-based Starknet sequencer 3 | 4 | Sequencer for Starknet based in CometBFT (formerly Tendermint Core) and [starknet-in-rust](https://github.com/lambdaclass/starknet_in_rust). 5 | 6 | ## Getting started 7 | 8 | First, install CometBFT: 9 | 10 | ```` 11 | make consensus_install 12 | ```` 13 | This will install CometBFT and leave the binary in `bin/comebft`. Additionally, it will run `cometbft init` which creates the required configuration file and stores it in `~/.cometbft/`. 14 | 15 | You can now run the sequencer ABCI with CometBFT by just running those two binaries. 16 | 17 | Run CometBFT node on a terminal: 18 | 19 | ```bash 20 | make node 21 | ``` 22 | Run the ABCI sequencer application: 23 | 24 | ```bash 25 | make abci 26 | ``` 27 | In order to reset CometBFT's state before rerunning it, make sure you run `make reset` 28 | 29 | ### Sending a transaction 30 | 31 | To send executions to the sequencer you need to have a compiled Cairo program (*.json files in the repo). Then you can send them like so: 32 | 33 | ```bash 34 | cargo run --release execute sequencer/programs/fibonacci.json main 35 | ``` 36 | 37 | ### Running Tendermint Core instead of CometBFT 38 | 39 | Current code can be run with both Tendermint and CometBFT (up to version 0.34.27). In order to use Tendermint Core the make command should include the `CONSENSUS` variable: 40 | 41 | ```bash 42 | make node CONSENSUS=tendermint 43 | ``` 44 | 45 | This will run Tendermint Core instead of CometBFT (and also will install and configure it if not present). 46 | 47 | ### Benchmark 48 | 49 | You can run a benchmark with 50 | 51 | ```bash 52 | cd sequencer 53 | cargo run --release --bin bench -- --nodes "{list-of-nodes}" --threads 4 --transactions-per-thread 1000 54 | ``` 55 | 56 | Where `list-of-nodes` is a list of addresses that are part of the Tendermint network (in the form of `ipaddr:socket`). 57 | The benchmark runs `fibonacci.json` (`fib(500)`), where requests are sent with a round-robin fashion to the list of nodes, through the number of threads you specify, with the amount of transactions per thread you desire. 58 | 59 | #### Example run 60 | 61 | ```bash 62 | > cargo run --release --bin bench -- --nodes "127.0.0.1:26157 127.0.0.1:26057" 63 | 64 | Time it took for all transactions to be delivered: 1308 ms 65 | ``` 66 | 67 | Note that this is the time for all transactions to return (ie; validation that they have entered the mempool), but no assumptions can be made in terms of transaction finality. 68 | 69 | ### Benchmarking with Tendermint Load Testing Framework 70 | 71 | There is an alternate way to benchmark the app: using [tm-load-test](https://github.com/informalsystems/tm-load-test). In order to do that there is a load_test command written in go in `/bench` directory. This needs Go v1.12 at least to be built. 72 | 73 | To build it: 74 | 75 | ```bash 76 | cd bench 77 | go build -o ./build/load_test ./cmd/load_test/main.go 78 | ``` 79 | 80 | and once built move back to root directory and use 81 | 82 | ```bash 83 | ./bench/build/load_test -c 10 -T 10 -r 1000 -s 250 --broadcast-tx-method async --endpoints ws://localhost:26657/websocket --stats-output result.csv 84 | ``` 85 | 86 | to run it against a local tendermint+abci node. 87 | 88 | `-c` is the amount of connectios per endpoint 89 | 90 | `-T` is the amount of seconds to run the test 91 | 92 | `-r` is the rate of tx per second to send 93 | 94 | `-s` is the maximum size of a transaction to be sent. 95 | 96 | Run 97 | ```bash 98 | ./bench/build/load_test -h 99 | ``` 100 | to get further information. 101 | 102 | In result.csv there will be a summary of the operation. eg: 103 | ```bash 104 | > cat result.csv 105 | Parameter,Value,Units 106 | total_time,10.875,seconds 107 | total_txs,55249,count 108 | total_bytes,1242747923,bytes 109 | avg_tx_rate,5080.169436,transactions per second 110 | avg_data_rate,114271208.808144,bytes per second 111 | ``` 112 | 113 | To run it against a cluster, several nodes can be provided in `--endpoints` parameter. eg: 114 | ```bash 115 | ./bench/build/load_test -c 5 -T 10 -r 1000 -s 250 --broadcast-tx-method async --endpoints ws://5.9.57.45:26657/websocket,ws://5.9.57.44:26657/websocket,ws://5.9.57.89:26657/websocket --stats-output result.csv 116 | ``` 117 | 118 | Check [tm-load-test](https://github.com/informalsystems/tm-load-test) and [Tendermint Load Testing Framework](https://github.com/informalsystems/tm-load-test/tree/main/pkg/loadtest) and for more information. 119 | 120 | ## Reference links 121 | * [Starknet sequencer](https://www.starknet.io/de/posts/engineering/starknets-new-sequencer#:~:text=What%20does%20the%20sequencer%20do%3F) 122 | * [Papyrus Starknet full node](https://medium.com/starkware/papyrus-an-open-source-starknet-full-node-396f7cd90202) 123 | * [Blockifier](https://github.com/starkware-libs/blockifier) 124 | * [tendermint-rs](https://github.com/informalsystems/tendermint-rs) 125 | * [ABCI overview](https://docs.tendermint.com/v0.34/introduction/what-is-tendermint.html#abci-overview) 126 | * [ABCI v0.34 reference](https://github.com/tendermint/tendermint/blob/v0.34.x/spec/abci/abci.md) 127 | * [CometBFT](https://github.com/cometbft/cometbft) 128 | * [About why app hash is needed](https://github.com/tendermint/tendermint/issues/1179). Also [this](https://github.com/tendermint/tendermint/blob/v0.34.x/spec/abci/apps.md#query-proofs). 129 | * [About Tendermint 0.34's future](https://github.com/tendermint/tendermint/issues/9972) 130 | ### Starknet 131 | * [Starknet State](https://docs.starknet.io/documentation/architecture_and_concepts/State/starknet-state/) 132 | * [Starknet architecture](https://david-barreto.com/starknets-architecture-review/) 133 | * [Starknet transaction lifecylce](https://docs.starknet.io/documentation/architecture_and_concepts/Blocks/transaction-life-cycle/) 134 | -------------------------------------------------------------------------------- /narwhal-abci/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /narwhal-abci/Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["node", "narwhal-abci", "starknet-abci"] 3 | -------------------------------------------------------------------------------- /narwhal-abci/README.md: -------------------------------------------------------------------------------- 1 | Based on [gakonst's narwhal-abci-evm implementation](https://github.com/gakonst/narwhal-abci-evm/blob/master/evm-abci/src/types.rs) 2 | 3 | # Narwhal & Bullshark x ABCI x Cairo 4 | 5 | Components: 6 | * Reliable stream of hashes of batches of transactions from Bullshark 7 | * Reconstruction of the ledger by querying Narwhal workers' stores for the confirmed batches of transactions 8 | * Delivery of the reconstructed ledger over ABCI to the application logic 9 | * Implementation of a Rust ABCI app using Cairo/Starknet-in-rust 10 | 11 | ![](./assets/architecture.png) 12 | 13 | ## Demo 14 | 15 | Setup/dependencies (from the main folder of the repository): 16 | * [Rust](https://www.rust-lang.org/) 17 | * [Python Poetry](https://python-poetry.org/) 18 | * [tmux](https://github.com/tmux/tmux) 19 | * `cd demo && poetry install` 20 | 21 | Run demo (from the main folder of the repository): 22 | 1. 1st terminal: `cd demo && cargo build && poetry run fab local` 23 | 2. 2nd terminal (after the testbed has started in 1st terminal): `cargo run --bin client` 24 | 25 | The second command will produce output like this: 26 | 27 | 28 | The demo consensus network is run by four nodes (each running on localhost), whose RPC endpoints are reachable on TCP ports 3002, 3009, 3016, and 3023, respectively. There are three accounts, Alice (initially 1.5 ETH), Bob (initially 0 ETH), and Charlie (initially 0 ETH). Alice performs a double spend, sending 1 ETH each to Bob and Charlie in two different transactions that get input to the nodes at ports 3009 and 3016, respectively. Note that only one transaction can make it. Eventually, nodes reach consensus on which transaction gets executed in Foundry's EVM, and the application state is updated in lockstep across all nodes. The update is reflected in subsequent balance queries. 29 | 30 | ## TODOs 31 | 32 | 1. Why does the state transition take a few seconds to get applied? 33 | 2. Can we make this work with Anvil instead of rebuilding a full EVM execution environment? 34 | -------------------------------------------------------------------------------- /narwhal-abci/assets/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lambdaclass/starknet_tendermint_sequencer/a29f5ec2267b1caf837230e86283b7db1a92830d/narwhal-abci/assets/architecture.png -------------------------------------------------------------------------------- /narwhal-abci/demo/.gitignore: -------------------------------------------------------------------------------- 1 | .committee.json 2 | .db-*/ 3 | .node-*.json 4 | .parameters.json 5 | benchmark_client 6 | logs/ 7 | node 8 | -------------------------------------------------------------------------------- /narwhal-abci/demo/README.md: -------------------------------------------------------------------------------- 1 | # Running Benchmarks 2 | This document explains how to benchmark the codebase and read benchmarks' results. It also provides a step-by-step tutorial to run benchmarks on [Amazon Web Services (AWS)](https://aws.amazon.com) accross multiple data centers (WAN). 3 | 4 | ## Local Benchmarks 5 | When running benchmarks, the codebase is automatically compiled with the feature flag `benchmark`. This enables the node to print some special log entries that are then read by the python scripts and used to compute performance. These special log entries are clearly indicated with comments in the code: make sure to not alter them (otherwise the benchmark scripts will fail to interpret the logs). 6 | 7 | ### Parametrize the benchmark 8 | After cloning the repo and [installing all dependencies](https://github.com/asonnino/narwhal#quick-start), you can use [Fabric](http://www.fabfile.org/) to run benchmarks on your local machine. Locate the task called `local` in the file [fabfile.py](https://github.com/asonnino/narwhal/blob/master/benchmark/fabfile.py): 9 | ```python 10 | @task 11 | def local(ctx): 12 | ... 13 | ``` 14 | The task specifies two types of parameters, the *benchmark parameters* and the *nodes parameters*. The benchmark parameters look as follows: 15 | ```python 16 | bench_params = { 17 | 'nodes': 4, 18 | 'workers': 1, 19 | 'rate': 50_000, 20 | 'tx_size': 512, 21 | 'faults': 0, 22 | 'duration': 20, 23 | } 24 | ``` 25 | They specify the number of primaries (`nodes`) and workers per primary (`workers`) to deploy, the input rate (tx/s) at which the clients submits transactions to the system (`rate`), the size of each transaction in bytes (`tx_size`), the number of faulty nodes ('faults), and the duration of the benchmark in seconds (`duration`). The minimum transaction size is 9 bytes, this ensure that the transactions of a client are all different. The benchmarking script will deploy as many clients as workers and divide the input rate equally amongst each client. For instance, if you configure the testbed with 4 nodes, 1 worker per node, and an input rate of 1,000 tx/s (as in the example above), the scripts will deploy 4 clients each submitting transactions to one node at a rate of 250 tx/s. When the parameters `faults` is set to `f > 0`, the last `f` nodes and clients are not booted; the system will thus run with `n-f` nodes (and `n-f` clients). 26 | 27 | The nodes parameters determine the configuration for the primaries and workers: 28 | ```python 29 | node_params = { 30 | 'header_size': 1_000, 31 | 'max_header_delay': 5_000, 32 | 'gc_depth': 50, 33 | 'sync_retry_delay': 10_000, 34 | 'sync_retry_nodes': 3, 35 | 'batch_size': 500_000, 36 | 'max_batch_delay': 100 37 | } 38 | ``` 39 | They are defined as follows: 40 | * `header_size`: The preferred header size. The primary creates a new header when it has enough parents and enough batches' digests to reach `header_size`. Denominated in bytes. 41 | * `max_header_delay`: The maximum delay that the primary waits between generating two headers, even if the header did not reach `max_header_size`. Denominated in ms. 42 | * `gc_depth`: The depth of the garbage collection (Denominated in number of rounds). 43 | * `sync_retry_delay`: The delay after which the synchronizer retries to send sync requests. Denominated in ms. 44 | * `sync_retry_nodes`: Determine with how many nodes to sync when re-trying to send sync-request. These nodes are picked at random from the committee. 45 | * `batch_size`: The preferred batch size. The workers seal a batch of transactions when it reaches this size. Denominated in bytes. 46 | * `max_batch_delay`: The delay after which the workers seal a batch of transactions, even if `max_batch_size` is not reached. Denominated in ms. 47 | 48 | ### Run the benchmark 49 | Once you specified both `bench_params` and `node_params` as desired, run: 50 | ``` 51 | $ fab local 52 | ``` 53 | This command first recompiles your code in `release` mode (and with the `benchmark` feature flag activated), thus ensuring you always benchmark the latest version of your code. This may take a long time the first time you run it. It then generates the configuration files and keys for each node, and runs the benchmarks with the specified parameters. It finally parses the logs and displays a summary of the execution similarly to the one below. All the configuration and key files are hidden JSON files; i.e., their name starts with a dot (`.`), such as `.committee.json`. 54 | ``` 55 | ----------------------------------------- 56 | SUMMARY: 57 | ----------------------------------------- 58 | + CONFIG: 59 | Faults: 0 node(s) 60 | Committee size: 4 node(s) 61 | Worker(s) per node: 1 worker(s) 62 | Collocate primary and workers: True 63 | Input rate: 50,000 tx/s 64 | Transaction size: 512 B 65 | Execution time: 19 s 66 | 67 | Header size: 1,000 B 68 | Max header delay: 100 ms 69 | GC depth: 50 round(s) 70 | Sync retry delay: 10,000 ms 71 | Sync retry nodes: 3 node(s) 72 | batch size: 500,000 B 73 | Max batch delay: 100 ms 74 | 75 | + RESULTS: 76 | Consensus TPS: 46,478 tx/s 77 | Consensus BPS: 23,796,531 B/s 78 | Consensus latency: 464 ms 79 | 80 | End-to-end TPS: 46,149 tx/s 81 | End-to-end BPS: 23,628,541 B/s 82 | End-to-end latency: 557 ms 83 | ----------------------------------------- 84 | ``` 85 | The 'Consensus TPS' and 'Consensus latency' respectively report the average throughput and latency without considering the client. The consensus latency thus refers to the time elapsed between the block's creation and its commit. In contrast, 'End-to-end TPS' and 'End-to-end latency' report the performance of the whole system, starting from when the client submits the transaction. The end-to-end latency is often called 'client-perceived latency'. To accurately measure this value without degrading performance, the client periodically submits 'sample' transactions that are tracked across all the modules until they get committed into a block; the benchmark scripts use sample transactions to estimate the end-to-end latency. 86 | 87 | ## AWS Benchmarks 88 | This repo integrates various python scripts to deploy and benchmark the codebase on [Amazon Web Services (AWS)](https://aws.amazon.com). They are particularly useful to run benchmarks in the WAN, across multiple data centers. This section provides a step-by-step tutorial explaining how to use them. 89 | 90 | ### Step 1. Set up your AWS credentials 91 | Set up your AWS credentials to enable programmatic access to your account from your local machine. These credentials will authorize your machine to create, delete, and edit instances on your AWS account programmatically. First of all, [find your 'access key id' and 'secret access key'](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-creds). Then, create a file `~/.aws/credentials` with the following content: 92 | ``` 93 | [default] 94 | aws_access_key_id = YOUR_ACCESS_KEY_ID 95 | aws_secret_access_key = YOUR_SECRET_ACCESS_KEY 96 | ``` 97 | Do not specify any AWS region in that file as the python scripts will allow you to handle multiple regions programmatically. 98 | 99 | ### Step 2. Add your SSH public key to your AWS account 100 | You must now [add your SSH public key to your AWS account](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html). This operation is manual (AWS exposes little APIs to manipulate keys) and needs to be repeated for each AWS region that you plan to use. Upon importing your key, AWS requires you to choose a 'name' for your key; ensure you set the same name on all AWS regions. This SSH key will be used by the python scripts to execute commands and upload/download files to your AWS instances. 101 | If you don't have an SSH key, you can create one using [ssh-keygen](https://www.ssh.com/ssh/keygen/): 102 | ``` 103 | $ ssh-keygen -f ~/.ssh/aws 104 | ``` 105 | 106 | ### Step 3. Configure the testbed 107 | The file [settings.json](https://github.com/asonnino/narwhal/blob/master/benchmark/settings.json) (located in [narwhal/benchmarks](https://github.com/asonnino/narwhal/blob/master/benchmark)) contains all the configuration parameters of the testbed to deploy. Its content looks as follows: 108 | ```json 109 | { 110 | "key": { 111 | "name": "aws", 112 | "path": "/absolute/key/path" 113 | }, 114 | "port": 5000, 115 | "repo": { 116 | "name": "narwhal", 117 | "url": "https://github.com/asonnino/narwhal.git", 118 | "branch": "master" 119 | }, 120 | "instances": { 121 | "type": "m5d.8xlarge", 122 | "regions": ["us-east-1", "eu-north-1", "ap-southeast-2", "us-west-1", "ap-northeast-1"] 123 | } 124 | } 125 | ``` 126 | The first block (`key`) contains information regarding your SSH key: 127 | ```json 128 | "key": { 129 | "name": "aws", 130 | "path": "/absolute/key/path" 131 | }, 132 | ``` 133 | Enter the name of your SSH key; this is the name you specified in the AWS web console in step 2. Also, enter the absolute path of your SSH private key (using a relative path won't work). 134 | 135 | 136 | The second block (`ports`) specifies the TCP ports to use: 137 | ```json 138 | "port": 5000, 139 | ``` 140 | Narwhal requires a number of TCP ports, depening on the number of workers per node, Each primary requires 2 ports (one to receive messages from other primaties and one to receive messages from its workers), and each worker requires 3 ports (one to receive client transactions, one to receive messages from its primary, and one to receive messages from other workers). Note that the script will open a large port range (5000-7000) to the WAN on all your AWS instances. 141 | 142 | The third block (`repo`) contains the information regarding the repository's name, the URL of the repo, and the branch containing the code to deploy: 143 | ```json 144 | "repo": { 145 | "name": "narwhal", 146 | "url": "https://github.com/asonnino/narwhal.git", 147 | "branch": "master" 148 | }, 149 | ``` 150 | Remember to update the `url` field to the name of your repo. Modifying the branch name is particularly useful when testing new functionalities without having to checkout the code locally. 151 | 152 | The the last block (`instances`) specifies the [AWS instance type](https://aws.amazon.com/ec2/instance-types) and the [AWS regions](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) to use: 153 | ```json 154 | "instances": { 155 | "type": "m5d.8xlarge", 156 | "regions": ["us-east-1", "eu-north-1", "ap-southeast-2", "us-west-1", "ap-northeast-1"] 157 | } 158 | ``` 159 | The instance type selects the hardware on which to deploy the testbed. For example, `m5d.8xlarge` instances come with 32 vCPUs (16 physical cores), 128 GB of RAM, and guarantee 10 Gbps of bandwidth. The python scripts will configure each instance with 300 GB of SSD hard drive. The `regions` field specifies the data centers to use. If you require more nodes than data centers, the python scripts will distribute the nodes as equally as possible amongst the data centers. All machines run a fresh install of Ubuntu Server 20.04. 160 | 161 | ### Step 4. Create a testbed 162 | The AWS instances are orchestrated with [Fabric](http://www.fabfile.org) from the file [fabfile.py](https://github.com/asonnino/narwhal/blob/master/benchmark/fabfile.pyy) (located in [narwhal/benchmarks](https://github.com/asonnino/narwhal/blob/master/benchmark)); you can list all possible commands as follows: 163 | ``` 164 | $ cd narwhal/benchmark 165 | $ fab --list 166 | ``` 167 | The command `fab create` creates new AWS instances; open [fabfile.py](https://github.com/asonnino/narwhal/blob/master/benchmark/fabfile.py) and locate the `create` task: 168 | ```python 169 | @task 170 | def create(ctx, nodes=2): 171 | ... 172 | ``` 173 | The parameter `nodes` determines how many instances to create in *each* AWS region. That is, if you specified 5 AWS regions as in the example of step 3, setting `nodes=2` will creates a total of 10 machines: 174 | ``` 175 | $ fab create 176 | 177 | Creating 10 instances |██████████████████████████████| 100.0% 178 | Waiting for all instances to boot... 179 | Successfully created 10 new instances 180 | ``` 181 | You can then clone the repo and install rust on the remote instances with `fab install`: 182 | ``` 183 | $ fab install 184 | 185 | Installing rust and cloning the repo... 186 | Initialized testbed of 10 nodes 187 | ``` 188 | This may take a long time as the command will first update all instances. 189 | The commands `fab stop` and `fab start` respectively stop and start the testbed without destroying it (it is good practice to stop the testbed when not in use as AWS can be quite expensive); and `fab destroy` terminates all instances and destroys the testbed. Note that, depending on the instance types, AWS instances may take up to several minutes to fully start or stop. The command `fab info` displays a nice summary of all available machines and information to manually connect to them (for debug). 190 | 191 | ### Step 5. Run a benchmark 192 | After setting up the testbed, running a benchmark on AWS is similar to running it locally (see [Run Local Benchmarks](https://github.com/asonnino/narwhal/tree/master/benchmark#local-benchmarks)). Locate the task `remote` in [fabfile.py](https://github.com/asonnino/narwhal/blob/master/benchmark/fabfile.py): 193 | ```python 194 | @task 195 | def remote(ctx): 196 | ... 197 | ``` 198 | The benchmark parameters are similar to [local benchmarks](https://github.com/asonnino/narwhal/tree/master/benchmark#local-benchmarks) but allow to specify the number of nodes and the input rate as arrays to automate multiple benchmarks with a single command. The parameter `runs` specifies the number of times to repeat each benchmark (to later compute the average and stdev of the results), and the parameter `collocate` specifies whether to collocate all the node's workers and the primary on the same machine. If `collocate` is set to `False`, the script will run one node per data center (AWS region), with its primary and each of its worker running on a dedicated instance. 199 | ```python 200 | bench_params = { 201 | 'nodes': [10, 20, 30], 202 | 'workers: 2, 203 | 'collocate': True, 204 | 'rate': [20_000, 30_000, 40_000], 205 | 'tx_size': 512, 206 | 'faults': 0, 207 | 'duration': 300, 208 | 'runs': 2, 209 | } 210 | ``` 211 | Similarly to local benchmarks, the scripts will deploy as many clients as workers and divide the input rate equally amongst each client. Each client is colocated with a worker, and only submit transactions to the worker with whom they share the machine. 212 | 213 | Once you specified both `bench_params` and `node_params` as desired, run: 214 | ``` 215 | $ fab remote 216 | ``` 217 | This command first updates all machines with the latest commit of the GitHub repo and branch specified in your file [settings.json](https://github.com/asonnino/narwhal/blob/master/benchmark/settings.json) (step 3); this ensures that benchmarks are always run with the latest version of the code. It then generates and uploads the configuration files to each machine, runs the benchmarks with the specified parameters, and downloads the logs. It finally parses the logs and prints the results into a folder called `results` (which is automatically created if it doesn't already exists). You can run `fab remote` multiple times without fearing to override previous results, the command either appends new results to a file containing existing ones or prints them in separate files. If anything goes wrong during a benchmark, you can always stop it by running `fab kill`. 218 | 219 | ### Step 6. Plot the results 220 | Once you have enough results, you can aggregate and plot them: 221 | ``` 222 | $ fab plot 223 | ``` 224 | This command creates a latency graph, a throughput graph, and a robustness graph in a folder called `plots` (which is automatically created if it doesn't already exists). You can adjust the plot parameters to filter which curves to add to the plot: 225 | ```python 226 | plot_params = { 227 | 'faults': [0], 228 | 'nodes': [10, 20, 50], 229 | 'workers': [1], 230 | 'collocate': True, 231 | 'tx_size': 512, 232 | 'max_latency': [3_500, 4_500] 233 | } 234 | ``` 235 | 236 | The first graph ('latency') plots the latency versus the throughput. It shows that the latency is low until a fairly neat threshold after which it drastically increases. Determining this threshold is crucial to understand the limits of the system. 237 | 238 | Another challenge is comparing apples-to-apples between different deployments of the system. The challenge here is again that latency and throughput are interdependent, as a result a throughput/number of nodes chart could be tricky to produce fairly. The way to do it is to define a maximum latency and measure the throughput at this point instead of simply pushing every system to its peak throughput (where latency is meaningless). The second graph ('tps') plots the maximum achievable throughput under a maximum latency for different numbers of nodes. 239 | -------------------------------------------------------------------------------- /narwhal-abci/demo/benchmark/__init__: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lambdaclass/starknet_tendermint_sequencer/a29f5ec2267b1caf837230e86283b7db1a92830d/narwhal-abci/demo/benchmark/__init__ -------------------------------------------------------------------------------- /narwhal-abci/demo/benchmark/__pycache__/logs.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lambdaclass/starknet_tendermint_sequencer/a29f5ec2267b1caf837230e86283b7db1a92830d/narwhal-abci/demo/benchmark/__pycache__/logs.cpython-310.pyc -------------------------------------------------------------------------------- /narwhal-abci/demo/benchmark/__pycache__/utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lambdaclass/starknet_tendermint_sequencer/a29f5ec2267b1caf837230e86283b7db1a92830d/narwhal-abci/demo/benchmark/__pycache__/utils.cpython-310.pyc -------------------------------------------------------------------------------- /narwhal-abci/demo/benchmark/aggregate.py: -------------------------------------------------------------------------------- 1 | # Copyright(C) Facebook, Inc. and its affiliates. 2 | from re import search 3 | from collections import defaultdict 4 | from statistics import mean, stdev 5 | from glob import glob 6 | from copy import deepcopy 7 | from os.path import join 8 | import os 9 | 10 | from benchmark.utils import PathMaker 11 | 12 | 13 | class Setup: 14 | def __init__(self, faults, nodes, workers, collocate, rate, tx_size): 15 | self.nodes = nodes 16 | self.workers = workers 17 | self.collocate = collocate 18 | self.rate = rate 19 | self.tx_size = tx_size 20 | self.faults = faults 21 | self.max_latency = 'any' 22 | 23 | def __str__(self): 24 | return ( 25 | f' Faults: {self.faults}\n' 26 | f' Committee size: {self.nodes}\n' 27 | f' Workers per node: {self.workers}\n' 28 | f' Collocate primary and workers: {self.collocate}\n' 29 | f' Input rate: {self.rate} tx/s\n' 30 | f' Transaction size: {self.tx_size} B\n' 31 | f' Max latency: {self.max_latency} ms\n' 32 | ) 33 | 34 | def __eq__(self, other): 35 | return isinstance(other, Setup) and str(self) == str(other) 36 | 37 | def __hash__(self): 38 | return hash(str(self)) 39 | 40 | @classmethod 41 | def from_str(cls, raw): 42 | faults = int(search(r'Faults: (\d+)', raw).group(1)) 43 | nodes = int(search(r'Committee size: (\d+)', raw).group(1)) 44 | workers = int(search(r'Worker\(s\) per node: (\d+)', raw).group(1)) 45 | collocate = 'True' == search( 46 | r'Collocate primary and workers: (True|False)', raw 47 | ).group(1) 48 | rate = int(search(r'Input rate: (\d+)', raw).group(1)) 49 | tx_size = int(search(r'Transaction size: (\d+)', raw).group(1)) 50 | return cls(faults, nodes, workers, collocate, rate, tx_size) 51 | 52 | 53 | class Result: 54 | def __init__(self, mean_tps, mean_latency, std_tps=0, std_latency=0): 55 | self.mean_tps = mean_tps 56 | self.mean_latency = mean_latency 57 | self.std_tps = std_tps 58 | self.std_latency = std_latency 59 | 60 | def __str__(self): 61 | return( 62 | f' TPS: {self.mean_tps} +/- {self.std_tps} tx/s\n' 63 | f' Latency: {self.mean_latency} +/- {self.std_latency} ms\n' 64 | ) 65 | 66 | @classmethod 67 | def from_str(cls, raw): 68 | tps = int(search(r'End-to-end TPS: (\d+)', raw).group(1)) 69 | latency = int(search(r'End-to-end latency: (\d+)', raw).group(1)) 70 | return cls(tps, latency) 71 | 72 | @classmethod 73 | def aggregate(cls, results): 74 | if len(results) == 1: 75 | return results[0] 76 | 77 | mean_tps = round(mean([x.mean_tps for x in results])) 78 | mean_latency = round(mean([x.mean_latency for x in results])) 79 | std_tps = round(stdev([x.mean_tps for x in results])) 80 | std_latency = round(stdev([x.mean_latency for x in results])) 81 | return cls(mean_tps, mean_latency, std_tps, std_latency) 82 | 83 | 84 | class LogAggregator: 85 | def __init__(self, max_latencies): 86 | assert isinstance(max_latencies, list) 87 | assert all(isinstance(x, int) for x in max_latencies) 88 | 89 | self.max_latencies = max_latencies 90 | 91 | data = '' 92 | for filename in glob(join(PathMaker.results_path(), '*.txt')): 93 | with open(filename, 'r') as f: 94 | data += f.read() 95 | 96 | records = defaultdict(list) 97 | for chunk in data.replace(',', '').split('SUMMARY')[1:]: 98 | if chunk: 99 | records[Setup.from_str(chunk)] += [Result.from_str(chunk)] 100 | 101 | self.records = {k: Result.aggregate(v) for k, v in records.items()} 102 | 103 | def print(self): 104 | if not os.path.exists(PathMaker.plots_path()): 105 | os.makedirs(PathMaker.plots_path()) 106 | 107 | results = [ 108 | self._print_latency(), 109 | self._print_tps(scalability=False), 110 | self._print_tps(scalability=True), 111 | ] 112 | for name, records in results: 113 | for setup, values in records.items(): 114 | data = '\n'.join( 115 | f' Variable value: X={x}\n{y}' for x, y in values 116 | ) 117 | string = ( 118 | '\n' 119 | '-----------------------------------------\n' 120 | ' RESULTS:\n' 121 | '-----------------------------------------\n' 122 | f'{setup}' 123 | '\n' 124 | f'{data}' 125 | '-----------------------------------------\n' 126 | ) 127 | 128 | max_lat = setup.max_latency 129 | filename = PathMaker.agg_file( 130 | name, 131 | setup.faults, 132 | setup.nodes, 133 | setup.workers, 134 | setup.collocate, 135 | setup.rate, 136 | setup.tx_size, 137 | max_latency=None if max_lat == 'any' else max_lat, 138 | ) 139 | with open(filename, 'w') as f: 140 | f.write(string) 141 | 142 | def _print_latency(self): 143 | records = deepcopy(self.records) 144 | organized = defaultdict(list) 145 | for setup, result in records.items(): 146 | rate = setup.rate 147 | setup.rate = 'any' 148 | organized[setup] += [(result.mean_tps, result, rate)] 149 | 150 | for setup, results in list(organized.items()): 151 | results.sort(key=lambda x: x[2]) 152 | organized[setup] = [(x, y) for x, y, _ in results] 153 | 154 | return 'latency', organized 155 | 156 | def _print_tps(self, scalability): 157 | records = deepcopy(self.records) 158 | organized = defaultdict(list) 159 | for max_latency in self.max_latencies: 160 | for setup, result in records.items(): 161 | setup = deepcopy(setup) 162 | if result.mean_latency <= max_latency: 163 | setup.rate = 'any' 164 | setup.max_latency = max_latency 165 | if scalability: 166 | variable = setup.workers 167 | setup.workers = 'x' 168 | else: 169 | variable = setup.nodes 170 | setup.nodes = 'x' 171 | 172 | new_point = all(variable != x[0] for x in organized[setup]) 173 | highest_tps = False 174 | for v, r in organized[setup]: 175 | if result.mean_tps > r.mean_tps and variable == v: 176 | organized[setup].remove((v, r)) 177 | highest_tps = True 178 | if new_point or highest_tps: 179 | organized[setup] += [(variable, result)] 180 | 181 | [v.sort(key=lambda x: x[0]) for v in organized.values()] 182 | return 'tps', organized 183 | -------------------------------------------------------------------------------- /narwhal-abci/demo/benchmark/commands.py: -------------------------------------------------------------------------------- 1 | # Copyright(C) Facebook, Inc. and its affiliates. 2 | from os.path import join 3 | import os 4 | 5 | from benchmark.utils import PathMaker 6 | 7 | NODE = "./../target/debug/node" 8 | 9 | 10 | class CommandMaker: 11 | 12 | @staticmethod 13 | def cleanup(): 14 | return ( 15 | f'rm -r .db-* ; rm .*.json ; mkdir -p {PathMaker.results_path()}' 16 | ) 17 | 18 | @staticmethod 19 | def clean_logs(): 20 | return f'rm -r {PathMaker.logs_path()} ; mkdir -p {PathMaker.logs_path()}' 21 | 22 | @staticmethod 23 | def compile(): 24 | return 'cargo build' 25 | # return 'cargo build --quiet --release --features benchmark' 26 | 27 | @staticmethod 28 | def generate_key(filename): 29 | assert isinstance(filename, str) 30 | return f'{NODE} generate_keys --filename {filename}' 31 | 32 | @staticmethod 33 | def run_primary(keys, committee, store, parameters, app_api, abci_api, debug=False): 34 | print(store, keys) 35 | assert isinstance(keys, str) 36 | assert isinstance(committee, str) 37 | assert isinstance(parameters, str) 38 | assert isinstance(debug, bool) 39 | v = '-vvv' if debug else '-vv' 40 | return (f'{NODE} {v} run --keys {keys} --committee {committee} ' 41 | f'--store {store} --parameters {parameters} primary --app-api {app_api} --abci-api {abci_api} ') 42 | 43 | @staticmethod 44 | def run_worker(keys, committee, store, parameters, id, debug=False): 45 | assert isinstance(keys, str) 46 | assert isinstance(committee, str) 47 | assert isinstance(parameters, str) 48 | assert isinstance(debug, bool) 49 | v = '-vvv' if debug else '-vv' 50 | return (f'{NODE} {v} run --keys {keys} --committee {committee} ' 51 | f'--store {store} --parameters {parameters} worker --id {id}') 52 | 53 | @staticmethod 54 | def run_client(address, size, rate, nodes): 55 | assert isinstance(address, str) 56 | assert isinstance(size, int) and size > 0 57 | assert isinstance(rate, int) and rate >= 0 58 | assert isinstance(nodes, list) 59 | assert all(isinstance(x, str) for x in nodes) 60 | nodes = f'--nodes {" ".join(nodes)}' if nodes else '' 61 | return f'./../target/debug/benchmark_client {address} --size {size} --rate {rate} {nodes}' 62 | 63 | @staticmethod 64 | def run_app(listen_on): 65 | assert isinstance(listen_on, str) 66 | return f'../target/debug/starknet-app --demo {listen_on}' 67 | 68 | @staticmethod 69 | def kill(): 70 | print("os.getenv('TMUX'):", os.getenv('TMUX')) 71 | if os.getenv('TMUX'): 72 | # running within tmux (Georgios' config) 73 | # kill all other sessions 74 | return "tmux kill-session -a" 75 | else: 76 | # running without tmux (Joachim's config) 77 | # This does not work when running in Tmux 78 | return 'tmux kill-server' 79 | 80 | @staticmethod 81 | def alias_binaries(origin): 82 | assert isinstance(origin, str) 83 | # This is aliasing only the release 84 | # print('Origin', origin) 85 | node, client = join(origin, 'node'), join(origin, 'benchmark_client') 86 | return f'rm node ; rm benchmark_client ; ln -s {node} . ; ln -s {client} .' 87 | -------------------------------------------------------------------------------- /narwhal-abci/demo/benchmark/config.py: -------------------------------------------------------------------------------- 1 | # Copyright(C) Facebook, Inc. and its affiliates. 2 | from json import dump, load 3 | from collections import OrderedDict 4 | 5 | 6 | class ConfigError(Exception): 7 | pass 8 | 9 | 10 | class Key: 11 | def __init__(self, name, secret): 12 | self.name = name 13 | self.secret = secret 14 | 15 | @classmethod 16 | def from_file(cls, filename): 17 | assert isinstance(filename, str) 18 | with open(filename, 'r') as f: 19 | data = load(f) 20 | return cls(data['name'], data['secret']) 21 | 22 | 23 | class Committee: 24 | ''' The committee looks as follows: 25 | "authorities: { 26 | "name": { 27 | "stake": 1, 28 | "primary: { 29 | "primary_to_primary": x.x.x.x:x, 30 | "worker_to_primary": x.x.x.x:x, 31 | }, 32 | "workers": { 33 | "0": { 34 | "primary_to_worker": x.x.x.x:x, 35 | "worker_to_worker": x.x.x.x:x, 36 | "transactions": x.x.x.x:x 37 | }, 38 | ... 39 | } 40 | }, 41 | ... 42 | } 43 | ''' 44 | 45 | def __init__(self, addresses, base_port): 46 | ''' The `addresses` field looks as follows: 47 | { 48 | "name": ["host", "host", ...], 49 | ... 50 | } 51 | ''' 52 | assert isinstance(addresses, OrderedDict) 53 | assert all(isinstance(x, str) for x in addresses.keys()) 54 | assert all( 55 | isinstance(x, list) and len(x) > 1 for x in addresses.values() 56 | ) 57 | assert all( 58 | isinstance(x, str) for y in addresses.values() for x in y 59 | ) 60 | assert len({len(x) for x in addresses.values()}) == 1 61 | assert isinstance(base_port, int) and base_port > 1024 62 | 63 | port = base_port 64 | self.json = {'authorities': OrderedDict()} 65 | for name, hosts in addresses.items(): 66 | host = hosts.pop(0) 67 | primary_addr = { 68 | 'primary_to_primary': f'{host}:{port}', 69 | 'worker_to_primary': f'{host}:{port + 1}', 70 | 71 | # new addresses for APIs 72 | 'api_rpc': f'{host}:{port + 2}', 73 | 'api_abci': f'{host}:{port + 3}', 74 | } 75 | port += 4 76 | 77 | workers_addr = OrderedDict() 78 | for j, host in enumerate(hosts): 79 | workers_addr[j] = { 80 | 'primary_to_worker': f'{host}:{port}', 81 | 'transactions': f'{host}:{port + 1}', 82 | 'worker_to_worker': f'{host}:{port + 2}', 83 | } 84 | port += 3 85 | 86 | self.json['authorities'][name] = { 87 | 'stake': 1, 88 | 'primary': primary_addr, 89 | 'workers': workers_addr 90 | } 91 | 92 | def primary_addresses(self, faults=0): 93 | ''' Returns an ordered list of primaries' addresses. ''' 94 | assert faults < self.size() 95 | addresses = [] 96 | good_nodes = self.size() - faults 97 | for authority in list(self.json['authorities'].values())[:good_nodes]: 98 | addresses += [authority['primary']['primary_to_primary']] 99 | return addresses 100 | 101 | def rpc_addresses(self, faults=0): 102 | ''' Returns an ordered list of rpcs' addresses. ''' 103 | assert faults < self.size() 104 | addresses = [] 105 | good_nodes = self.size() - faults 106 | for authority in list(self.json['authorities'].values())[:good_nodes]: 107 | addresses += [authority['primary']['api_rpc']] 108 | return addresses 109 | 110 | def app_addresses(self, faults=0): 111 | ''' Returns an ordered list of apps' addresses. ''' 112 | assert faults < self.size() 113 | addresses = [] 114 | good_nodes = self.size() - faults 115 | for authority in list(self.json['authorities'].values())[:good_nodes]: 116 | addresses += [authority['primary']['api_abci']] 117 | return addresses 118 | 119 | def workers_addresses(self, faults=0): 120 | ''' Returns an ordered list of list of workers' addresses. ''' 121 | assert faults < self.size() 122 | addresses = [] 123 | good_nodes = self.size() - faults 124 | for authority in list(self.json['authorities'].values())[:good_nodes]: 125 | authority_addresses = [] 126 | for id, worker in authority['workers'].items(): 127 | authority_addresses += [(id, worker['transactions'])] 128 | addresses.append(authority_addresses) 129 | return addresses 130 | 131 | def ips(self, name=None): 132 | ''' Returns all the ips associated with an authority (in any order). ''' 133 | if name is None: 134 | names = list(self.json['authorities'].keys()) 135 | else: 136 | names = [name] 137 | 138 | ips = set() 139 | for name in names: 140 | addresses = self.json['authorities'][name]['primary'] 141 | ips.add(self.ip(addresses['primary_to_primary'])) 142 | ips.add(self.ip(addresses['worker_to_primary'])) 143 | 144 | for worker in self.json['authorities'][name]['workers'].values(): 145 | ips.add(self.ip(worker['primary_to_worker'])) 146 | ips.add(self.ip(worker['worker_to_worker'])) 147 | ips.add(self.ip(worker['transactions'])) 148 | 149 | return list(ips) 150 | 151 | def remove_nodes(self, nodes): 152 | ''' remove the `nodes` last nodes from the committee. ''' 153 | assert nodes < self.size() 154 | for _ in range(nodes): 155 | self.json['authorities'].popitem() 156 | 157 | def size(self): 158 | ''' Returns the number of authorities. ''' 159 | return len(self.json['authorities']) 160 | 161 | def workers(self): 162 | ''' Returns the total number of workers (all authorities altogether). ''' 163 | return sum(len(x['workers']) for x in self.json['authorities'].values()) 164 | 165 | def print(self, filename): 166 | print(filename) 167 | assert isinstance(filename, str) 168 | with open(filename, 'w') as f: 169 | dump(self.json, f, indent=4, sort_keys=True) 170 | 171 | @staticmethod 172 | def ip(address): 173 | assert isinstance(address, str) 174 | return address.split(':')[0] 175 | 176 | 177 | class LocalCommittee(Committee): 178 | def __init__(self, names, port, workers): 179 | assert isinstance(names, list) 180 | assert all(isinstance(x, str) for x in names) 181 | assert isinstance(port, int) 182 | assert isinstance(workers, int) and workers > 0 183 | addresses = OrderedDict((x, ['127.0.0.1']*(1+workers)) for x in names) 184 | super().__init__(addresses, port) 185 | 186 | 187 | class NodeParameters: 188 | def __init__(self, json): 189 | inputs = [] 190 | try: 191 | inputs += [json['header_size']] 192 | inputs += [json['max_header_delay']] 193 | inputs += [json['gc_depth']] 194 | inputs += [json['sync_retry_delay']] 195 | inputs += [json['sync_retry_nodes']] 196 | inputs += [json['batch_size']] 197 | inputs += [json['max_batch_delay']] 198 | except KeyError as e: 199 | raise ConfigError(f'Malformed parameters: missing key {e}') 200 | 201 | if not all(isinstance(x, int) for x in inputs): 202 | raise ConfigError('Invalid parameters type') 203 | 204 | self.json = json 205 | 206 | def print(self, filename): 207 | assert isinstance(filename, str) 208 | with open(filename, 'w') as f: 209 | dump(self.json, f, indent=4, sort_keys=True) 210 | 211 | 212 | class BenchParameters: 213 | def __init__(self, json): 214 | try: 215 | self.faults = int(json['faults']) 216 | 217 | nodes = json['nodes'] 218 | nodes = nodes if isinstance(nodes, list) else [nodes] 219 | if not nodes or any(x <= 1 for x in nodes): 220 | raise ConfigError('Missing or invalid number of nodes') 221 | self.nodes = [int(x) for x in nodes] 222 | 223 | rate = json['rate'] 224 | rate = rate if isinstance(rate, list) else [rate] 225 | if not rate: 226 | raise ConfigError('Missing input rate') 227 | self.rate = [int(x) for x in rate] 228 | 229 | 230 | self.workers = int(json['workers']) 231 | 232 | if 'collocate' in json: 233 | self.collocate = bool(json['collocate']) 234 | else: 235 | self.collocate = True 236 | 237 | self.tx_size = int(json['tx_size']) 238 | 239 | self.duration = int(json['duration']) 240 | 241 | self.runs = int(json['runs']) if 'runs' in json else 1 242 | except KeyError as e: 243 | raise ConfigError(f'Malformed bench parameters: missing key {e}') 244 | 245 | except ValueError: 246 | raise ConfigError('Invalid parameters type') 247 | 248 | if min(self.nodes) <= self.faults: 249 | raise ConfigError('There should be more nodes than faults') 250 | 251 | 252 | class PlotParameters: 253 | def __init__(self, json): 254 | try: 255 | faults = json['faults'] 256 | faults = faults if isinstance(faults, list) else [faults] 257 | self.faults = [int(x) for x in faults] if faults else [0] 258 | 259 | nodes = json['nodes'] 260 | nodes = nodes if isinstance(nodes, list) else [nodes] 261 | if not nodes: 262 | raise ConfigError('Missing number of nodes') 263 | self.nodes = [int(x) for x in nodes] 264 | 265 | workers = json['workers'] 266 | workers = workers if isinstance(workers, list) else [workers] 267 | if not workers: 268 | raise ConfigError('Missing number of workers') 269 | self.workers = [int(x) for x in workers] 270 | 271 | if 'collocate' in json: 272 | self.collocate = bool(json['collocate']) 273 | else: 274 | self.collocate = True 275 | 276 | self.tx_size = int(json['tx_size']) 277 | 278 | max_lat = json['max_latency'] 279 | max_lat = max_lat if isinstance(max_lat, list) else [max_lat] 280 | if not max_lat: 281 | raise ConfigError('Missing max latency') 282 | self.max_latency = [int(x) for x in max_lat] 283 | 284 | except KeyError as e: 285 | raise ConfigError(f'Malformed bench parameters: missing key {e}') 286 | 287 | except ValueError: 288 | raise ConfigError('Invalid parameters type') 289 | 290 | if len(self.nodes) > 1 and len(self.workers) > 1: 291 | raise ConfigError( 292 | 'Either the "nodes" or the "workers can be a list (not both)' 293 | ) 294 | 295 | def scalability(self): 296 | return len(self.workers) > 1 297 | -------------------------------------------------------------------------------- /narwhal-abci/demo/benchmark/instance.py: -------------------------------------------------------------------------------- 1 | # Copyright(C) Facebook, Inc. and its affiliates. 2 | import boto3 3 | from botocore.exceptions import ClientError 4 | from collections import defaultdict, OrderedDict 5 | from time import sleep 6 | 7 | from benchmark.utils import Print, BenchError, progress_bar 8 | from benchmark.settings import Settings, SettingsError 9 | 10 | 11 | class AWSError(Exception): 12 | def __init__(self, error): 13 | assert isinstance(error, ClientError) 14 | self.message = error.response['Error']['Message'] 15 | self.code = error.response['Error']['Code'] 16 | super().__init__(self.message) 17 | 18 | 19 | class InstanceManager: 20 | INSTANCE_NAME = 'dag-node' 21 | SECURITY_GROUP_NAME = 'dag' 22 | 23 | def __init__(self, settings): 24 | assert isinstance(settings, Settings) 25 | self.settings = settings 26 | self.clients = OrderedDict() 27 | for region in settings.aws_regions: 28 | self.clients[region] = boto3.client('ec2', region_name=region) 29 | 30 | @classmethod 31 | def make(cls, settings_file='settings.json'): 32 | try: 33 | return cls(Settings.load(settings_file)) 34 | except SettingsError as e: 35 | raise BenchError('Failed to load settings', e) 36 | 37 | def _get(self, state): 38 | # Possible states are: 'pending', 'running', 'shutting-down', 39 | # 'terminated', 'stopping', and 'stopped'. 40 | ids, ips = defaultdict(list), defaultdict(list) 41 | for region, client in self.clients.items(): 42 | r = client.describe_instances( 43 | Filters=[ 44 | { 45 | 'Name': 'tag:Name', 46 | 'Values': [self.INSTANCE_NAME] 47 | }, 48 | { 49 | 'Name': 'instance-state-name', 50 | 'Values': state 51 | } 52 | ] 53 | ) 54 | instances = [y for x in r['Reservations'] for y in x['Instances']] 55 | for x in instances: 56 | ids[region] += [x['InstanceId']] 57 | if 'PublicIpAddress' in x: 58 | ips[region] += [x['PublicIpAddress']] 59 | return ids, ips 60 | 61 | def _wait(self, state): 62 | # Possible states are: 'pending', 'running', 'shutting-down', 63 | # 'terminated', 'stopping', and 'stopped'. 64 | while True: 65 | sleep(1) 66 | ids, _ = self._get(state) 67 | if sum(len(x) for x in ids.values()) == 0: 68 | break 69 | 70 | def _create_security_group(self, client): 71 | client.create_security_group( 72 | Description='HotStuff node', 73 | GroupName=self.SECURITY_GROUP_NAME, 74 | ) 75 | 76 | client.authorize_security_group_ingress( 77 | GroupName=self.SECURITY_GROUP_NAME, 78 | IpPermissions=[ 79 | { 80 | 'IpProtocol': 'tcp', 81 | 'FromPort': 22, 82 | 'ToPort': 22, 83 | 'IpRanges': [{ 84 | 'CidrIp': '0.0.0.0/0', 85 | 'Description': 'Debug SSH access', 86 | }], 87 | 'Ipv6Ranges': [{ 88 | 'CidrIpv6': '::/0', 89 | 'Description': 'Debug SSH access', 90 | }], 91 | }, 92 | { 93 | 'IpProtocol': 'tcp', 94 | 'FromPort': self.settings.base_port, 95 | 'ToPort': self.settings.base_port + 2_000, 96 | 'IpRanges': [{ 97 | 'CidrIp': '0.0.0.0/0', 98 | 'Description': 'Dag port', 99 | }], 100 | 'Ipv6Ranges': [{ 101 | 'CidrIpv6': '::/0', 102 | 'Description': 'Dag port', 103 | }], 104 | } 105 | ] 106 | ) 107 | 108 | def _get_ami(self, client): 109 | # The AMI changes with regions. 110 | response = client.describe_images( 111 | Filters=[{ 112 | 'Name': 'description', 113 | 'Values': ['Canonical, Ubuntu, 20.04 LTS, amd64 focal image build on 2020-10-26'] 114 | }] 115 | ) 116 | return response['Images'][0]['ImageId'] 117 | 118 | def create_instances(self, instances): 119 | assert isinstance(instances, int) and instances > 0 120 | 121 | # Create the security group in every region. 122 | for client in self.clients.values(): 123 | try: 124 | self._create_security_group(client) 125 | except ClientError as e: 126 | error = AWSError(e) 127 | if error.code != 'InvalidGroup.Duplicate': 128 | raise BenchError('Failed to create security group', error) 129 | 130 | try: 131 | # Create all instances. 132 | size = instances * len(self.clients) 133 | progress = progress_bar( 134 | self.clients.values(), prefix=f'Creating {size} instances' 135 | ) 136 | for client in progress: 137 | client.run_instances( 138 | ImageId=self._get_ami(client), 139 | InstanceType=self.settings.instance_type, 140 | KeyName=self.settings.key_name, 141 | MaxCount=instances, 142 | MinCount=instances, 143 | SecurityGroups=[self.SECURITY_GROUP_NAME], 144 | TagSpecifications=[{ 145 | 'ResourceType': 'instance', 146 | 'Tags': [{ 147 | 'Key': 'Name', 148 | 'Value': self.INSTANCE_NAME 149 | }] 150 | }], 151 | EbsOptimized=True, 152 | BlockDeviceMappings=[{ 153 | 'DeviceName': '/dev/sda1', 154 | 'Ebs': { 155 | 'VolumeType': 'gp2', 156 | 'VolumeSize': 200, 157 | 'DeleteOnTermination': True 158 | } 159 | }], 160 | ) 161 | 162 | # Wait for the instances to boot. 163 | Print.info('Waiting for all instances to boot...') 164 | self._wait(['pending']) 165 | Print.heading(f'Successfully created {size} new instances') 166 | except ClientError as e: 167 | raise BenchError('Failed to create AWS instances', AWSError(e)) 168 | 169 | def terminate_instances(self): 170 | try: 171 | ids, _ = self._get(['pending', 'running', 'stopping', 'stopped']) 172 | size = sum(len(x) for x in ids.values()) 173 | if size == 0: 174 | Print.heading(f'All instances are shut down') 175 | return 176 | 177 | # Terminate instances. 178 | for region, client in self.clients.items(): 179 | if ids[region]: 180 | client.terminate_instances(InstanceIds=ids[region]) 181 | 182 | # Wait for all instances to properly shut down. 183 | Print.info('Waiting for all instances to shut down...') 184 | self._wait(['shutting-down']) 185 | for client in self.clients.values(): 186 | client.delete_security_group( 187 | GroupName=self.SECURITY_GROUP_NAME 188 | ) 189 | 190 | Print.heading(f'Testbed of {size} instances destroyed') 191 | except ClientError as e: 192 | raise BenchError('Failed to terminate instances', AWSError(e)) 193 | 194 | def start_instances(self, max): 195 | size = 0 196 | try: 197 | ids, _ = self._get(['stopping', 'stopped']) 198 | for region, client in self.clients.items(): 199 | if ids[region]: 200 | target = ids[region] 201 | target = target if len(target) < max else target[:max] 202 | size += len(target) 203 | client.start_instances(InstanceIds=target) 204 | Print.heading(f'Starting {size} instances') 205 | except ClientError as e: 206 | raise BenchError('Failed to start instances', AWSError(e)) 207 | 208 | def stop_instances(self): 209 | try: 210 | ids, _ = self._get(['pending', 'running']) 211 | for region, client in self.clients.items(): 212 | if ids[region]: 213 | client.stop_instances(InstanceIds=ids[region]) 214 | size = sum(len(x) for x in ids.values()) 215 | Print.heading(f'Stopping {size} instances') 216 | except ClientError as e: 217 | raise BenchError(AWSError(e)) 218 | 219 | def hosts(self, flat=False): 220 | try: 221 | _, ips = self._get(['pending', 'running']) 222 | return [x for y in ips.values() for x in y] if flat else ips 223 | except ClientError as e: 224 | raise BenchError('Failed to gather instances IPs', AWSError(e)) 225 | 226 | def print_info(self): 227 | hosts = self.hosts() 228 | key = self.settings.key_path 229 | text = '' 230 | for region, ips in hosts.items(): 231 | text += f'\n Region: {region.upper()}\n' 232 | for i, ip in enumerate(ips): 233 | new_line = '\n' if (i+1) % 6 == 0 else '' 234 | text += f'{new_line} {i}\tssh -i {key} ubuntu@{ip}\n' 235 | print( 236 | '\n' 237 | '----------------------------------------------------------------\n' 238 | ' INFO:\n' 239 | '----------------------------------------------------------------\n' 240 | f' Available machines: {sum(len(x) for x in hosts.values())}\n' 241 | f'{text}' 242 | '----------------------------------------------------------------\n' 243 | ) 244 | -------------------------------------------------------------------------------- /narwhal-abci/demo/benchmark/local.py: -------------------------------------------------------------------------------- 1 | # Copyright(C) Facebook, Inc. and its affiliates. 2 | import subprocess 3 | from math import ceil 4 | from os.path import basename, splitext 5 | from time import sleep 6 | 7 | from benchmark.commands import CommandMaker 8 | from benchmark.config import Key, LocalCommittee, NodeParameters, BenchParameters, ConfigError 9 | from benchmark.logs import LogParser, ParseError 10 | from benchmark.utils import Print, BenchError, PathMaker 11 | 12 | 13 | class LocalBench: 14 | BASE_PORT = 3000 15 | 16 | def __init__(self, bench_parameters_dict, node_parameters_dict): 17 | print("bench params dict", bench_parameters_dict) 18 | print("node params dict", node_parameters_dict) 19 | try: 20 | self.bench_parameters = BenchParameters(bench_parameters_dict) 21 | self.node_parameters = NodeParameters(node_parameters_dict) 22 | except ConfigError as e: 23 | raise BenchError('Invalid nodes or bench parameters', e) 24 | 25 | def __getattr__(self, attr): 26 | return getattr(self.bench_parameters, attr) 27 | 28 | def _background_run(self, command, log_file): 29 | name = splitext(basename(log_file))[0] 30 | cmd = f'{command} &> {log_file}' 31 | # cmd = f'{command}' 32 | print("Background run:", ['tmux', 'new', '-d', '-s', name, cmd]) 33 | subprocess.run(['tmux', 'new', '-d', '-s', name, cmd], check=True) 34 | 35 | def _kill_nodes(self): 36 | # try: 37 | cmd = CommandMaker.kill().split() 38 | subprocess.run(cmd)#, stderr=subprocess.DEVNULL) 39 | # except subprocess.SubprocessError as e: 40 | # raise BenchError('Failed to kill testbed', e) 41 | 42 | def run(self, debug=False): 43 | assert isinstance(debug, bool) 44 | Print.heading('Starting local benchmark') 45 | 46 | # Kill any previous testbed. 47 | self._kill_nodes() 48 | 49 | try: 50 | Print.info('Setting up testbed...') 51 | nodes, rate = self.nodes[0], self.rate[0] 52 | 53 | # Cleanup all files. 54 | cmd = f'{CommandMaker.clean_logs()} ; {CommandMaker.cleanup()}' 55 | subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL) 56 | # sleep(0.5) # Removing the store may take time. 57 | 58 | print(cmd) 59 | 60 | # Recompile the latest code. 61 | cmd = CommandMaker.compile().split() 62 | print(cmd) 63 | subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path()) 64 | 65 | # Create alias for the client and nodes binary. 66 | cmd = CommandMaker.alias_binaries(PathMaker.binary_path()) 67 | print(cmd) 68 | subprocess.run([cmd], shell=True) 69 | 70 | # Generate configuration files. 71 | keys = [] 72 | key_files = [PathMaker.key_file(i) for i in range(nodes)] 73 | for filename in key_files: 74 | cmd = CommandMaker.generate_key(filename).split() 75 | subprocess.run(cmd, check=True) 76 | keys += [Key.from_file(filename)] 77 | 78 | print(key_files, keys) 79 | 80 | names = [x.name for x in keys] 81 | committee = LocalCommittee(names, self.BASE_PORT, self.workers) 82 | # prints to .committee.json 83 | # print(PathMaker.committee_file()) 84 | committee.print(PathMaker.committee_file()) 85 | 86 | print(names, committee) 87 | 88 | 89 | self.node_parameters.print(PathMaker.parameters_file()) 90 | 91 | # Run the clients (they will wait for the nodes to be ready). 92 | # Worker transaction endpoint (3003, 3008 etc.) 93 | # Probably the TPU equivalent? 94 | workers_addresses = committee.workers_addresses(self.faults) 95 | 96 | 97 | print("[+] Spinning up apps") 98 | # Run the apps 99 | for i, address in enumerate(committee.app_addresses(self.faults)): 100 | cmd = CommandMaker.run_app(address) 101 | log_file = PathMaker.app_log_file(i) 102 | # Each one of these starts a new tmux session 103 | self._background_run(cmd, log_file) 104 | 105 | sleep(1) 106 | 107 | # print("[+] Spinning up clients") 108 | # # The benchmark clients connect to the worker addresses to submit transactions 109 | # # Starts 1 client for each worker process. 110 | # rate_share = ceil(rate / committee.workers()) 111 | # for i, addresses in enumerate(workers_addresses): 112 | # for (id, address) in addresses: 113 | # cmd = CommandMaker.run_client( 114 | # address, 115 | # self.tx_size, 116 | # rate_share, 117 | # [x for y in workers_addresses for _, x in y] 118 | # ) 119 | # log_file = PathMaker.client_log_file(i, id) 120 | # print("--> [+] Running", cmd, log_file) 121 | # self._background_run(cmd, log_file) 122 | 123 | 124 | print("[+] Spinning up primaries") 125 | # Run the primaries (except the faulty ones). 126 | for i, address in enumerate(committee.primary_addresses(self.faults)): 127 | cmd = CommandMaker.run_primary( 128 | PathMaker.key_file(i), 129 | PathMaker.committee_file(), 130 | PathMaker.db_path(i), 131 | PathMaker.parameters_file(), 132 | app_api = committee.app_addresses(self.faults)[i], 133 | abci_api = committee.rpc_addresses(self.faults)[i], 134 | debug=debug 135 | ) 136 | log_file = PathMaker.primary_log_file(i) 137 | # Each one of these starts a new tmux session 138 | self._background_run(cmd, log_file) 139 | 140 | 141 | print("[+] Spinning up workers") 142 | # Run the workers (except the faulty ones). 143 | for i, addresses in enumerate(workers_addresses): 144 | for (id, address) in addresses: 145 | cmd = CommandMaker.run_worker( 146 | PathMaker.key_file(i), 147 | PathMaker.committee_file(), 148 | PathMaker.db_path(i, id), 149 | PathMaker.parameters_file(), 150 | id, # The worker's id. 151 | debug=debug 152 | ) 153 | log_file = PathMaker.worker_log_file(i, id) 154 | self._background_run(cmd, log_file) 155 | 156 | 157 | # Wait for all transactions to be processed. 158 | Print.info(f'Running benchmark ({self.duration} sec)...') 159 | sleep(self.duration) 160 | self._kill_nodes() 161 | 162 | # # Parse logs and return the parser. 163 | # Print.info('Parsing logs...') 164 | # return LogParser.process(PathMaker.logs_path(), faults=self.faults) 165 | 166 | except Exception as e: 167 | self._kill_nodes() 168 | raise BenchError('Failed to run benchmark', e) 169 | -------------------------------------------------------------------------------- /narwhal-abci/demo/benchmark/logs.py: -------------------------------------------------------------------------------- 1 | # Copyright(C) Facebook, Inc. and its affiliates. 2 | from datetime import datetime 3 | from glob import glob 4 | from multiprocessing import Pool 5 | from os.path import join 6 | from re import findall, search 7 | from statistics import mean 8 | 9 | from benchmark.utils import Print 10 | 11 | 12 | class ParseError(Exception): 13 | pass 14 | 15 | 16 | class LogParser: 17 | def __init__(self, clients, primaries, workers, faults=0): 18 | inputs = [clients, primaries, workers] 19 | assert all(isinstance(x, list) for x in inputs) 20 | assert all(isinstance(x, str) for y in inputs for x in y) 21 | assert all(x for x in inputs) 22 | 23 | self.faults = faults 24 | if isinstance(faults, int): 25 | self.committee_size = len(primaries) + int(faults) 26 | self.workers = len(workers) // len(primaries) 27 | else: 28 | self.committee_size = '?' 29 | self.workers = '?' 30 | 31 | # Parse the clients logs. 32 | try: 33 | with Pool() as p: 34 | results = p.map(self._parse_clients, clients) 35 | except (ValueError, IndexError, AttributeError) as e: 36 | raise ParseError(f'Failed to parse clients\' logs: {e}') 37 | self.size, self.rate, self.start, misses, self.sent_samples \ 38 | = zip(*results) 39 | self.misses = sum(misses) 40 | 41 | # Parse the primaries logs. 42 | try: 43 | with Pool() as p: 44 | results = p.map(self._parse_primaries, primaries) 45 | except (ValueError, IndexError, AttributeError) as e: 46 | raise ParseError(f'Failed to parse nodes\' logs: {e}') 47 | proposals, commits, self.configs, primary_ips = zip(*results) 48 | self.proposals = self._merge_results([x.items() for x in proposals]) 49 | self.commits = self._merge_results([x.items() for x in commits]) 50 | 51 | # Parse the workers logs. 52 | try: 53 | with Pool() as p: 54 | results = p.map(self._parse_workers, workers) 55 | except (ValueError, IndexError, AttributeError) as e: 56 | raise ParseError(f'Failed to parse workers\' logs: {e}') 57 | sizes, self.received_samples, workers_ips = zip(*results) 58 | self.sizes = { 59 | k: v for x in sizes for k, v in x.items() if k in self.commits 60 | } 61 | 62 | # Determine whether the primary and the workers are collocated. 63 | self.collocate = set(primary_ips) == set(workers_ips) 64 | 65 | # Check whether clients missed their target rate. 66 | if self.misses != 0: 67 | Print.warn( 68 | f'Clients missed their target rate {self.misses:,} time(s)' 69 | ) 70 | 71 | def _merge_results(self, input): 72 | # Keep the earliest timestamp. 73 | merged = {} 74 | for x in input: 75 | for k, v in x: 76 | if not k in merged or merged[k] > v: 77 | merged[k] = v 78 | return merged 79 | 80 | def _parse_clients(self, log): 81 | if search(r'Error', log) is not None: 82 | raise ParseError('Client(s) panicked') 83 | 84 | size = int(search(r'Transactions size: (\d+)', log).group(1)) 85 | rate = int(search(r'Transactions rate: (\d+)', log).group(1)) 86 | 87 | tmp = search(r'\[(.*Z) .* Start ', log).group(1) 88 | start = self._to_posix(tmp) 89 | 90 | misses = len(findall(r'rate too high', log)) 91 | 92 | tmp = findall(r'\[(.*Z) .* sample transaction (\d+)', log) 93 | samples = {int(s): self._to_posix(t) for t, s in tmp} 94 | 95 | return size, rate, start, misses, samples 96 | 97 | def _parse_primaries(self, log): 98 | if search(r'(?:panicked|Error)', log) is not None: 99 | raise ParseError('Primary(s) panicked') 100 | 101 | tmp = findall(r'\[(.*Z) .* Created B\d+\([^ ]+\) -> ([^ ]+=)', log) 102 | tmp = [(d, self._to_posix(t)) for t, d in tmp] 103 | proposals = self._merge_results([tmp]) 104 | 105 | tmp = findall(r'\[(.*Z) .* Committed B\d+\([^ ]+\) -> ([^ ]+=)', log) 106 | tmp = [(d, self._to_posix(t)) for t, d in tmp] 107 | commits = self._merge_results([tmp]) 108 | 109 | configs = { 110 | 'header_size': int( 111 | search(r'Header size .* (\d+)', log).group(1) 112 | ), 113 | 'max_header_delay': int( 114 | search(r'Max header delay .* (\d+)', log).group(1) 115 | ), 116 | 'gc_depth': int( 117 | search(r'Garbage collection depth .* (\d+)', log).group(1) 118 | ), 119 | 'sync_retry_delay': int( 120 | search(r'Sync retry delay .* (\d+)', log).group(1) 121 | ), 122 | 'sync_retry_nodes': int( 123 | search(r'Sync retry nodes .* (\d+)', log).group(1) 124 | ), 125 | 'batch_size': int( 126 | search(r'Batch size .* (\d+)', log).group(1) 127 | ), 128 | 'max_batch_delay': int( 129 | search(r'Max batch delay .* (\d+)', log).group(1) 130 | ), 131 | } 132 | 133 | ip = search(r'booted on (\d+.\d+.\d+.\d+)', log).group(1) 134 | 135 | return proposals, commits, configs, ip 136 | 137 | def _parse_workers(self, log): 138 | if search(r'(?:panic|Error)', log) is not None: 139 | raise ParseError('Worker(s) panicked') 140 | 141 | tmp = findall(r'Batch ([^ ]+) contains (\d+) B', log) 142 | sizes = {d: int(s) for d, s in tmp} 143 | 144 | tmp = findall(r'Batch ([^ ]+) contains sample tx (\d+)', log) 145 | samples = {int(s): d for d, s in tmp} 146 | 147 | ip = search(r'booted on (\d+.\d+.\d+.\d+)', log).group(1) 148 | 149 | return sizes, samples, ip 150 | 151 | def _to_posix(self, string): 152 | x = datetime.fromisoformat(string.replace('Z', '+00:00')) 153 | return datetime.timestamp(x) 154 | 155 | def _consensus_throughput(self): 156 | if not self.commits: 157 | return 0, 0, 0 158 | start, end = min(self.proposals.values()), max(self.commits.values()) 159 | duration = end - start 160 | bytes = sum(self.sizes.values()) 161 | bps = bytes / duration 162 | tps = bps / self.size[0] 163 | return tps, bps, duration 164 | 165 | def _consensus_latency(self): 166 | latency = [c - self.proposals[d] for d, c in self.commits.items()] 167 | return mean(latency) if latency else 0 168 | 169 | def _end_to_end_throughput(self): 170 | if not self.commits: 171 | return 0, 0, 0 172 | start, end = min(self.start), max(self.commits.values()) 173 | duration = end - start 174 | bytes = sum(self.sizes.values()) 175 | bps = bytes / duration 176 | tps = bps / self.size[0] 177 | return tps, bps, duration 178 | 179 | def _end_to_end_latency(self): 180 | latency = [] 181 | for sent, received in zip(self.sent_samples, self.received_samples): 182 | for tx_id, batch_id in received.items(): 183 | if batch_id in self.commits: 184 | assert tx_id in sent # We receive txs that we sent. 185 | start = sent[tx_id] 186 | end = self.commits[batch_id] 187 | latency += [end-start] 188 | return mean(latency) if latency else 0 189 | 190 | def result(self): 191 | header_size = self.configs[0]['header_size'] 192 | max_header_delay = self.configs[0]['max_header_delay'] 193 | gc_depth = self.configs[0]['gc_depth'] 194 | sync_retry_delay = self.configs[0]['sync_retry_delay'] 195 | sync_retry_nodes = self.configs[0]['sync_retry_nodes'] 196 | batch_size = self.configs[0]['batch_size'] 197 | max_batch_delay = self.configs[0]['max_batch_delay'] 198 | 199 | consensus_latency = self._consensus_latency() * 1_000 200 | consensus_tps, consensus_bps, _ = self._consensus_throughput() 201 | end_to_end_tps, end_to_end_bps, duration = self._end_to_end_throughput() 202 | end_to_end_latency = self._end_to_end_latency() * 1_000 203 | 204 | return ( 205 | '\n' 206 | '-----------------------------------------\n' 207 | ' SUMMARY:\n' 208 | '-----------------------------------------\n' 209 | ' + CONFIG:\n' 210 | f' Faults: {self.faults} node(s)\n' 211 | f' Committee size: {self.committee_size} node(s)\n' 212 | f' Worker(s) per node: {self.workers} worker(s)\n' 213 | f' Collocate primary and workers: {self.collocate}\n' 214 | f' Input rate: {sum(self.rate):,} tx/s\n' 215 | f' Transaction size: {self.size[0]:,} B\n' 216 | f' Execution time: {round(duration):,} s\n' 217 | '\n' 218 | f' Header size: {header_size:,} B\n' 219 | f' Max header delay: {max_header_delay:,} ms\n' 220 | f' GC depth: {gc_depth:,} round(s)\n' 221 | f' Sync retry delay: {sync_retry_delay:,} ms\n' 222 | f' Sync retry nodes: {sync_retry_nodes:,} node(s)\n' 223 | f' batch size: {batch_size:,} B\n' 224 | f' Max batch delay: {max_batch_delay:,} ms\n' 225 | '\n' 226 | ' + RESULTS:\n' 227 | f' Consensus TPS: {round(consensus_tps):,} tx/s\n' 228 | f' Consensus BPS: {round(consensus_bps):,} B/s\n' 229 | f' Consensus latency: {round(consensus_latency):,} ms\n' 230 | '\n' 231 | f' End-to-end TPS: {round(end_to_end_tps):,} tx/s\n' 232 | f' End-to-end BPS: {round(end_to_end_bps):,} B/s\n' 233 | f' End-to-end latency: {round(end_to_end_latency):,} ms\n' 234 | '-----------------------------------------\n' 235 | ) 236 | 237 | def print(self, filename): 238 | assert isinstance(filename, str) 239 | with open(filename, 'a') as f: 240 | f.write(self.result()) 241 | 242 | @classmethod 243 | def process(cls, directory, faults=0): 244 | assert isinstance(directory, str) 245 | 246 | clients = [] 247 | for filename in sorted(glob(join(directory, 'client-*.log'))): 248 | with open(filename, 'r') as f: 249 | clients += [f.read()] 250 | primaries = [] 251 | for filename in sorted(glob(join(directory, 'primary-*.log'))): 252 | with open(filename, 'r') as f: 253 | primaries += [f.read()] 254 | workers = [] 255 | for filename in sorted(glob(join(directory, 'worker-*.log'))): 256 | with open(filename, 'r') as f: 257 | workers += [f.read()] 258 | 259 | return cls(clients, primaries, workers, faults=faults) 260 | -------------------------------------------------------------------------------- /narwhal-abci/demo/benchmark/plot.py: -------------------------------------------------------------------------------- 1 | # Copyright(C) Facebook, Inc. and its affiliates. 2 | from collections import defaultdict 3 | from re import findall, search, split 4 | import matplotlib.pyplot as plt 5 | import matplotlib.ticker as tick 6 | from glob import glob 7 | from itertools import cycle 8 | 9 | from benchmark.utils import PathMaker 10 | from benchmark.config import PlotParameters 11 | from benchmark.aggregate import LogAggregator 12 | 13 | 14 | @tick.FuncFormatter 15 | def default_major_formatter(x, pos): 16 | if pos is None: 17 | return 18 | if x >= 1_000: 19 | return f'{x/1000:.0f}k' 20 | else: 21 | return f'{x:.0f}' 22 | 23 | 24 | @tick.FuncFormatter 25 | def sec_major_formatter(x, pos): 26 | if pos is None: 27 | return 28 | return f'{float(x)/1000:.1f}' 29 | 30 | 31 | @tick.FuncFormatter 32 | def mb_major_formatter(x, pos): 33 | if pos is None: 34 | return 35 | return f'{x:,.0f}' 36 | 37 | 38 | class PlotError(Exception): 39 | pass 40 | 41 | 42 | class Ploter: 43 | def __init__(self, filenames): 44 | if not filenames: 45 | raise PlotError('No data to plot') 46 | 47 | self.results = [] 48 | try: 49 | for filename in filenames: 50 | with open(filename, 'r') as f: 51 | self.results += [f.read().replace(',', '')] 52 | except OSError as e: 53 | raise PlotError(f'Failed to load log files: {e}') 54 | 55 | def _natural_keys(self, text): 56 | def try_cast(text): return int(text) if text.isdigit() else text 57 | return [try_cast(c) for c in split('(\d+)', text)] 58 | 59 | def _tps(self, data): 60 | values = findall(r' TPS: (\d+) \+/- (\d+)', data) 61 | values = [(int(x), int(y)) for x, y in values] 62 | return list(zip(*values)) 63 | 64 | def _latency(self, data, scale=1): 65 | values = findall(r' Latency: (\d+) \+/- (\d+)', data) 66 | values = [(float(x)/scale, float(y)/scale) for x, y in values] 67 | return list(zip(*values)) 68 | 69 | def _variable(self, data): 70 | return [int(x) for x in findall(r'Variable value: X=(\d+)', data)] 71 | 72 | def _tps2bps(self, x): 73 | data = self.results[0] 74 | size = int(search(r'Transaction size: (\d+)', data).group(1)) 75 | return x * size / 10**6 76 | 77 | def _bps2tps(self, x): 78 | data = self.results[0] 79 | size = int(search(r'Transaction size: (\d+)', data).group(1)) 80 | return x * 10**6 / size 81 | 82 | def _plot(self, x_label, y_label, y_axis, z_axis, type): 83 | plt.figure() 84 | markers = cycle(['o', 'v', 's', 'p', 'D', 'P']) 85 | self.results.sort(key=self._natural_keys, reverse=(type == 'tps')) 86 | for result in self.results: 87 | y_values, y_err = y_axis(result) 88 | x_values = self._variable(result) 89 | if len(y_values) != len(y_err) or len(y_err) != len(x_values): 90 | raise PlotError('Unequal number of x, y, and y_err values') 91 | 92 | plt.errorbar( 93 | x_values, y_values, yerr=y_err, label=z_axis(result), 94 | linestyle='dotted', marker=next(markers), capsize=3 95 | ) 96 | 97 | plt.legend(loc='lower center', bbox_to_anchor=(0.5, 1), ncol=3) 98 | plt.xlim(xmin=0) 99 | plt.ylim(bottom=0) 100 | plt.xlabel(x_label, fontweight='bold') 101 | plt.ylabel(y_label[0], fontweight='bold') 102 | plt.xticks(weight='bold') 103 | plt.yticks(weight='bold') 104 | plt.grid() 105 | ax = plt.gca() 106 | ax.xaxis.set_major_formatter(default_major_formatter) 107 | ax.yaxis.set_major_formatter(default_major_formatter) 108 | if 'latency' in type: 109 | ax.yaxis.set_major_formatter(sec_major_formatter) 110 | if len(y_label) > 1: 111 | secaxy = ax.secondary_yaxis( 112 | 'right', functions=(self._tps2bps, self._bps2tps) 113 | ) 114 | secaxy.set_ylabel(y_label[1]) 115 | secaxy.yaxis.set_major_formatter(mb_major_formatter) 116 | 117 | for x in ['pdf', 'png']: 118 | plt.savefig(PathMaker.plot_file(type, x), bbox_inches='tight') 119 | 120 | @staticmethod 121 | def nodes(data): 122 | x = search(r'Committee size: (\d+)', data).group(1) 123 | f = search(r'Faults: (\d+)', data).group(1) 124 | faults = f'({f} faulty)' if f != '0' else '' 125 | return f'{x} nodes {faults}' 126 | 127 | @staticmethod 128 | def workers(data): 129 | x = search(r'Workers per node: (\d+)', data).group(1) 130 | f = search(r'Faults: (\d+)', data).group(1) 131 | faults = f'({f} faulty)' if f != '0' else '' 132 | return f'{x} workers {faults}' 133 | 134 | @staticmethod 135 | def max_latency(data): 136 | x = search(r'Max latency: (\d+)', data).group(1) 137 | f = search(r'Faults: (\d+)', data).group(1) 138 | faults = f'({f} faulty)' if f != '0' else '' 139 | return f'Max latency: {float(x) / 1000:,.1f} s {faults}' 140 | 141 | @classmethod 142 | def plot_latency(cls, files, scalability): 143 | assert isinstance(files, list) 144 | assert all(isinstance(x, str) for x in files) 145 | z_axis = cls.workers if scalability else cls.nodes 146 | x_label = 'Throughput (tx/s)' 147 | y_label = ['Latency (s)'] 148 | ploter = cls(files) 149 | ploter._plot(x_label, y_label, ploter._latency, z_axis, 'latency') 150 | 151 | @classmethod 152 | def plot_tps(cls, files, scalability): 153 | assert isinstance(files, list) 154 | assert all(isinstance(x, str) for x in files) 155 | z_axis = cls.max_latency 156 | x_label = 'Workers per node' if scalability else 'Committee size' 157 | y_label = ['Throughput (tx/s)', 'Throughput (MB/s)'] 158 | ploter = cls(files) 159 | ploter._plot(x_label, y_label, ploter._tps, z_axis, 'tps') 160 | 161 | @classmethod 162 | def plot(cls, params_dict): 163 | try: 164 | params = PlotParameters(params_dict) 165 | except PlotError as e: 166 | raise PlotError('Invalid nodes or bench parameters', e) 167 | 168 | # Aggregate the logs. 169 | LogAggregator(params.max_latency).print() 170 | 171 | # Make the latency, tps, and robustness graphs. 172 | iterator = params.workers if params.scalability() else params.nodes 173 | latency_files, tps_files = [], [] 174 | for f in params.faults: 175 | for x in iterator: 176 | latency_files += glob( 177 | PathMaker.agg_file( 178 | 'latency', 179 | f, 180 | x if not params.scalability() else params.nodes[0], 181 | x if params.scalability() else params.workers[0], 182 | params.collocate, 183 | 'any', 184 | params.tx_size, 185 | ) 186 | ) 187 | 188 | for l in params.max_latency: 189 | tps_files += glob( 190 | PathMaker.agg_file( 191 | 'tps', 192 | f, 193 | 'x' if not params.scalability() else params.nodes[0], 194 | 'x' if params.scalability() else params.workers[0], 195 | params.collocate, 196 | 'any', 197 | params.tx_size, 198 | max_latency=l 199 | ) 200 | ) 201 | 202 | cls.plot_latency(latency_files, params.scalability()) 203 | cls.plot_tps(tps_files, params.scalability()) 204 | -------------------------------------------------------------------------------- /narwhal-abci/demo/benchmark/remote.py: -------------------------------------------------------------------------------- 1 | # Copyright(C) Facebook, Inc. and its affiliates. 2 | from collections import OrderedDict 3 | from fabric import Connection, ThreadingGroup as Group 4 | from fabric.exceptions import GroupException 5 | from paramiko import RSAKey 6 | from paramiko.ssh_exception import PasswordRequiredException, SSHException 7 | from os.path import basename, splitext 8 | from time import sleep 9 | from math import ceil 10 | from copy import deepcopy 11 | import subprocess 12 | 13 | from benchmark.config import Committee, Key, NodeParameters, BenchParameters, ConfigError 14 | from benchmark.utils import BenchError, Print, PathMaker, progress_bar 15 | from benchmark.commands import CommandMaker 16 | from benchmark.logs import LogParser, ParseError 17 | from benchmark.instance import InstanceManager 18 | 19 | 20 | class FabricError(Exception): 21 | ''' Wrapper for Fabric exception with a meaningfull error message. ''' 22 | 23 | def __init__(self, error): 24 | assert isinstance(error, GroupException) 25 | message = list(error.result.values())[-1] 26 | super().__init__(message) 27 | 28 | 29 | class ExecutionError(Exception): 30 | pass 31 | 32 | 33 | class Bench: 34 | def __init__(self, ctx): 35 | self.manager = InstanceManager.make() 36 | self.settings = self.manager.settings 37 | try: 38 | ctx.connect_kwargs.pkey = RSAKey.from_private_key_file( 39 | self.manager.settings.key_path 40 | ) 41 | self.connect = ctx.connect_kwargs 42 | except (IOError, PasswordRequiredException, SSHException) as e: 43 | raise BenchError('Failed to load SSH key', e) 44 | 45 | def _check_stderr(self, output): 46 | if isinstance(output, dict): 47 | for x in output.values(): 48 | if x.stderr: 49 | raise ExecutionError(x.stderr) 50 | else: 51 | if output.stderr: 52 | raise ExecutionError(output.stderr) 53 | 54 | def install(self): 55 | Print.info('Installing rust and cloning the repo...') 56 | cmd = [ 57 | 'sudo apt-get update', 58 | 'sudo apt-get -y upgrade', 59 | 'sudo apt-get -y autoremove', 60 | 61 | # The following dependencies prevent the error: [error: linker `cc` not found]. 62 | 'sudo apt-get -y install build-essential', 63 | 'sudo apt-get -y install cmake', 64 | 65 | # Install rust (non-interactive). 66 | 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y', 67 | 'source $HOME/.cargo/env', 68 | 'rustup default stable', 69 | 70 | # This is missing from the Rocksdb installer (needed for Rocksdb). 71 | 'sudo apt-get install -y clang', 72 | 73 | # Clone the repo. 74 | f'(git clone {self.settings.repo_url} || (cd {self.settings.repo_name} ; git pull))' 75 | ] 76 | hosts = self.manager.hosts(flat=True) 77 | try: 78 | g = Group(*hosts, user='ubuntu', connect_kwargs=self.connect) 79 | g.run(' && '.join(cmd), hide=True) 80 | Print.heading(f'Initialized testbed of {len(hosts)} nodes') 81 | except (GroupException, ExecutionError) as e: 82 | e = FabricError(e) if isinstance(e, GroupException) else e 83 | raise BenchError('Failed to install repo on testbed', e) 84 | 85 | def kill(self, hosts=[], delete_logs=False): 86 | assert isinstance(hosts, list) 87 | assert isinstance(delete_logs, bool) 88 | hosts = hosts if hosts else self.manager.hosts(flat=True) 89 | delete_logs = CommandMaker.clean_logs() if delete_logs else 'true' 90 | cmd = [delete_logs, f'({CommandMaker.kill()} || true)'] 91 | try: 92 | g = Group(*hosts, user='ubuntu', connect_kwargs=self.connect) 93 | g.run(' && '.join(cmd), hide=True) 94 | except GroupException as e: 95 | raise BenchError('Failed to kill nodes', FabricError(e)) 96 | 97 | def _select_hosts(self, bench_parameters): 98 | # Collocate the primary and its workers on the same machine. 99 | if bench_parameters.collocate: 100 | nodes = max(bench_parameters.nodes) 101 | 102 | # Ensure there are enough hosts. 103 | hosts = self.manager.hosts() 104 | if sum(len(x) for x in hosts.values()) < nodes: 105 | return [] 106 | 107 | # Select the hosts in different data centers. 108 | ordered = zip(*hosts.values()) 109 | ordered = [x for y in ordered for x in y] 110 | return ordered[:nodes] 111 | 112 | # Spawn the primary and each worker on a different machine. Each 113 | # authority runs in a single data center. 114 | else: 115 | primaries = max(bench_parameters.nodes) 116 | 117 | # Ensure there are enough hosts. 118 | hosts = self.manager.hosts() 119 | if len(hosts.keys()) < primaries: 120 | return [] 121 | for ips in hosts.values(): 122 | if len(ips) < bench_parameters.workers + 1: 123 | return [] 124 | 125 | # Ensure the primary and its workers are in the same region. 126 | selected = [] 127 | for region in list(hosts.keys())[:primaries]: 128 | ips = list(hosts[region])[:bench_parameters.workers + 1] 129 | selected.append(ips) 130 | return selected 131 | 132 | def _background_run(self, host, command, log_file): 133 | name = splitext(basename(log_file))[0] 134 | cmd = f'tmux new -d -s "{name}" "{command} |& tee {log_file}"' 135 | c = Connection(host, user='ubuntu', connect_kwargs=self.connect) 136 | output = c.run(cmd, hide=True) 137 | self._check_stderr(output) 138 | 139 | def _update(self, hosts, collocate): 140 | if collocate: 141 | ips = list(set(hosts)) 142 | else: 143 | ips = list(set([x for y in hosts for x in y])) 144 | 145 | Print.info( 146 | f'Updating {len(ips)} machines (branch "{self.settings.branch}")...' 147 | ) 148 | cmd = [ 149 | f'(cd {self.settings.repo_name} && git fetch -f)', 150 | f'(cd {self.settings.repo_name} && git checkout -f {self.settings.branch})', 151 | f'(cd {self.settings.repo_name} && git pull -f)', 152 | 'source $HOME/.cargo/env', 153 | f'(cd {self.settings.repo_name}/node && {CommandMaker.compile()})', 154 | CommandMaker.alias_binaries( 155 | f'./{self.settings.repo_name}/target/release/' 156 | ) 157 | ] 158 | g = Group(*ips, user='ubuntu', connect_kwargs=self.connect) 159 | g.run(' && '.join(cmd), hide=True) 160 | 161 | def _config(self, hosts, node_parameters, bench_parameters): 162 | Print.info('Generating configuration files...') 163 | 164 | # Cleanup all local configuration files. 165 | cmd = CommandMaker.cleanup() 166 | subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL) 167 | 168 | # Recompile the latest code. 169 | cmd = CommandMaker.compile().split() 170 | subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path()) 171 | 172 | # Create alias for the client and nodes binary. 173 | cmd = CommandMaker.alias_binaries(PathMaker.binary_path()) 174 | subprocess.run([cmd], shell=True) 175 | 176 | # Generate configuration files. 177 | keys = [] 178 | key_files = [PathMaker.key_file(i) for i in range(len(hosts))] 179 | for filename in key_files: 180 | cmd = CommandMaker.generate_key(filename).split() 181 | subprocess.run(cmd, check=True) 182 | keys += [Key.from_file(filename)] 183 | 184 | names = [x.name for x in keys] 185 | 186 | if bench_parameters.collocate: 187 | workers = bench_parameters.workers 188 | addresses = OrderedDict( 189 | (x, [y] * (workers + 1)) for x, y in zip(names, hosts) 190 | ) 191 | else: 192 | addresses = OrderedDict( 193 | (x, y) for x, y in zip(names, hosts) 194 | ) 195 | committee = Committee(addresses, self.settings.base_port) 196 | committee.print(PathMaker.committee_file()) 197 | 198 | node_parameters.print(PathMaker.parameters_file()) 199 | 200 | # Cleanup all nodes and upload configuration files. 201 | names = names[:len(names)-bench_parameters.faults] 202 | progress = progress_bar(names, prefix='Uploading config files:') 203 | for i, name in enumerate(progress): 204 | for ip in committee.ips(name): 205 | c = Connection(ip, user='ubuntu', connect_kwargs=self.connect) 206 | c.run(f'{CommandMaker.cleanup()} || true', hide=True) 207 | c.put(PathMaker.committee_file(), '.') 208 | c.put(PathMaker.key_file(i), '.') 209 | c.put(PathMaker.parameters_file(), '.') 210 | 211 | return committee 212 | 213 | def _run_single(self, rate, committee, bench_parameters, debug=False): 214 | faults = bench_parameters.faults 215 | 216 | # Kill any potentially unfinished run and delete logs. 217 | hosts = committee.ips() 218 | self.kill(hosts=hosts, delete_logs=True) 219 | 220 | # Run the clients (they will wait for the nodes to be ready). 221 | # Filter all faulty nodes from the client addresses (or they will wait 222 | # for the faulty nodes to be online). 223 | Print.info('Booting clients...') 224 | workers_addresses = committee.workers_addresses(faults) 225 | rate_share = ceil(rate / committee.workers()) 226 | for i, addresses in enumerate(workers_addresses): 227 | for (id, address) in addresses: 228 | host = Committee.ip(address) 229 | cmd = CommandMaker.run_client( 230 | address, 231 | bench_parameters.tx_size, 232 | rate_share, 233 | [x for y in workers_addresses for _, x in y] 234 | ) 235 | log_file = PathMaker.client_log_file(i, id) 236 | self._background_run(host, cmd, log_file) 237 | 238 | # Run the primaries (except the faulty ones). 239 | Print.info('Booting primaries...') 240 | for i, address in enumerate(committee.primary_addresses(faults)): 241 | host = Committee.ip(address) 242 | cmd = CommandMaker.run_primary( 243 | PathMaker.key_file(i), 244 | PathMaker.committee_file(), 245 | PathMaker.db_path(i), 246 | PathMaker.parameters_file(), 247 | debug=debug 248 | ) 249 | log_file = PathMaker.primary_log_file(i) 250 | self._background_run(host, cmd, log_file) 251 | 252 | # Run the workers (except the faulty ones). 253 | Print.info('Booting workers...') 254 | for i, addresses in enumerate(workers_addresses): 255 | for (id, address) in addresses: 256 | host = Committee.ip(address) 257 | cmd = CommandMaker.run_worker( 258 | PathMaker.key_file(i), 259 | PathMaker.committee_file(), 260 | PathMaker.db_path(i, id), 261 | PathMaker.parameters_file(), 262 | id, # The worker's id. 263 | debug=debug 264 | ) 265 | log_file = PathMaker.worker_log_file(i, id) 266 | self._background_run(host, cmd, log_file) 267 | 268 | # Wait for all transactions to be processed. 269 | duration = bench_parameters.duration 270 | for _ in progress_bar(range(20), prefix=f'Running benchmark ({duration} sec):'): 271 | sleep(ceil(duration / 20)) 272 | self.kill(hosts=hosts, delete_logs=False) 273 | 274 | def _logs(self, committee, faults): 275 | # Delete local logs (if any). 276 | cmd = CommandMaker.clean_logs() 277 | subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL) 278 | 279 | # Download log files. 280 | workers_addresses = committee.workers_addresses(faults) 281 | progress = progress_bar(workers_addresses, prefix='Downloading workers logs:') 282 | for i, addresses in enumerate(progress): 283 | for id, address in addresses: 284 | host = Committee.ip(address) 285 | c = Connection(host, user='ubuntu', connect_kwargs=self.connect) 286 | c.get( 287 | PathMaker.client_log_file(i, id), 288 | local=PathMaker.client_log_file(i, id) 289 | ) 290 | c.get( 291 | PathMaker.worker_log_file(i, id), 292 | local=PathMaker.worker_log_file(i, id) 293 | ) 294 | 295 | primary_addresses = committee.primary_addresses(faults) 296 | progress = progress_bar(primary_addresses, prefix='Downloading primaries logs:') 297 | for i, address in enumerate(progress): 298 | host = Committee.ip(address) 299 | c = Connection(host, user='ubuntu', connect_kwargs=self.connect) 300 | c.get( 301 | PathMaker.primary_log_file(i), 302 | local=PathMaker.primary_log_file(i) 303 | ) 304 | 305 | # Parse logs and return the parser. 306 | Print.info('Parsing logs and computing performance...') 307 | return LogParser.process(PathMaker.logs_path(), faults=faults) 308 | 309 | def run(self, bench_parameters_dict, node_parameters_dict, debug=False): 310 | assert isinstance(debug, bool) 311 | Print.heading('Starting remote benchmark') 312 | try: 313 | bench_parameters = BenchParameters(bench_parameters_dict) 314 | node_parameters = NodeParameters(node_parameters_dict) 315 | except ConfigError as e: 316 | raise BenchError('Invalid nodes or bench parameters', e) 317 | 318 | # Select which hosts to use. 319 | selected_hosts = self._select_hosts(bench_parameters) 320 | if not selected_hosts: 321 | Print.warn('There are not enough instances available') 322 | return 323 | 324 | # Update nodes. 325 | try: 326 | self._update(selected_hosts, bench_parameters.collocate) 327 | except (GroupException, ExecutionError) as e: 328 | e = FabricError(e) if isinstance(e, GroupException) else e 329 | raise BenchError('Failed to update nodes', e) 330 | 331 | # Upload all configuration files. 332 | try: 333 | committee = self._config( 334 | selected_hosts, node_parameters, bench_parameters 335 | ) 336 | except (subprocess.SubprocessError, GroupException) as e: 337 | e = FabricError(e) if isinstance(e, GroupException) else e 338 | raise BenchError('Failed to configure nodes', e) 339 | 340 | # Run benchmarks. 341 | for n in bench_parameters.nodes: 342 | committee_copy = deepcopy(committee) 343 | committee_copy.remove_nodes(committee.size() - n) 344 | 345 | for r in bench_parameters.rate: 346 | Print.heading(f'\nRunning {n} nodes (input rate: {r:,} tx/s)') 347 | 348 | # Run the benchmark. 349 | for i in range(bench_parameters.runs): 350 | Print.heading(f'Run {i+1}/{bench_parameters.runs}') 351 | try: 352 | self._run_single( 353 | r, committee_copy, bench_parameters, debug 354 | ) 355 | 356 | faults = bench_parameters.faults 357 | logger = self._logs(committee_copy, faults) 358 | logger.print(PathMaker.result_file( 359 | faults, 360 | n, 361 | bench_parameters.workers, 362 | bench_parameters.collocate, 363 | r, 364 | bench_parameters.tx_size, 365 | )) 366 | except (subprocess.SubprocessError, GroupException, ParseError) as e: 367 | self.kill(hosts=selected_hosts) 368 | if isinstance(e, GroupException): 369 | e = FabricError(e) 370 | Print.error(BenchError('Benchmark failed', e)) 371 | continue 372 | -------------------------------------------------------------------------------- /narwhal-abci/demo/benchmark/settings.py: -------------------------------------------------------------------------------- 1 | # Copyright(C) Facebook, Inc. and its affiliates. 2 | from json import load, JSONDecodeError 3 | 4 | 5 | class SettingsError(Exception): 6 | pass 7 | 8 | 9 | class Settings: 10 | def __init__(self, key_name, key_path, base_port, repo_name, repo_url, 11 | branch, instance_type, aws_regions): 12 | inputs_str = [ 13 | key_name, key_path, repo_name, repo_url, branch, instance_type 14 | ] 15 | if isinstance(aws_regions, list): 16 | regions = aws_regions 17 | else: 18 | regions = [aws_regions] 19 | inputs_str += regions 20 | ok = all(isinstance(x, str) for x in inputs_str) 21 | ok &= isinstance(base_port, int) 22 | ok &= len(regions) > 0 23 | if not ok: 24 | raise SettingsError('Invalid settings types') 25 | 26 | self.key_name = key_name 27 | self.key_path = key_path 28 | 29 | self.base_port = base_port 30 | 31 | self.repo_name = repo_name 32 | self.repo_url = repo_url 33 | self.branch = branch 34 | 35 | self.instance_type = instance_type 36 | self.aws_regions = regions 37 | 38 | @classmethod 39 | def load(cls, filename): 40 | try: 41 | with open(filename, 'r') as f: 42 | data = load(f) 43 | 44 | return cls( 45 | data['key']['name'], 46 | data['key']['path'], 47 | data['port'], 48 | data['repo']['name'], 49 | data['repo']['url'], 50 | data['repo']['branch'], 51 | data['instances']['type'], 52 | data['instances']['regions'], 53 | ) 54 | except (OSError, JSONDecodeError) as e: 55 | raise SettingsError(str(e)) 56 | 57 | except KeyError as e: 58 | raise SettingsError(f'Malformed settings: missing key {e}') 59 | -------------------------------------------------------------------------------- /narwhal-abci/demo/benchmark/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright(C) Facebook, Inc. and its affiliates. 2 | from os.path import join 3 | 4 | 5 | class BenchError(Exception): 6 | def __init__(self, message, error): 7 | assert isinstance(error, Exception) 8 | self.message = message 9 | self.cause = error 10 | super().__init__(message) 11 | 12 | 13 | class PathMaker: 14 | @staticmethod 15 | def binary_path(): 16 | return join('..', 'target', 'release') 17 | 18 | @staticmethod 19 | def node_crate_path(): 20 | return join('..', 'node') 21 | 22 | @staticmethod 23 | def committee_file(): 24 | return '.committee.json' 25 | 26 | @staticmethod 27 | def parameters_file(): 28 | return '.parameters.json' 29 | 30 | @staticmethod 31 | def key_file(i): 32 | assert isinstance(i, int) and i >= 0 33 | return f'.node-{i}.json' 34 | 35 | @staticmethod 36 | def db_path(i, j=None): 37 | assert isinstance(i, int) and i >= 0 38 | assert (isinstance(j, int) and i >= 0) or j is None 39 | worker_id = f'-{j}' if j is not None else '' 40 | return f'.db-{i}{worker_id}' 41 | 42 | @staticmethod 43 | def logs_path(): 44 | return 'logs' 45 | 46 | @staticmethod 47 | def primary_log_file(i): 48 | assert isinstance(i, int) and i >= 0 49 | return join(PathMaker.logs_path(), f'primary-{i}.log') 50 | 51 | @staticmethod 52 | def app_log_file(i): 53 | assert isinstance(i, int) and i >= 0 54 | return join(PathMaker.logs_path(), f'app-{i}.log') 55 | 56 | @staticmethod 57 | def worker_log_file(i, j): 58 | assert isinstance(i, int) and i >= 0 59 | assert isinstance(j, int) and i >= 0 60 | return join(PathMaker.logs_path(), f'worker-{i}-{j}.log') 61 | 62 | @staticmethod 63 | def client_log_file(i, j): 64 | assert isinstance(i, int) and i >= 0 65 | assert isinstance(j, int) and i >= 0 66 | return join(PathMaker.logs_path(), f'client-{i}-{j}.log') 67 | 68 | @staticmethod 69 | def results_path(): 70 | return 'results' 71 | 72 | @staticmethod 73 | def result_file(faults, nodes, workers, collocate, rate, tx_size): 74 | return join( 75 | PathMaker.results_path(), 76 | f'bench-{faults}-{nodes}-{workers}-{collocate}-{rate}-{tx_size}.txt' 77 | ) 78 | 79 | @staticmethod 80 | def plots_path(): 81 | return 'plots' 82 | 83 | @staticmethod 84 | def agg_file(type, faults, nodes, workers, collocate, rate, tx_size, max_latency=None): 85 | if max_latency is None: 86 | name = f'{type}-bench-{faults}-{nodes}-{workers}-{collocate}-{rate}-{tx_size}.txt' 87 | else: 88 | name = f'{type}-{max_latency}-bench-{faults}-{nodes}-{workers}-{collocate}-{rate}-{tx_size}.txt' 89 | return join(PathMaker.plots_path(), name) 90 | 91 | @staticmethod 92 | def plot_file(name, ext): 93 | return join(PathMaker.plots_path(), f'{name}.{ext}') 94 | 95 | 96 | class Color: 97 | HEADER = '\033[95m' 98 | OK_BLUE = '\033[94m' 99 | OK_GREEN = '\033[92m' 100 | WARNING = '\033[93m' 101 | FAIL = '\033[91m' 102 | END = '\033[0m' 103 | BOLD = '\033[1m' 104 | UNDERLINE = '\033[4m' 105 | 106 | 107 | class Print: 108 | @staticmethod 109 | def heading(message): 110 | assert isinstance(message, str) 111 | print(f'{Color.OK_GREEN}{message}{Color.END}') 112 | 113 | @staticmethod 114 | def info(message): 115 | assert isinstance(message, str) 116 | print(message) 117 | 118 | @staticmethod 119 | def warn(message): 120 | assert isinstance(message, str) 121 | print(f'{Color.BOLD}{Color.WARNING}WARN{Color.END}: {message}') 122 | 123 | @staticmethod 124 | def error(e): 125 | assert isinstance(e, BenchError) 126 | print(f'\n{Color.BOLD}{Color.FAIL}ERROR{Color.END}: {e}\n') 127 | causes, current_cause = [], e.cause 128 | while isinstance(current_cause, BenchError): 129 | causes += [f' {len(causes)}: {e.cause}\n'] 130 | current_cause = current_cause.cause 131 | causes += [f' {len(causes)}: {type(current_cause)}\n'] 132 | causes += [f' {len(causes)}: {current_cause}\n'] 133 | print(f'Caused by: \n{"".join(causes)}\n') 134 | 135 | 136 | def progress_bar(iterable, prefix='', suffix='', decimals=1, length=30, fill='█', print_end='\r'): 137 | total = len(iterable) 138 | 139 | def printProgressBar(iteration): 140 | formatter = '{0:.'+str(decimals)+'f}' 141 | percent = formatter.format(100 * (iteration / float(total))) 142 | filledLength = int(length * iteration // total) 143 | bar = fill * filledLength + '-' * (length - filledLength) 144 | print(f'\r{prefix} |{bar}| {percent}% {suffix}', end=print_end) 145 | 146 | printProgressBar(0) 147 | for i, item in enumerate(iterable): 148 | yield item 149 | printProgressBar(i + 1) 150 | print() 151 | -------------------------------------------------------------------------------- /narwhal-abci/demo/cleanup-logs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ve 2 | 3 | cat logs/primary-0.log | grep -i LEDGER | awk '{ print $7; }' > logs/primary-0-parsed.log 4 | cat logs/primary-1.log | grep -i LEDGER | awk '{ print $7; }' > logs/primary-1-parsed.log 5 | cat logs/primary-2.log | grep -i LEDGER | awk '{ print $7; }' > logs/primary-2-parsed.log 6 | cat logs/primary-3.log | grep -i LEDGER | awk '{ print $7; }' > logs/primary-3-parsed.log 7 | -------------------------------------------------------------------------------- /narwhal-abci/demo/fabfile.py: -------------------------------------------------------------------------------- 1 | # Copyright(C) Facebook, Inc. and its affiliates. 2 | from fabric import task 3 | 4 | from benchmark.local import LocalBench 5 | from benchmark.logs import ParseError, LogParser 6 | from benchmark.utils import Print 7 | from benchmark.plot import Ploter, PlotError 8 | from benchmark.instance import InstanceManager 9 | from benchmark.remote import Bench, BenchError 10 | 11 | 12 | @task 13 | def local(ctx, debug=True): 14 | ''' Run benchmarks on localhost ''' 15 | bench_params = { 16 | 'faults': 0, 17 | 'nodes': 4, # only start 2 nodes 18 | 'workers': 1, 19 | 'rate': 0, 20 | 'tx_size': 512, 21 | 'duration': 5*60, 22 | } 23 | node_params = { 24 | 'header_size': 50, # bytes 25 | 'max_header_delay': 1_000, # ms 26 | 'gc_depth': 50, # rounds 27 | 'sync_retry_delay': 10_000, # ms 28 | 'sync_retry_nodes': 3, # number of nodes 29 | 'batch_size': 500_000, # bytes 30 | 'max_batch_delay': 200 # ms 31 | } 32 | 33 | bencher = LocalBench(bench_params, node_params) 34 | print(bencher) 35 | bencher.run() 36 | 37 | # try: 38 | # ret = LocalBench(bench_params, node_params).run(debug) 39 | # print(ret.result()) 40 | # except BenchError as e: 41 | # Print.error(e) 42 | 43 | 44 | @task 45 | def create(ctx, nodes=10): 46 | ''' Create a testbed''' 47 | try: 48 | InstanceManager.make().create_instances(nodes) 49 | except BenchError as e: 50 | Print.error(e) 51 | 52 | 53 | @task 54 | def destroy(ctx): 55 | ''' Destroy the testbed ''' 56 | try: 57 | InstanceManager.make().terminate_instances() 58 | except BenchError as e: 59 | Print.error(e) 60 | 61 | 62 | @task 63 | def start(ctx, max=2): 64 | ''' Start at most `max` machines per data center ''' 65 | try: 66 | InstanceManager.make().start_instances(max) 67 | except BenchError as e: 68 | Print.error(e) 69 | 70 | 71 | @task 72 | def stop(ctx): 73 | ''' Stop all machines ''' 74 | try: 75 | InstanceManager.make().stop_instances() 76 | except BenchError as e: 77 | Print.error(e) 78 | 79 | 80 | @task 81 | def info(ctx): 82 | ''' Display connect information about all the available machines ''' 83 | try: 84 | InstanceManager.make().print_info() 85 | except BenchError as e: 86 | Print.error(e) 87 | 88 | 89 | @task 90 | def install(ctx): 91 | ''' Install the codebase on all machines ''' 92 | try: 93 | Bench(ctx).install() 94 | except BenchError as e: 95 | Print.error(e) 96 | 97 | 98 | @task 99 | def remote(ctx, debug=False): 100 | ''' Run benchmarks on AWS ''' 101 | bench_params = { 102 | 'faults': 0, 103 | 'nodes': [10, 20], 104 | 'workers': 1, 105 | 'collocate': True, 106 | 'rate': [10_000, 50_000], 107 | 'tx_size': 512, 108 | 'duration': 300, 109 | 'runs': 2, 110 | } 111 | node_params = { 112 | 'header_size': 50, # bytes 113 | 'max_header_delay': 5_000, # ms 114 | 'gc_depth': 50, # rounds 115 | 'sync_retry_delay': 10_000, # ms 116 | 'sync_retry_nodes': 3, # number of nodes 117 | 'batch_size': 500_000, # bytes 118 | 'max_batch_delay': 200 # ms 119 | } 120 | try: 121 | Bench(ctx).run(bench_params, node_params, debug) 122 | except BenchError as e: 123 | Print.error(e) 124 | 125 | 126 | @task 127 | def plot(ctx): 128 | ''' Plot performance using the logs generated by "fab remote" ''' 129 | plot_params = { 130 | 'faults': [0], 131 | 'nodes': [10, 20, 50], 132 | 'workers': [1], 133 | 'collocate': True, 134 | 'tx_size': 512, 135 | 'max_latency': [2_500, 4_500] 136 | } 137 | try: 138 | Ploter.plot(plot_params) 139 | except PlotError as e: 140 | Print.error(BenchError('Failed to plot performance', e)) 141 | 142 | 143 | @task 144 | def kill(ctx): 145 | ''' Stop execution on all machines ''' 146 | try: 147 | Bench(ctx).kill() 148 | except BenchError as e: 149 | Print.error(e) 150 | 151 | 152 | @task 153 | def logs(ctx): 154 | ''' Print a summary of the logs ''' 155 | try: 156 | print(LogParser.process('./logs', faults='?').result()) 157 | except ParseError as e: 158 | Print.error(BenchError('Failed to parse logs', e)) 159 | -------------------------------------------------------------------------------- /narwhal-abci/demo/node_params.json: -------------------------------------------------------------------------------- 1 | { 2 | 'header_size': 50, # bytes 3 | 'max_header_delay': 5_000, # ms 4 | 'gc_depth': 50, # rounds 5 | 'sync_retry_delay': 10_000, # ms 6 | 'sync_retry_nodes': 3, # number of nodes 7 | 'batch_size': 500_000, # bytes 8 | 'max_batch_delay': 200 # ms 9 | } 10 | 11 | -------------------------------------------------------------------------------- /narwhal-abci/demo/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "benchmark" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Georgios Konstantopoulos "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.10" 9 | boto3 = "1.16.0" 10 | Fabric = "2.6.0" 11 | matplotlib = "3.3.4" 12 | 13 | [tool.poetry.dev-dependencies] 14 | 15 | [build-system] 16 | requires = ["poetry-core>=1.0.0"] 17 | build-backend = "poetry.core.masonry.api" 18 | -------------------------------------------------------------------------------- /narwhal-abci/narwhal-abci/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "narwhal-abci" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | tokio = { version = "1.19.2", features = ["full"] } 10 | tokio-util = { version = "0.6.2", features= ["codec"] } 11 | warp = "0.3.2" 12 | rocksdb = "0.16.0" 13 | serde = { version = "1.0", features = ["derive"] } 14 | log = "0.4.11" 15 | futures = "0.3.15" 16 | eyre = "0.6.8" 17 | 18 | bincode = "1.3.3" 19 | hex = "0.4.3" 20 | 21 | tendermint-abci = { version = "0.23.7", features = ["client"] } 22 | tendermint-proto = "0.23.7" 23 | 24 | narwhal_primary = { package = "primary", git = "https://github.com/asonnino/narwhal/" } 25 | narwhal_crypto = { package = "crypto", git = "https://github.com/asonnino/narwhal/" } 26 | serde_json = "1.0.82" 27 | -------------------------------------------------------------------------------- /narwhal-abci/narwhal-abci/src/abci_server.rs: -------------------------------------------------------------------------------- 1 | use crate::{AbciQueryQuery, BroadcastTxQuery}; 2 | 3 | use eyre::WrapErr; 4 | use futures::SinkExt; 5 | use tendermint_proto::abci::ResponseQuery; 6 | use tokio::sync::mpsc::Sender; 7 | use tokio::sync::oneshot::{channel as oneshot_channel, Sender as OneShotSender}; 8 | 9 | use tokio::net::TcpStream; 10 | use tokio_util::codec::{Framed, LengthDelimitedCodec}; 11 | use warp::{Filter, Rejection}; 12 | 13 | use std::net::SocketAddr; 14 | 15 | /// Simple HTTP API server which listens to messages on: 16 | /// * `broadcast_tx`: forwards them to Narwhal's mempool/worker socket, which will proceed to put 17 | /// it in the consensus process and eventually forward it to the application. 18 | /// * `abci_query`: forwards them over a channel to a handler (typically the application). 19 | pub struct AbciApi { 20 | mempool_address: SocketAddr, 21 | tx: Sender<(OneShotSender, AbciQueryQuery)>, 22 | } 23 | 24 | impl AbciApi { 25 | pub fn new( 26 | mempool_address: SocketAddr, 27 | tx: Sender<(OneShotSender, AbciQueryQuery)>, 28 | ) -> Self { 29 | Self { 30 | mempool_address, 31 | tx, 32 | } 33 | } 34 | } 35 | 36 | impl AbciApi { 37 | pub fn routes(self) -> impl Filter + Clone { 38 | let route_broadcast_tx = warp::path("broadcast_tx") 39 | .and(warp::query::()) 40 | .and_then(move |req: BroadcastTxQuery| async move { 41 | log::warn!("broadcast_tx: {:?}", req); 42 | 43 | let stream = TcpStream::connect(self.mempool_address) 44 | .await 45 | .wrap_err(format!( 46 | "ROUTE_BROADCAST_TX failed to connect to {}", 47 | self.mempool_address 48 | )) 49 | .unwrap(); 50 | let mut transport = Framed::new(stream, LengthDelimitedCodec::new()); 51 | 52 | if let Err(e) = transport.send(req.tx.clone().into()).await { 53 | Ok::<_, Rejection>(format!("ERROR IN: broadcast_tx: {:?}. Err: {}", req, e)) 54 | } else { 55 | Ok::<_, Rejection>(format!("broadcast_tx: {:?}", req)) 56 | } 57 | }); 58 | 59 | let route_abci_query = warp::path("abci_query") 60 | .and(warp::query::()) 61 | .and_then(move |req: AbciQueryQuery| { 62 | let tx_abci_queries = self.tx.clone(); 63 | async move { 64 | log::warn!("abci_query: {:?}", req); 65 | 66 | let (tx, rx) = oneshot_channel(); 67 | match tx_abci_queries.send((tx, req.clone())).await { 68 | Ok(_) => {} 69 | Err(err) => log::error!("Error forwarding abci query: {}", err), 70 | }; 71 | let resp = rx.await.unwrap(); 72 | // Return the value 73 | Ok::<_, Rejection>(resp.value) 74 | } 75 | }); 76 | 77 | route_broadcast_tx.or(route_abci_query) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /narwhal-abci/narwhal-abci/src/engine.rs: -------------------------------------------------------------------------------- 1 | use crate::AbciQueryQuery; 2 | use std::net::SocketAddr; 3 | use std::time::Instant; 4 | use tokio::sync::mpsc::Receiver; 5 | use tokio::sync::oneshot::Sender as OneShotSender; 6 | 7 | // Tendermint Types 8 | use tendermint_abci::{Client as AbciClient, ClientBuilder}; 9 | use tendermint_proto::abci::{ 10 | RequestBeginBlock, RequestDeliverTx, RequestEndBlock, RequestInfo, RequestInitChain, 11 | RequestQuery, ResponseQuery, 12 | }; 13 | use tendermint_proto::types::Header; 14 | 15 | // Narwhal types 16 | use narwhal_crypto::Digest; 17 | use narwhal_primary::Certificate; 18 | 19 | /// The engine drives the ABCI Application by concurrently polling for: 20 | /// 1. Calling the BeginBlock -> DeliverTx -> EndBlock -> Commit event loop on the ABCI App on each Bullshark 21 | /// certificate received. It will also call Info and InitChain to initialize the ABCI App if 22 | /// necessary. 23 | /// 2. Processing Query & Broadcast Tx messages received from the Primary's ABCI Server API and forwarding them to the 24 | /// ABCI App via a Tendermint protobuf client. 25 | pub struct Engine { 26 | /// The address of the ABCI app 27 | pub app_address: SocketAddr, 28 | /// The path to the Primary's store, so that the Engine can query each of the Primary's workers 29 | /// for the data corresponding to a Certificate 30 | pub store_path: String, 31 | /// Messages received from the ABCI Server to be forwarded to the engine. 32 | pub rx_abci_queries: Receiver<(OneShotSender, AbciQueryQuery)>, 33 | /// The last block height, initialized to the application's latest block by default 34 | pub last_block_height: i64, 35 | pub client: AbciClient, 36 | pub req_client: AbciClient, 37 | } 38 | 39 | impl Engine { 40 | pub fn new( 41 | app_address: SocketAddr, 42 | store_path: &str, 43 | rx_abci_queries: Receiver<(OneShotSender, AbciQueryQuery)>, 44 | ) -> Self { 45 | let mut client = ClientBuilder::default().connect(&app_address).unwrap(); 46 | 47 | let last_block_height = client 48 | .info(RequestInfo::default()) 49 | .map(|res| res.last_block_height) 50 | .unwrap_or_default(); 51 | 52 | // Instantiate a new client to not be locked in an Info connection 53 | let client = ClientBuilder::default().connect(&app_address).unwrap(); 54 | let req_client = ClientBuilder::default().connect(&app_address).unwrap(); 55 | Self { 56 | app_address, 57 | store_path: store_path.to_string(), 58 | rx_abci_queries, 59 | last_block_height, 60 | client, 61 | req_client, 62 | } 63 | } 64 | 65 | /// Receives an ordered list of certificates and apply any application-specific logic. 66 | pub async fn run(&mut self, mut rx_output: Receiver) -> eyre::Result<()> { 67 | self.init_chain()?; 68 | 69 | loop { 70 | tokio::select! { 71 | Some(certificate) = rx_output.recv() => { 72 | self.handle_cert(certificate)?; 73 | }, 74 | Some((tx, req)) = self.rx_abci_queries.recv() => { 75 | self.handle_abci_query(tx, req)?; 76 | } 77 | else => break, 78 | } 79 | } 80 | 81 | Ok(()) 82 | } 83 | 84 | /// On each new certificate, increment the block height to proposed and run through the 85 | /// BeginBlock -> DeliverTx for each tx in the certificate -> EndBlock -> Commit event loop. 86 | fn handle_cert(&mut self, certificate: Certificate) -> eyre::Result<()> { 87 | // increment block 88 | let proposed_block_height = self.last_block_height + 1; 89 | 90 | // save it for next time 91 | self.last_block_height = proposed_block_height; 92 | 93 | // drive the app through the event loop 94 | let now = Instant::now(); 95 | self.begin_block(proposed_block_height)?; 96 | self.reconstruct_and_deliver_txs(certificate)?; 97 | self.end_block(proposed_block_height)?; 98 | self.commit()?; 99 | log::info!("time in block: {} ms", now.elapsed().as_millis()); 100 | 101 | Ok(()) 102 | } 103 | 104 | /// Handles ABCI queries coming to the primary and forwards them to the ABCI App. Each 105 | /// handle call comes with a Sender channel which is used to send the response back to the 106 | /// Primary and then to the client. 107 | /// 108 | /// Client => Primary => handle_cert => ABCI App => Primary => Client 109 | fn handle_abci_query( 110 | &mut self, 111 | tx: OneShotSender, 112 | req: AbciQueryQuery, 113 | ) -> eyre::Result<()> { 114 | let req_height = req.height.unwrap_or(0); 115 | let req_prove = req.prove.unwrap_or(false); 116 | 117 | let resp = self.req_client.query(RequestQuery { 118 | data: req.data.into(), 119 | path: req.path, 120 | height: req_height as i64, 121 | prove: req_prove, 122 | })?; 123 | 124 | if let Err(err) = tx.send(resp) { 125 | eyre::bail!("{:?}", err); 126 | } 127 | Ok(()) 128 | } 129 | 130 | /// Opens a RocksDB handle to a Worker's database and tries to read the batch 131 | /// stored at the provided certificate's digest. 132 | fn reconstruct_batch(&self, digest: Digest, worker_id: u32) -> eyre::Result> { 133 | // Open the database to each worker 134 | // TODO: Figure out if this is expensive 135 | let db = rocksdb::DB::open_for_read_only( 136 | &rocksdb::Options::default(), 137 | self.worker_db(worker_id), 138 | true, 139 | )?; 140 | 141 | // Query the db 142 | let key = digest.to_vec(); 143 | match db.get(&key) { 144 | Ok(Some(res)) => Ok(res), 145 | Ok(None) => eyre::bail!("digest {} not found", digest), 146 | Err(err) => eyre::bail!(err), 147 | } 148 | } 149 | 150 | /// Calls DeliverTx on the ABCI app 151 | /// Deserializes a raw abtch as `WorkerMesssage::Batch` and proceeds to deliver 152 | /// each transaction over the DeliverTx API. 153 | fn deliver_batch(&mut self, batch: Vec) -> eyre::Result<()> { 154 | // Deserialize and parse the message. 155 | match bincode::deserialize(&batch) { 156 | Ok(WorkerMessage::Batch(batch)) => { 157 | batch.into_iter().try_for_each(|tx| { 158 | self.deliver_tx(tx)?; 159 | Ok::<_, eyre::Error>(()) 160 | })?; 161 | } 162 | _ => eyre::bail!("unrecognized message format"), 163 | }; 164 | Ok(()) 165 | } 166 | 167 | /// Reconstructs the batch corresponding to the provided Primary's certificate from the Workers' stores 168 | /// and proceeds to deliver each tx to the App over ABCI's DeliverTx endpoint. 169 | fn reconstruct_and_deliver_txs(&mut self, certificate: Certificate) -> eyre::Result<()> { 170 | // Try reconstructing the batches from the cert digests 171 | // 172 | // NB: 173 | // This is maybe a false positive by Clippy, without the `collect` the Iterator fails 174 | // iterator fails to compile because we're mutably borrowing in the `try_for_each` 175 | // when we've already immutably borrowed in the `.map`. 176 | #[allow(clippy::needless_collect)] 177 | let batches = certificate 178 | .header 179 | .payload 180 | .into_iter() 181 | .map(|(digest, worker_id)| self.reconstruct_batch(digest, worker_id)) 182 | .collect::>(); 183 | 184 | // Deliver 185 | batches.into_iter().try_for_each(|batch| { 186 | // this will throw an error if the deserialization failed anywhere 187 | let batch = batch?; 188 | self.deliver_batch(batch)?; 189 | Ok::<_, eyre::Error>(()) 190 | })?; 191 | 192 | Ok(()) 193 | } 194 | 195 | /// Helper function for getting the database handle to a worker associated 196 | /// with a primary (e.g. Primary db-0 -> Worker-0 db-0-0, Wroekr-1 db-0-1 etc.) 197 | fn worker_db(&self, id: u32) -> String { 198 | format!("{}-{}", self.store_path, id) 199 | } 200 | } 201 | 202 | // Tendermint Lifecycle Helpers 203 | impl Engine { 204 | /// Calls the `InitChain` hook on the app, ignores "already initialized" errors. 205 | pub fn init_chain(&mut self) -> eyre::Result<()> { 206 | let mut client = ClientBuilder::default().connect(&self.app_address)?; 207 | match client.init_chain(RequestInitChain::default()) { 208 | Ok(_) => {} 209 | Err(err) => { 210 | // ignore errors about the chain being uninitialized 211 | if err.to_string().contains("already initialized") { 212 | log::warn!("chain was already initialized: {}", err); 213 | return Ok(()); 214 | } 215 | eyre::bail!(err) 216 | } 217 | }; 218 | Ok(()) 219 | } 220 | 221 | /// Calls the `BeginBlock` hook on the ABCI app. For now, it just makes a request with 222 | /// the new block height. 223 | // If we wanted to, we could add additional arguments to be forwarded from the Consensus 224 | // to the App logic on the beginning of each block. 225 | fn begin_block(&mut self, height: i64) -> eyre::Result<()> { 226 | let req = RequestBeginBlock { 227 | header: Some(Header { 228 | height, 229 | ..Default::default() 230 | }), 231 | ..Default::default() 232 | }; 233 | 234 | self.client.begin_block(req)?; 235 | Ok(()) 236 | } 237 | 238 | /// Calls the `DeliverTx` hook on the ABCI app. 239 | fn deliver_tx(&mut self, tx: Transaction) -> eyre::Result<()> { 240 | self.client.deliver_tx(RequestDeliverTx { tx })?; 241 | Ok(()) 242 | } 243 | 244 | /// Calls the `EndBlock` hook on the ABCI app. For now, it just makes a request with 245 | /// the proposed block height. 246 | // If we wanted to, we could add additional arguments to be forwarded from the Consensus 247 | // to the App logic on the end of each block. 248 | fn end_block(&mut self, height: i64) -> eyre::Result<()> { 249 | let req = RequestEndBlock { height }; 250 | self.client.end_block(req)?; 251 | Ok(()) 252 | } 253 | 254 | /// Calls the `Commit` hook on the ABCI app. 255 | fn commit(&mut self) -> eyre::Result<()> { 256 | self.client.commit()?; 257 | Ok(()) 258 | } 259 | } 260 | 261 | // Helpers for deserializing batches, because `narwhal::worker` is not part 262 | // of the public API. TODO -> make a PR to expose it. 263 | pub type Transaction = Vec; 264 | pub type Batch = Vec; 265 | #[derive(serde::Deserialize)] 266 | pub enum WorkerMessage { 267 | Batch(Batch), 268 | } 269 | -------------------------------------------------------------------------------- /narwhal-abci/narwhal-abci/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod abci_server; 2 | pub use abci_server::AbciApi; 3 | 4 | mod engine; 5 | pub use engine::Engine; 6 | 7 | use serde::{Deserialize, Serialize}; 8 | 9 | #[derive(Serialize, Deserialize, Debug, Clone)] 10 | pub struct BroadcastTxQuery { 11 | tx: String, 12 | } 13 | 14 | #[derive(Serialize, Deserialize, Debug, Clone)] 15 | pub struct AbciQueryQuery { 16 | path: String, 17 | data: String, 18 | height: Option, 19 | prove: Option, 20 | } 21 | -------------------------------------------------------------------------------- /narwhal-abci/node/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "node" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | tokio = { version = "1.19.2", features = ["full"] } 10 | tokio-util = { version = "0.6.2", features= ["codec"] } 11 | clap = "2.33.3" 12 | env_logger = "0.7.1" 13 | log = "0.4.11" 14 | bytes = "1.0.1" 15 | bincode = "1.3.1" 16 | rand = "0.7.3" 17 | futures = "0.3.15" 18 | tendermint-abci = { version = "0.23.7", features = ["client"] } 19 | tendermint-proto = "0.23.7" 20 | 21 | config = { git = "https://github.com/asonnino/narwhal/" } 22 | store = { git = "https://github.com/asonnino/narwhal/" } 23 | crypto = { git = "https://github.com/asonnino/narwhal/" } 24 | primary = { git = "https://github.com/asonnino/narwhal/" } 25 | worker = { git = "https://github.com/asonnino/narwhal/" } 26 | consensus = { git = "https://github.com/asonnino/narwhal/" } 27 | 28 | hex = "0.4.3" 29 | ethers = { git = "https://github.com/gakonst/ethers-rs " } 30 | serde_json = "1.0.82" 31 | reqwest = "0.11.11" 32 | 33 | rocksdb = "0.16.0" 34 | warp = "0.3.2" 35 | serde = { version = "1.0", features = ["derive"] } 36 | 37 | narwhal-abci = { path = "../narwhal-abci" } 38 | eyre = "0.6.8" 39 | -------------------------------------------------------------------------------- /narwhal-abci/node/src/main.rs: -------------------------------------------------------------------------------- 1 | use crypto::PublicKey; 2 | use eyre::{Result, WrapErr}; 3 | use std::net::SocketAddr; 4 | 5 | // Copyright(C) Facebook, Inc. and its affiliates. 6 | use clap::{crate_name, crate_version, App, AppSettings, ArgMatches, SubCommand}; 7 | use config::Export as _; 8 | use config::Import as _; 9 | use config::{Committee, KeyPair, Parameters, WorkerId}; 10 | use consensus::Consensus; 11 | use env_logger::Env; 12 | use primary::Primary; 13 | use store::Store; 14 | use tokio::sync::mpsc::{channel, Receiver}; 15 | use worker::Worker; 16 | 17 | use narwhal_abci::{AbciApi, Engine}; 18 | 19 | /// The default channel capacity. 20 | pub const CHANNEL_CAPACITY: usize = 1_000; 21 | 22 | #[tokio::main] 23 | async fn main() -> Result<()> { 24 | let matches = App::new(crate_name!()) 25 | .version(crate_version!()) 26 | .about("A research implementation of Narwhal and Tusk.") 27 | .args_from_usage("-v... 'Sets the level of verbosity'") 28 | .subcommand( 29 | SubCommand::with_name("generate_keys") 30 | .about("Print a fresh key pair to file") 31 | .args_from_usage("--filename= 'The file where to print the new key pair'"), 32 | ) 33 | .subcommand( 34 | SubCommand::with_name("run") 35 | .about("Run a node") 36 | .args_from_usage("--keys= 'The file containing the node keys'") 37 | .args_from_usage("--committee= 'The file containing committee information'") 38 | .args_from_usage("--parameters=[FILE] 'The file containing the node parameters'") 39 | .args_from_usage("--store= 'The path where to create the data store'") 40 | .subcommand( 41 | SubCommand::with_name("primary") 42 | .about("Run a single primary") 43 | .args_from_usage( 44 | "--app-api= 'The host of the ABCI app receiving transactions'", 45 | ) 46 | .args_from_usage( 47 | "--abci-api= 'The address to receive ABCI connections to'", 48 | ), 49 | ) 50 | .subcommand( 51 | SubCommand::with_name("worker") 52 | .about("Run a single worker") 53 | .args_from_usage("--id= 'The worker id'"), 54 | ) 55 | .setting(AppSettings::SubcommandRequiredElseHelp), 56 | ) 57 | .setting(AppSettings::SubcommandRequiredElseHelp) 58 | .get_matches(); 59 | 60 | let log_level = match matches.occurrences_of("v") { 61 | 0 => "error", 62 | 1 => "warn", 63 | 2 => "info", 64 | 3 => "debug", 65 | _ => "trace", 66 | }; 67 | let mut logger = env_logger::Builder::from_env(Env::default().default_filter_or(log_level)); 68 | #[cfg(feature = "benchmark")] 69 | logger.format_timestamp_millis(); 70 | logger.init(); 71 | 72 | match matches.subcommand() { 73 | ("generate_keys", Some(sub_matches)) => KeyPair::new() 74 | .export(sub_matches.value_of("filename").unwrap()) 75 | .context("Failed to generate key pair")?, 76 | ("run", Some(sub_matches)) => run(sub_matches).await?, 77 | _ => unreachable!(), 78 | } 79 | Ok(()) 80 | } 81 | 82 | // Runs either a worker or a primary. 83 | async fn run(matches: &ArgMatches<'_>) -> Result<()> { 84 | let key_file = matches.value_of("keys").unwrap(); 85 | let committee_file = matches.value_of("committee").unwrap(); 86 | let parameters_file = matches.value_of("parameters"); 87 | let store_path = matches.value_of("store").unwrap(); 88 | 89 | // Read the committee and node's keypair from file. 90 | let keypair = KeyPair::import(key_file).context("Failed to load the node's keypair")?; 91 | let committee = 92 | Committee::import(committee_file).context("Failed to load the committee information")?; 93 | 94 | // Load default parameters if none are specified. 95 | let parameters = match parameters_file { 96 | Some(filename) => { 97 | Parameters::import(filename).context("Failed to load the node's parameters")? 98 | } 99 | None => Parameters::default(), 100 | }; 101 | 102 | // Make the data store. 103 | let store = Store::new(store_path).context("Failed to create a store")?; 104 | 105 | // Channels the sequence of certificates. 106 | let (tx_output, mut rx_output) = channel(CHANNEL_CAPACITY); 107 | 108 | // Check whether to run a primary, a worker, or an entire authority. 109 | match matches.subcommand() { 110 | // Spawn the primary and consensus core. 111 | ("primary", Some(sub_matches)) => { 112 | let (tx_new_certificates, rx_new_certificates) = channel(CHANNEL_CAPACITY); 113 | let (tx_feedback, rx_feedback) = channel(CHANNEL_CAPACITY); 114 | 115 | let keypair_name = keypair.name; 116 | 117 | let app_api = sub_matches.value_of("app-api").unwrap().to_string(); 118 | let abci_api = sub_matches.value_of("abci-api").unwrap().to_string(); 119 | 120 | Primary::spawn( 121 | keypair, 122 | committee.clone(), 123 | parameters.clone(), 124 | store.clone(), 125 | /* tx_consensus */ tx_new_certificates, 126 | /* rx_consensus */ rx_feedback, 127 | ); 128 | Consensus::spawn( 129 | committee.clone(), 130 | parameters.gc_depth, 131 | /* rx_primary */ rx_new_certificates, 132 | /* tx_primary */ tx_feedback, 133 | tx_output, 134 | ); 135 | 136 | process( 137 | rx_output, 138 | store_path, 139 | keypair_name, 140 | committee, 141 | abci_api, 142 | app_api, 143 | ) 144 | .await?; 145 | } 146 | 147 | // Spawn a single worker. 148 | ("worker", Some(sub_matches)) => { 149 | let id = sub_matches 150 | .value_of("id") 151 | .unwrap() 152 | .parse::() 153 | .context("The worker id must be a positive integer")?; 154 | 155 | Worker::spawn( 156 | keypair.name, 157 | id, 158 | committee.clone(), 159 | parameters, 160 | store.clone(), 161 | ); 162 | 163 | // for a worker there is nothing coming here ... 164 | rx_output.recv().await; 165 | } 166 | 167 | _ => unreachable!(), 168 | } 169 | 170 | // If this expression is reached, the program ends and all other tasks terminate. 171 | unreachable!(); 172 | } 173 | 174 | async fn process( 175 | rx_output: Receiver, 176 | store_path: &str, 177 | keypair_name: PublicKey, 178 | committee: Committee, 179 | abci_api: String, 180 | app_api: String, 181 | ) -> eyre::Result<()> { 182 | // address of mempool 183 | let mempool_address = committee 184 | .worker(&keypair_name.clone(), &0) 185 | .expect("Our public key or worker id is not in the committee") 186 | .transactions; 187 | 188 | // ABCI queries will be sent using this from the RPC to the ABCI client 189 | let (tx_abci_queries, rx_abci_queries) = channel(CHANNEL_CAPACITY); 190 | 191 | tokio::spawn(async move { 192 | let api = AbciApi::new(mempool_address, tx_abci_queries); 193 | // let tx_abci_queries = tx_abci_queries.clone(); 194 | // Spawn the ABCI RPC endpoint 195 | let mut address = abci_api.parse::().unwrap(); 196 | address.set_ip("0.0.0.0".parse().unwrap()); 197 | warp::serve(api.routes()).run(address).await 198 | }); 199 | 200 | // Analyze the consensus' output. 201 | // Spawn the network receiver listening to messages from the other primaries. 202 | let mut app_address = app_api.parse::().unwrap(); 203 | app_address.set_ip("0.0.0.0".parse().unwrap()); 204 | let mut engine = Engine::new(app_address, store_path, rx_abci_queries); 205 | engine.run(rx_output).await?; 206 | 207 | Ok(()) 208 | } 209 | -------------------------------------------------------------------------------- /narwhal-abci/starknet-abci/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "starknet-abci" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | abci-rs = { version = "0.11.3", features = ["async-api" ] } 10 | async-trait = "0.1.56" 11 | eyre = "0.6.8" 12 | hex = "0.4.3" 13 | serde_json = "1.0.82" 14 | tendermint-abci = { version = "0.23.7", features = ["client"] } 15 | tendermint-proto = "0.23.7" 16 | tokio = { version = "1.19.2", features = ["macros"] } 17 | clap = { version = "3.0.10", features = [ 18 | "derive", 19 | "env", 20 | "unicode", 21 | "wrap_help", 22 | ] } 23 | clap_complete = "3.0.4" 24 | serde = { version = "1.0.138", features = ["derive"] } 25 | reqwest = "0.11.11" 26 | sha2 = "0.10.6" 27 | tracing = "0.1.35" 28 | tracing-subscriber = { version = "0.3", features = ["registry", "env-filter", "fmt"] } 29 | tracing-error = "0.2.0" 30 | yansi = "0.5.1" 31 | uuid = { version = "1.2.1", features = ["v4"] } 32 | anyhow = "1.0.66" 33 | once_cell = "1.13.0" 34 | bincode = "1.3.3" 35 | starknet-rs = { git = "https://github.com/lambdaclass/starknet_in_rust", branch= "publish-structs" } 36 | # This was copied from starkent_in_rust/Cargo.toml as it seems it is missing an export for it 37 | felt = { git = "https://github.com/lambdaclass/cairo-rs", package = "cairo-felt", rev = "8dba86dbec935fa04a255e2edf3d5d184950fa22" } 38 | -------------------------------------------------------------------------------- /narwhal-abci/starknet-abci/src/app.rs: -------------------------------------------------------------------------------- 1 | 2 | use crate::{Consensus, Info, Mempool, Snapshot, State}; 3 | use std::sync::{Arc, Mutex}; 4 | 5 | pub struct App { 6 | pub mempool: Mempool, 7 | pub snapshot: Snapshot, 8 | pub consensus: Consensus, 9 | pub info: Info, 10 | } 11 | 12 | impl Default for App { 13 | fn default() -> Self { 14 | Self::new() 15 | } 16 | } 17 | 18 | // demo 19 | impl App { 20 | pub fn new() -> Self { 21 | let state = State { 22 | block_height: Default::default(), 23 | app_hash: Default::default(), 24 | }; 25 | 26 | let committed_state = Arc::new(Mutex::new(state.clone())); 27 | 28 | let consensus = Consensus::new(state); 29 | let mempool = Mempool::default(); 30 | 31 | let info = Info { 32 | state: committed_state, 33 | }; 34 | let snapshot = Snapshot::default(); 35 | 36 | App { 37 | consensus, 38 | mempool, 39 | info, 40 | snapshot, 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /narwhal-abci/starknet-abci/src/bin/client.rs: -------------------------------------------------------------------------------- 1 | use eyre::Result; 2 | use starknet_abci::transaction::{Transaction, TransactionType}; 3 | 4 | async fn send_transaction(host: &str) -> Result<()> { 5 | let transaction_type = TransactionType::FunctionExecution { 6 | function: "main".to_string(), 7 | program_name: "fibonacci.json".to_string(), 8 | }; 9 | 10 | let tx = Transaction::with_type(transaction_type).unwrap(); 11 | 12 | let tx = serde_json::to_string(&tx)?; 13 | 14 | let client = reqwest::Client::new(); 15 | client 16 | .get(format!("{}/broadcast_tx", host)) 17 | .query(&[("tx", tx)]) 18 | .send() 19 | .await?; 20 | 21 | Ok(()) 22 | } 23 | 24 | #[tokio::main] 25 | async fn main() -> Result<()> { 26 | // the ABCI port on the various narwhal primaries 27 | let hosts = ["http://127.0.0.1:3002", "http://127.0.0.1:3009", "http://127.0.0.1:3016"]; 28 | 29 | unsafe{ 30 | for i in 0..200 { 31 | let tx_result = send_transaction(hosts.get_unchecked(i%3)).await; 32 | match tx_result 33 | { 34 | Ok(_) => println!("transaction committed to {}", hosts.get_unchecked(i%3)), 35 | Err(e) => println!("error: {}", e), 36 | } 37 | std::thread::sleep(std::time::Duration::from_millis(5)); 38 | } 39 | } 40 | Ok(()) 41 | } 42 | -------------------------------------------------------------------------------- /narwhal-abci/starknet-abci/src/bin/starknet-app.rs: -------------------------------------------------------------------------------- 1 | use abci::async_api::Server; 2 | use starknet_abci::App; 3 | use std::net::SocketAddr; 4 | 5 | use clap::Parser; 6 | 7 | #[derive(Debug, Clone, Parser)] 8 | struct Args { 9 | #[clap(default_value = "0.0.0.0:26658")] 10 | host: String, 11 | #[clap(long, short)] 12 | demo: bool, 13 | } 14 | 15 | use tracing_error::ErrorLayer; 16 | 17 | use tracing_subscriber::prelude::*; 18 | 19 | /// Initializes a tracing Subscriber for logging 20 | #[allow(dead_code)] 21 | pub fn subscriber() { 22 | tracing_subscriber::Registry::default() 23 | .with(tracing_subscriber::EnvFilter::new("starknet-app=info")) 24 | .with(ErrorLayer::default()) 25 | .with(tracing_subscriber::fmt::layer()) 26 | .init() 27 | } 28 | 29 | #[tokio::main] 30 | async fn main() -> eyre::Result<()> { 31 | let args = Args::parse(); 32 | subscriber(); 33 | 34 | let App { 35 | consensus, 36 | mempool, 37 | info, 38 | snapshot, 39 | } = App::new(); 40 | let server = Server::new(consensus, mempool, info, snapshot); 41 | 42 | dbg!(&args.host); 43 | let addr = args.host.parse::().unwrap(); 44 | 45 | server.run(addr).await?; 46 | 47 | Ok(()) 48 | } 49 | -------------------------------------------------------------------------------- /narwhal-abci/starknet-abci/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod app; 2 | pub use app::App; 3 | 4 | pub mod types; 5 | pub use types::{Consensus, Info, Mempool, Snapshot, State}; 6 | pub mod transaction; -------------------------------------------------------------------------------- /narwhal-abci/starknet-abci/src/transaction.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use anyhow::{ensure, Result}; 4 | use felt::Felt; 5 | use serde::{Deserialize, Serialize}; 6 | use sha2::{Digest, Sha256}; 7 | use starknet_rs::business_logic::execution::execution_entry_point::ExecutionEntryPoint; 8 | use starknet_rs::business_logic::execution::objects::{CallType, TransactionExecutionContext}; 9 | use starknet_rs::business_logic::fact_state::contract_state::ContractState; 10 | use starknet_rs::business_logic::fact_state::in_memory_state_reader::InMemoryStateReader; 11 | use starknet_rs::business_logic::fact_state::state::ExecutionResourcesManager; 12 | use starknet_rs::business_logic::state::cached_state::CachedState; 13 | use starknet_rs::definitions::general_config::StarknetGeneralConfig; 14 | use starknet_rs::services::api::contract_class::{ContractClass, EntryPointType}; 15 | use starknet_rs::utils::Address; 16 | use uuid::Uuid; 17 | 18 | #[derive(Clone, Serialize, Deserialize, Debug)] 19 | pub struct Transaction { 20 | pub id: String, 21 | pub transaction_hash: String, // this acts 22 | pub transaction_type: TransactionType, 23 | } 24 | 25 | #[derive(Clone, Serialize, Deserialize, Debug)] 26 | pub enum TransactionType { 27 | /// Create new contract class. 28 | Declare, 29 | 30 | /// Create an instance of a contract which will have storage assigned. (Accounts are a contract themselves) 31 | Deploy, 32 | 33 | /// Execute a function from a deployed contract. 34 | Invoke, 35 | 36 | // TODO: Remove this when other transactions are implemented 37 | FunctionExecution { 38 | function: String, 39 | program_name: String, 40 | }, 41 | } 42 | 43 | impl Transaction { 44 | pub fn with_type(transaction_type: TransactionType) -> Result { 45 | Ok(Transaction { 46 | transaction_hash: transaction_type.compute_and_hash()?, 47 | transaction_type, 48 | id: Uuid::new_v4().to_string(), 49 | }) 50 | } 51 | 52 | /// Verify that the transaction id is consistent with its contents, by checking its sha256 hash. 53 | pub fn verify(&self) -> Result<()> { 54 | ensure!( 55 | self.transaction_hash == self.transaction_type.compute_and_hash()?, 56 | "Corrupted transaction: Inconsistent transaction id" 57 | ); 58 | 59 | Ok(()) 60 | } 61 | } 62 | 63 | impl TransactionType { 64 | pub fn compute_and_hash(&self) -> Result { 65 | match self { 66 | TransactionType::FunctionExecution { 67 | function, 68 | program_name: _, 69 | } => { 70 | let general_config = StarknetGeneralConfig::default(); 71 | 72 | let tx_execution_context = TransactionExecutionContext::create_for_testing( 73 | Address(0.into()), 74 | 10, 75 | 0.into(), 76 | general_config.invoke_tx_max_n_steps(), 77 | 1, 78 | ); 79 | 80 | let contract_address = Address(1111.into()); 81 | let class_hash = [1; 32]; 82 | let program = include_str!("../programs/fibonacci.json"); 83 | let contract_class = ContractClass::try_from(program.to_string()) 84 | .expect("Could not load contract from JSON"); 85 | 86 | let contract_state = ContractState::new( 87 | class_hash, 88 | tx_execution_context.nonce().clone(), 89 | Default::default(), 90 | ); 91 | let mut state_reader = InMemoryStateReader::new(HashMap::new(), HashMap::new()); 92 | state_reader 93 | .contract_states_mut() 94 | .insert(contract_address.clone(), contract_state); 95 | 96 | let mut state = CachedState::new( 97 | state_reader, 98 | Some([(class_hash, contract_class)].into_iter().collect()), 99 | ); 100 | 101 | let entry_point = ExecutionEntryPoint::new( 102 | contract_address, 103 | vec![], 104 | Felt::from_bytes_be(&starknet_rs::utils::calculate_sn_keccak( 105 | function.as_bytes(), 106 | )), 107 | Address(0.into()), 108 | EntryPointType::External, 109 | CallType::Delegate.into(), 110 | class_hash.into(), 111 | ); 112 | 113 | let mut resources_manager = ExecutionResourcesManager::default(); 114 | 115 | entry_point 116 | .execute( 117 | &mut state, 118 | &general_config, 119 | &mut resources_manager, 120 | &tx_execution_context, 121 | ) 122 | .expect("Could not execute contract"); 123 | 124 | let mut hasher = Sha256::new(); 125 | hasher.update(function); 126 | let hash = hasher.finalize().as_slice().to_owned(); 127 | Ok(hex::encode(hash)) 128 | } 129 | TransactionType::Declare => todo!(), 130 | TransactionType::Deploy => todo!(), 131 | TransactionType::Invoke => todo!(), 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /narwhal-abci/starknet-abci/src/types.rs: -------------------------------------------------------------------------------- 1 | use once_cell::sync::Lazy; 2 | use starknet_rs::testing::starknet_state::StarknetState; 3 | use tracing::{debug, info}; 4 | use std::{sync::Arc, sync::Mutex,time::Instant}; 5 | use sha2::{Digest, Sha256}; 6 | use crate::transaction::{Transaction, TransactionType}; 7 | 8 | use abci::{ 9 | async_api::{ 10 | Consensus as ConsensusTrait, Info as InfoTrait, Mempool as MempoolTrait, 11 | Snapshot as SnapshotTrait, 12 | }, 13 | async_trait, 14 | types::*, 15 | }; 16 | 17 | 18 | /// The app's state, containing a Revm DB. 19 | // TODO: Should we instead try to replace this with Anvil and implement traits for it? 20 | #[derive(Clone, Debug)] 21 | pub struct State { 22 | pub block_height: i64, 23 | pub app_hash: Vec, 24 | } 25 | 26 | impl Default for State { 27 | fn default() -> Self { 28 | Self { 29 | block_height: 0, 30 | app_hash: Vec::new(), 31 | } 32 | } 33 | } 34 | 35 | // because we don't get a `&mut self` in the ABCI API, we opt to have a mod-level variable 36 | // and because beginblock, endblock and deliver_tx all happen in the same thread, this is safe to do 37 | // an alternative would be Arc>, but we want to avoid extra-overhead of locks for the benchmark's sake 38 | static mut TRANSACTIONS: usize = 0; 39 | static mut TIMER: Lazy = Lazy::new(Instant::now); 40 | 41 | #[derive(serde::Serialize, serde::Deserialize, Debug)] 42 | pub struct TransactionResult { 43 | gas: u64, 44 | } 45 | 46 | pub struct Consensus { 47 | pub committed_state: Arc>, 48 | pub current_state: Arc>, 49 | hasher: Arc>, 50 | starknet_state: StarknetState, 51 | } 52 | 53 | impl Consensus { 54 | pub fn new(state: State) -> Self { 55 | let committed_state = Arc::new(Mutex::new(state.clone())); 56 | let current_state = Arc::new(Mutex::new(state)); 57 | 58 | Consensus { 59 | committed_state, 60 | current_state, 61 | hasher: Arc::new(Mutex::new(Sha256::new())), 62 | starknet_state: StarknetState::new(None), 63 | } 64 | } 65 | } 66 | 67 | #[async_trait] 68 | impl ConsensusTrait for Consensus { 69 | #[tracing::instrument(skip(self))] 70 | async fn init_chain(&self, _init_chain_request: RequestInitChain) -> ResponseInitChain { 71 | ResponseInitChain::default() 72 | } 73 | 74 | #[tracing::instrument(skip(self))] 75 | async fn begin_block(&self, _begin_block_request: RequestBeginBlock) -> ResponseBeginBlock { 76 | // because begin_block, [deliver_tx] and end_block/commit are on the same thread, this is safe to do (see declaration of statics) 77 | unsafe { 78 | info!( 79 | "{} ms passed between begin_block() calls. {} transactions, {} tps", 80 | (*TIMER).elapsed().as_millis(), 81 | TRANSACTIONS, 82 | (TRANSACTIONS * 1000) as f32 / ((*TIMER).elapsed().as_millis() as f32) 83 | ); 84 | TRANSACTIONS = 0; 85 | 86 | *TIMER = Instant::now(); 87 | } 88 | 89 | Default::default() 90 | } 91 | 92 | #[tracing::instrument(skip(self))] 93 | async fn deliver_tx(&self, request: RequestDeliverTx) -> ResponseDeliverTx { 94 | tracing::trace!("delivering tx"); 95 | 96 | let tx: Transaction = serde_json::from_slice(&request.tx).unwrap(); 97 | 98 | // Validation consists of getting the hash and checking whether it is equal 99 | // to the tx id. The hash executes the program and hashes the trace. 100 | 101 | let tx_hash = tx 102 | .transaction_type 103 | .compute_and_hash() 104 | .map(|x| x == tx.transaction_hash); 105 | 106 | // because begin_block, [deliver_tx] and end_block/commit are on the same thread, this is safe to do (see declaration of statics) 107 | unsafe { 108 | TRANSACTIONS += 1; 109 | } 110 | 111 | match tx_hash { 112 | Ok(true) => { 113 | let _ = self 114 | .hasher 115 | .lock() 116 | .map(|mut hash| hash.update(tx.transaction_hash.clone())); 117 | 118 | // prepare this transaction to be queried by app.tx_id 119 | let index_event = Event { 120 | r#type: "app".to_string(), 121 | attributes: vec![EventAttribute { 122 | key: "tx_id".to_string().into_bytes(), 123 | value: tx.transaction_hash.to_string().into_bytes(), 124 | index: true, 125 | }], 126 | }; 127 | let mut events = vec![index_event]; 128 | 129 | match tx.transaction_type { 130 | TransactionType::FunctionExecution { 131 | function, 132 | program_name: _, 133 | } => { 134 | let function_event = Event { 135 | r#type: "function".to_string(), 136 | attributes: vec![EventAttribute { 137 | key: "function".to_string().into_bytes(), 138 | value: function.into_bytes(), 139 | index: true, 140 | }], 141 | }; 142 | events.push(function_event); 143 | } 144 | TransactionType::Declare => todo!(), 145 | TransactionType::Deploy => todo!(), 146 | TransactionType::Invoke => todo!(), 147 | } 148 | 149 | ResponseDeliverTx { 150 | events, 151 | data: tx.transaction_hash.into_bytes(), 152 | ..Default::default() 153 | } 154 | } 155 | Ok(false) => ResponseDeliverTx { 156 | code: 1, 157 | log: "Error delivering transaction. Integrity check failed.".to_string(), 158 | info: "Error delivering transaction. Integrity check failed.".to_string(), 159 | ..Default::default() 160 | }, 161 | Err(e) => ResponseDeliverTx { 162 | code: 1, 163 | log: format!("Error delivering transaction: {e}"), 164 | info: format!("Error delivering transaction: {e}"), 165 | ..Default::default() 166 | }, 167 | } 168 | } 169 | 170 | #[tracing::instrument(skip(self))] 171 | async fn end_block(&self, _end_block_request: RequestEndBlock) -> ResponseEndBlock { 172 | // because begin_block, [deliver_tx] and end_block/commit are on the same thread, this is safe to do (see declaration of statics) 173 | unsafe { 174 | info!( 175 | "Committing block with {} transactions in {} ms. TPS: {}", 176 | TRANSACTIONS, 177 | (*TIMER).elapsed().as_millis(), 178 | (TRANSACTIONS * 1000) as f32 / ((*TIMER).elapsed().as_millis() as f32) 179 | ); 180 | } 181 | ResponseEndBlock { 182 | ..Default::default() 183 | } 184 | } 185 | 186 | #[tracing::instrument(skip(self))] 187 | async fn commit(&self, _commit_request: RequestCommit) -> ResponseCommit { 188 | 189 | let app_hash: Result, String> = Ok(vec![]); 190 | 191 | // because begin_block, [deliver_tx] and end_block/commit are on the same thread, this is safe to do (see declaration of statics) 192 | unsafe { 193 | info!( 194 | "Committing block with {} transactions in {} ms. TPS: {}", 195 | TRANSACTIONS, 196 | (*TIMER).elapsed().as_millis(), 197 | (TRANSACTIONS * 1000) as f32 / ((*TIMER).elapsed().as_millis() as f32) 198 | ); 199 | } 200 | 201 | match app_hash { 202 | Ok(hash) => ResponseCommit { 203 | data: hash, 204 | retain_height: 0, 205 | }, 206 | // error should be handled here 207 | _ => ResponseCommit { 208 | data: vec![], 209 | retain_height: 0, 210 | }, 211 | } 212 | } 213 | } 214 | 215 | #[derive(Debug, Clone, Default)] 216 | pub struct Mempool; 217 | 218 | #[async_trait] 219 | impl MempoolTrait for Mempool { 220 | async fn check_tx(&self, _check_tx_request: RequestCheckTx) -> ResponseCheckTx { 221 | ResponseCheckTx::default() 222 | } 223 | } 224 | 225 | #[derive(Debug, Clone)] 226 | pub struct Info { 227 | pub state: Arc>, 228 | } 229 | 230 | #[async_trait] 231 | impl InfoTrait for Info { 232 | // replicate the eth_call interface 233 | async fn query(&self, query_request: RequestQuery) -> ResponseQuery { 234 | ResponseQuery { 235 | key: query_request.data, 236 | ..Default::default() 237 | } 238 | } 239 | 240 | async fn info(&self, info_request: RequestInfo) -> ResponseInfo { 241 | debug!( 242 | "Got info request. Tendermint version: {}; Block version: {}; P2P version: {}", 243 | info_request.version, info_request.block_version, info_request.p2p_version 244 | ); 245 | 246 | ResponseInfo { 247 | data: "cairo-app".to_string(), 248 | version: "0.1.0".to_string(), 249 | app_version: 1, 250 | last_block_height: 0i64, 251 | 252 | // using a fixed hash, see the commit() hook 253 | last_block_app_hash: vec![], 254 | } 255 | } 256 | } 257 | 258 | #[derive(Debug, Clone, Default)] 259 | pub struct Snapshot; 260 | 261 | impl SnapshotTrait for Snapshot {} 262 | 263 | 264 | /* 265 | #[cfg(test)] 266 | mod tests { 267 | use super::*; 268 | // use ethers::prelude::*; 269 | 270 | #[tokio::test] 271 | async fn run_and_query_tx() { 272 | let val = ethers::utils::parse_units(1, 18).unwrap(); 273 | let alice = Address::random(); 274 | let bob = Address::random(); 275 | 276 | let mut state = State::default(); 277 | 278 | // give alice some money 279 | state.db.insert_account_info( 280 | alice, 281 | revm::AccountInfo { 282 | balance: val, 283 | ..Default::default() 284 | }, 285 | ); 286 | 287 | // make the tx 288 | let tx = TransactionRequest::new() 289 | .from(alice) 290 | .to(bob) 291 | .gas_price(0) 292 | .data(vec![1, 2, 3, 4, 5]) 293 | .gas(31000) 294 | .value(val); 295 | 296 | // Send it over an ABCI message 297 | 298 | let consensus = Consensus::new(state); 299 | 300 | let req = RequestDeliverTx { 301 | tx: serde_json::to_vec(&tx).unwrap(), 302 | }; 303 | let res = consensus.deliver_tx(req).await; 304 | let res: TransactionResult = serde_json::from_slice(&res.data).unwrap(); 305 | // tx passed 306 | assert_eq!(res.exit, Return::Stop); 307 | 308 | // now we query the state for bob's balance 309 | let info = Info { 310 | state: consensus.current_state.clone(), 311 | }; 312 | let res = info 313 | .query(RequestQuery { 314 | data: serde_json::to_vec(&Query::Balance(bob)).unwrap(), 315 | ..Default::default() 316 | }) 317 | .await; 318 | let res: QueryResponse = serde_json::from_slice(&res.value).unwrap(); 319 | let balance = res.as_balance(); 320 | assert_eq!(balance, val); 321 | } 322 | } */ 323 | -------------------------------------------------------------------------------- /sequencer/.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /.DS_Store 4 | **/target/ 5 | target/ 6 | 7 | .vscode/ 8 | 9 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 10 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 11 | Cargo.lock 12 | 13 | hello/build 14 | bin/ 15 | abci.* 16 | testnet/ 17 | localnet/ 18 | ./tendermint-install 19 | 20 | # Grafana local storage 21 | grafana-storage/ 22 | -------------------------------------------------------------------------------- /sequencer/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "starknet-sequencer" 3 | version = "0.1.0" 4 | edition = "2021" 5 | default-run = "cli" 6 | 7 | [[bin]] 8 | path = "src/cli/main.rs" 9 | doctest = false 10 | name = "cli" 11 | 12 | [[bin]] 13 | path = "src/abci/main.rs" 14 | name = "abci" 15 | 16 | [[bin]] 17 | path = "src/bench/main.rs" 18 | name = "bench" 19 | 20 | [lib] 21 | path = "src/lib/mod.rs" 22 | doctest = false 23 | name = "lib" 24 | 25 | [profile.test] 26 | opt-level = 3 27 | debug-assertions = true 28 | 29 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 30 | 31 | [dependencies] 32 | anyhow = "1.0.66" 33 | bincode = "1.3.3" 34 | clap = { version = "4.0.5", features = ["derive", "env"] } 35 | once_cell = "*" 36 | futures = "0.3.26" 37 | hex = "0.4.3" 38 | sha2 = "0.10.6" 39 | serde = "1.0" 40 | serde_json = { version = "1.0", features = ["raw_value"] } 41 | tendermint = "0.30.0" 42 | tendermint-abci = "0.30.0" 43 | tendermint-proto = { version = "0.30.0", default-features = false } 44 | tendermint-rpc = { version = "0.30.0", features = ["http-client"] } 45 | tracing = "0.1" 46 | tracing-subscriber = {version = "0.3", features = ["env-filter", "fmt", "std"]} 47 | tokio = { version = "1.15.0", features = ["full"] } 48 | uuid = { version = "1.2.1", features = ["v4"] } 49 | starknet-rs = { git = "https://github.com/lambdaclass/starknet_in_rust", rev = "4ab3433c51df485cd205142ce96a92559b21a2e2" } 50 | # This was copied from starkent_in_rust/Cargo.toml as it seems it is missing an export for it 51 | felt = { git = "https://github.com/lambdaclass/cairo-rs", package = "cairo-felt", rev="77fe09ebbf72710935b455b1c5ff56b0bad7a4b8" } 52 | num-traits = "0.2.15" 53 | 54 | [dev-dependencies] 55 | assert_fs = "1.0.9" 56 | assert_cmd = "2.0.6" 57 | retry = "2.0.0" 58 | serial_test = "1.0.0" 59 | ctor = "0.1.23" -------------------------------------------------------------------------------- /sequencer/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /sequencer/bench/cmd/load_test/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/informalsystems/tm-load-test/pkg/loadtest" 5 | "github.com/lambdaclass/load_tester/pkg/abci" 6 | ) 7 | 8 | func main() { 9 | if err := loadtest.RegisterClientFactory("my-abci-app-name", &abci.MyABCIAppClientFactory{}); err != nil { 10 | panic(err) 11 | } 12 | // The loadtest.Run method will handle CLI argument parsing, errors, 13 | // configuration, instantiating the load test and/or coordinator/worker 14 | // operations, etc. All it needs is to know which client factory to use for 15 | // its load testing. 16 | loadtest.Run(&loadtest.CLIConfig{ 17 | AppName: "my-load-tester", 18 | AppShortDesc: "Load testing application for My ABCI App (TM)", 19 | AppLongDesc: "Some long description on how to use the tool", 20 | DefaultClientFactory: "my-abci-app-name", 21 | }) 22 | } -------------------------------------------------------------------------------- /sequencer/bench/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/lambdaclass/load_tester 2 | 3 | go 1.19 4 | 5 | require github.com/informalsystems/tm-load-test v1.3.0 6 | 7 | require ( 8 | github.com/beorn7/perks v1.0.1 // indirect 9 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 10 | github.com/golang/protobuf v1.5.2 // indirect 11 | github.com/google/uuid v1.3.0 // indirect 12 | github.com/gorilla/websocket v1.5.0 // indirect 13 | github.com/inconshreveable/mousetrap v1.0.1 // indirect 14 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect 15 | github.com/prometheus/client_golang v1.14.0 // indirect 16 | github.com/prometheus/client_model v0.3.0 // indirect 17 | github.com/prometheus/common v0.37.0 // indirect 18 | github.com/prometheus/procfs v0.8.0 // indirect 19 | github.com/satori/go.uuid v1.2.0 // indirect 20 | github.com/sirupsen/logrus v1.9.0 // indirect 21 | github.com/spf13/cobra v1.6.1 // indirect 22 | github.com/spf13/pflag v1.0.5 // indirect 23 | golang.org/x/sys v0.4.0 // indirect 24 | google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 // indirect 25 | ) 26 | -------------------------------------------------------------------------------- /sequencer/bench/pkg/abci/client.go: -------------------------------------------------------------------------------- 1 | package abci 2 | 3 | import ( 4 | "github.com/informalsystems/tm-load-test/pkg/loadtest" 5 | "github.com/google/uuid" 6 | "log" 7 | "os/exec" 8 | "bytes" 9 | ) 10 | 11 | // MyABCIAppClientFactory creates instances of MyABCIAppClient 12 | type MyABCIAppClientFactory struct {} 13 | 14 | // MyABCIAppClientFactory implements loadtest.ClientFactory 15 | var _ loadtest.ClientFactory = (*MyABCIAppClientFactory)(nil) 16 | 17 | // MyABCIAppClient is responsible for generating transactions. Only one client 18 | // will be created per connection to the remote Tendermint RPC endpoint, and 19 | // each client will be responsible for maintaining its own state in a 20 | // thread-safe manner. 21 | type MyABCIAppClient struct { 22 | tx []byte 23 | } 24 | 25 | // MyABCIAppClient implements loadtest.Client 26 | var _ loadtest.Client = (*MyABCIAppClient)(nil) 27 | 28 | func (f *MyABCIAppClientFactory) ValidateConfig(cfg loadtest.Config) error { 29 | // Do any checks here that you need to ensure that the load test 30 | // configuration is compatible with your client. 31 | return nil 32 | } 33 | 34 | func (f *MyABCIAppClientFactory) NewClient(cfg loadtest.Config) (loadtest.Client, error) { 35 | cmd := exec.Command("cargo", "run", "--release", "programs/fibonacci.json", "main", "--no-broadcast") 36 | var outb, errb bytes.Buffer 37 | cmd.Stdout = &outb 38 | cmd.Stderr = &errb 39 | err := cmd.Run() 40 | if err != nil { 41 | log.Fatal(err) 42 | } 43 | return &MyABCIAppClient{tx: outb.Bytes()}, nil 44 | } 45 | 46 | // GenerateTx must return the raw bytes that make up the transaction for your 47 | // ABCI app. The conversion to base64 will automatically be handled by the 48 | // loadtest package, so don't worry about that. Only return an error here if you 49 | // want to completely fail the entire load test operation. 50 | func (c *MyABCIAppClient) GenerateTx() ([]byte, error) { 51 | var newTx []byte 52 | newTx = append(newTx, c.tx[0:8]...) 53 | // Replacing the uuid with a new random one to prevent getting duplicated tx rejected 54 | newTx = append(newTx, []byte(uuid.New().String())...) 55 | newTx = append(newTx, c.tx[44:]...) 56 | return newTx, nil 57 | } -------------------------------------------------------------------------------- /sequencer/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | prometheus: 3 | image: prom/prometheus 4 | ports: 5 | - "9090:9090" 6 | volumes: 7 | - ./prometheus.yml:/etc/prometheus/prometheus.yml 8 | grafana: 9 | image: grafana/grafana-oss 10 | ports: 11 | - "3000:3000" 12 | volumes: 13 | - ./grafana-storage:/var/lib/grafana 14 | -------------------------------------------------------------------------------- /sequencer/playbooks/deploy.yaml: -------------------------------------------------------------------------------- 1 | - name: Deploy new version 2 | hosts: tendermint-nodes 3 | gather_facts: false 4 | any_errors_fatal: true 5 | become: true 6 | become_user: root 7 | 8 | pre_tasks: 9 | 10 | - name: Stop tendermint service 11 | ansible.builtin.systemd: 12 | state: stopped 13 | name: tendermint 14 | daemon_reload: true 15 | 16 | - name: Stop abci service 17 | ansible.builtin.systemd: 18 | state: stopped 19 | name: abci 20 | daemon_reload: true 21 | 22 | tasks: 23 | 24 | - name: Reset tendermint network 25 | ansible.builtin.shell: | 26 | /usr/bin/tendermint unsafe_reset_all --home /.tendermint 27 | 28 | - name: Delete starknet_tendermint_sequencer repo 29 | ansible.builtin.file: 30 | state: absent 31 | path: /root/starknet_tendermint_sequencer 32 | 33 | - name: Clone starknet_tendermint_sequencer repo 34 | ansible.builtin.git: 35 | repo: git@github.com:lambdaclass/starknet_tendermint_sequencer.git 36 | dest: /root/starknet_tendermint_sequencer 37 | version: main 38 | accept_hostkey: true 39 | 40 | post_tasks: 41 | 42 | - name: Start abci service 43 | ansible.builtin.systemd: 44 | state: started 45 | name: abci 46 | daemon_reload: true 47 | 48 | - name: Start tendermint service 49 | ansible.builtin.systemd: 50 | state: started 51 | name: tendermint 52 | daemon_reload: true 53 | -------------------------------------------------------------------------------- /sequencer/programs/factorial.cairo: -------------------------------------------------------------------------------- 1 | // factorial(n) = n! 2 | func factorial(n) -> (result: felt) { 3 | if (n == 1) { 4 | return (n,); 5 | } 6 | let (a) = factorial(n - 1); 7 | return (n * a,); 8 | } 9 | 10 | func main() { 11 | // Make sure the factorial(10) == 3628800 12 | let (y) = factorial(10); 13 | y = 3628800; 14 | return (); 15 | } -------------------------------------------------------------------------------- /sequencer/programs/fibonacci.cairo: -------------------------------------------------------------------------------- 1 | %lang starknet 2 | 3 | @external 4 | func main() { 5 | // Call fib(1, 1, 10). 6 | let result: felt = fib(1, 1, 500); 7 | ret; 8 | } 9 | 10 | func fib(first_element, second_element, n) -> (res: felt) { 11 | jmp fib_body if n != 0; 12 | tempvar result = second_element; 13 | return (second_element,); 14 | 15 | fib_body: 16 | tempvar y = first_element + second_element; 17 | return fib(second_element, y, n - 1); 18 | } 19 | -------------------------------------------------------------------------------- /sequencer/prometheus.yml: -------------------------------------------------------------------------------- 1 | # docker run -p 9090:9090 -v /Users/aminarria/Lambda/starknet_tendermint_sequencer/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus 2 | 3 | # my global config 4 | global: 5 | scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. 6 | evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. 7 | # scrape_timeout is set to the global default (10s). 8 | 9 | # Alertmanager configuration 10 | alerting: 11 | alertmanagers: 12 | - static_configs: 13 | - targets: 14 | # - alertmanager:9093 15 | 16 | # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. 17 | rule_files: 18 | # - "first_rules.yml" 19 | # - "second_rules.yml" 20 | 21 | # A scrape configuration containing exactly one endpoint to scrape: 22 | # Here it's Prometheus itself. 23 | scrape_configs: 24 | # The job name is added as a label `job=` to any timeseries scraped from this config. 25 | - job_name: "prometheus" 26 | 27 | # metrics_path defaults to '/metrics' 28 | # scheme defaults to 'http'. 29 | 30 | static_configs: 31 | - targets: ["localhost:9090"] 32 | - targets: ["host.docker.internal:26660"] 33 | labels: 34 | groups: 'local-tendermint' 35 | - targets: ["5.9.57.44:26660", "5.9.57.45:26660", "5.9.57.89:26660"] 36 | labels: 37 | groups: 'tendermint' 38 | -------------------------------------------------------------------------------- /sequencer/src/abci/main.rs: -------------------------------------------------------------------------------- 1 | use application::StarknetApp; 2 | use clap::Parser; 3 | use tendermint_abci::ServerBuilder; 4 | use tracing_subscriber::{filter::LevelFilter, util::SubscriberInitExt}; 5 | 6 | mod application; 7 | 8 | #[derive(Debug, Parser)] 9 | #[clap(author, version, about)] 10 | struct Cli { 11 | /// Bind the TCP server to this host. 12 | #[clap(long, default_value = "127.0.0.1")] 13 | host: String, 14 | 15 | /// Bind the TCP server to this port. 16 | #[clap(short, long, default_value = "26658")] 17 | port: u16, 18 | 19 | /// The default server read buffer size, in bytes, for each incoming client 20 | /// connection. 21 | #[clap(short, long, default_value = "1048576")] 22 | read_buf_size: usize, 23 | 24 | /// Increase output logging verbosity to DEBUG level. 25 | #[clap(short, long)] 26 | verbose: bool, 27 | 28 | /// Suppress all output logging (overrides --verbose). 29 | #[clap(short, long)] 30 | quiet: bool, 31 | } 32 | 33 | fn main() { 34 | let cli: Cli = Cli::parse(); 35 | let log_level = if cli.quiet { 36 | LevelFilter::OFF 37 | } else if cli.verbose { 38 | LevelFilter::DEBUG 39 | } else { 40 | LevelFilter::INFO 41 | }; 42 | 43 | let subscriber = tracing_subscriber::fmt() 44 | // Use a more compact, abbreviated log format 45 | .compact() 46 | .with_max_level(log_level) 47 | // Display the thread ID an event was recorded on 48 | .with_thread_ids(true) 49 | // Don't display the event's target (module path) 50 | .with_target(false) 51 | // Build the subscriber 52 | .finish(); 53 | 54 | subscriber.init(); 55 | 56 | let app = StarknetApp::new(); 57 | let server = ServerBuilder::new(cli.read_buf_size) 58 | .bind(format!("{}:{}", cli.host, cli.port), app) 59 | .unwrap(); 60 | 61 | server.listen().unwrap(); 62 | } 63 | -------------------------------------------------------------------------------- /sequencer/src/bench/main.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | use lib::{Transaction, TransactionType}; 3 | use std::net::SocketAddr; 4 | use std::time::Instant; 5 | use tendermint_rpc::{Client, HttpClient}; 6 | use tracing::{info, metadata::LevelFilter}; 7 | use tracing_subscriber::util::SubscriberInitExt; 8 | use uuid::Uuid; 9 | 10 | #[derive(Parser)] 11 | #[clap()] 12 | pub struct Cli { 13 | /// Amount of concurrent threads from which transactions will be sent. Each thread will have a client connection to the network. 14 | #[clap(short, long, value_parser, value_name = "UINT", default_value_t = 4)] 15 | pub threads: i32, 16 | 17 | /// Number of transactions per second each thread sends out. 18 | #[clap(short, long, value_parser, value_name = "UINT", default_value_t = 1000)] 19 | pub transactions_per_thread: usize, 20 | 21 | /// Nodes to which transactions will be sent to (round-robin). 22 | #[clap( 23 | long, 24 | value_parser, 25 | value_name = "ADDR", 26 | use_value_delimiter = true, 27 | value_delimiter = ' ' 28 | )] 29 | nodes: Vec, 30 | } 31 | 32 | #[tokio::main()] 33 | #[allow(unreachable_code)] 34 | async fn main() { 35 | todo!("New execute method needs to be implemented on bench/main.rs"); 36 | 37 | let cli = Cli::parse(); 38 | 39 | tracing_subscriber::fmt() 40 | // Use a more compact, abbreviated log format 41 | .compact() 42 | // Display the thread ID an event was recorded on 43 | .with_thread_ids(true) 44 | .with_max_level(LevelFilter::INFO) 45 | // Build and init the subscriber 46 | .finish() 47 | .init(); 48 | 49 | // prepare transactions 50 | let program = include_str!("../../programs/fibonacci.json").to_string(); 51 | 52 | let transaction_type = TransactionType::Declare { program }; 53 | 54 | let transaction = Transaction::with_type(transaction_type).unwrap(); 55 | info!( 56 | "Single benchmark transaction size: {} bytes", 57 | bincode::serialize(&transaction).unwrap().len() 58 | ); 59 | 60 | let mut handles = vec![]; 61 | 62 | let time = Instant::now(); 63 | 64 | // prepare a pool of transactions for each thread in order to have them sent out as soon as possible 65 | for _i in 0..cli.threads { 66 | let mut transactions = Vec::with_capacity(cli.transactions_per_thread); 67 | 68 | for _i in 0..cli.transactions_per_thread { 69 | let t = transaction.clone(); 70 | // in order to not have Tendermint see the transactions as duplicate and discard them, 71 | // clone the transactions with a different ID 72 | let t = Transaction { 73 | id: Uuid::new_v4().to_string(), 74 | transaction_hash: t.transaction_hash, 75 | transaction_type: t.transaction_type, 76 | }; 77 | 78 | transactions.push(bincode::serialize(&t).unwrap()); 79 | } 80 | let nodes = cli.nodes.clone(); 81 | 82 | handles.push(tokio::spawn(async move { 83 | run(transactions.clone(), &nodes).await; 84 | })); 85 | } 86 | 87 | futures::future::join_all(handles).await; 88 | info!( 89 | "Time it took for all transactions to be delivered: {} ms", 90 | time.elapsed().as_millis() 91 | ); 92 | } 93 | 94 | async fn run(transactions: Vec>, nodes: &Vec) { 95 | let time = Instant::now(); 96 | let mut clients = vec![]; 97 | for i in 0..nodes.len() { 98 | let url = format!("http://{}", &nodes.get(i).unwrap()); 99 | clients.push(HttpClient::new(url.as_str()).unwrap()); 100 | } 101 | 102 | let n_transactions = transactions.len(); 103 | // for each transaction in this thread, send transactions in a round robin fashion to each node 104 | for (i, t) in transactions.into_iter().enumerate() { 105 | let c = clients.get(i % clients.len()); // get destination node 106 | let response = c.unwrap().broadcast_tx_async(t).await; 107 | 108 | match &response { 109 | Ok(_) => {} 110 | Err(v) => info!("failure: {}", v), 111 | } 112 | 113 | let response = response.unwrap(); 114 | match response.code { 115 | tendermint::abci::Code::Ok => {} 116 | tendermint::abci::Code::Err(code) => { 117 | info!("Error executing transaction {}: {}", code, response.log); 118 | } 119 | } 120 | } 121 | info!( 122 | "transactions sent: {} in {} ms", 123 | n_transactions, 124 | time.elapsed().as_millis() 125 | ); 126 | } 127 | -------------------------------------------------------------------------------- /sequencer/src/cli/main.rs: -------------------------------------------------------------------------------- 1 | use crate::tendermint::broadcast; 2 | use anyhow::{bail, Result}; 3 | use clap::{Args, Parser, Subcommand}; 4 | use lib::{Transaction, TransactionType}; 5 | use serde_json::json; 6 | use std::fs; 7 | use std::path::PathBuf; 8 | use std::str; 9 | use tracing_subscriber::util::SubscriberInitExt; 10 | use tracing_subscriber::EnvFilter; 11 | 12 | pub mod tendermint; 13 | const LOCAL_SEQUENCER_URL: &str = "http://127.0.0.1:26657"; 14 | 15 | #[derive(Parser)] 16 | struct Cli { 17 | /// Subcommand to execute 18 | #[command(subcommand)] 19 | command: Command, 20 | 21 | /// Output log lines to stdout based on the desired log level (RUST_LOG env var). 22 | #[clap(short, long, global = false, default_value_t = false)] 23 | pub verbose: bool, 24 | 25 | /// Just run the program and return the transaction in the stdio 26 | #[clap(short, long, global = false, default_value_t = false)] 27 | pub no_broadcast: bool, 28 | 29 | /// Tendermint node url 30 | #[clap(short, long, env = "SEQUENCER_URL", default_value = LOCAL_SEQUENCER_URL)] 31 | pub url: String, 32 | } 33 | 34 | #[derive(Subcommand)] 35 | enum Command { 36 | Declare(DeclareArgs), 37 | DeployAccount(DeployArgs), 38 | Invoke(InvokeArgs), 39 | Get(GetArgs), 40 | } 41 | 42 | #[derive(Args)] 43 | pub struct GetArgs { 44 | transaction_id: String, 45 | } 46 | 47 | #[derive(Args)] 48 | pub struct DeclareArgs { 49 | #[arg(long)] 50 | contract: PathBuf, 51 | } 52 | 53 | #[derive(Args)] 54 | pub struct DeployArgs { 55 | class_hash: String, 56 | // TODO: randomize salt by default? 57 | #[arg(long, default_value = "1111")] 58 | salt: i32, 59 | #[arg(long, num_args=1.., value_delimiter = ' ')] 60 | inputs: Option>, 61 | } 62 | 63 | #[derive(Args, Debug)] 64 | pub struct InvokeArgs { 65 | /// Contract Address 66 | #[clap(short, long)] 67 | address: String, 68 | 69 | /// Function name 70 | #[clap(short, long)] 71 | function: String, 72 | 73 | /// Function input values 74 | #[clap(long, num_args=1.., value_delimiter = ' ')] 75 | inputs: Option>, 76 | 77 | /// tendermint node url 78 | #[clap(short, long, env = "SEQUENCER_URL", default_value = LOCAL_SEQUENCER_URL)] 79 | pub url: String, 80 | } 81 | 82 | #[tokio::main()] 83 | async fn main() { 84 | let cli = Cli::parse(); 85 | 86 | if cli.verbose { 87 | tracing_subscriber::fmt() 88 | // Use a more compact, abbreviated log format 89 | .compact() 90 | .with_env_filter(EnvFilter::from_default_env()) 91 | // Build and init the subscriber 92 | .finish() 93 | .init(); 94 | } 95 | 96 | let result = match cli.command { 97 | Command::Declare(declare_args) => do_declare(declare_args, &cli.url).await, 98 | Command::DeployAccount(deploy_args) => do_deploy(deploy_args, &cli.url).await, 99 | Command::Invoke(invoke_args) => do_invoke(invoke_args, &cli.url).await, 100 | Command::Get(get_args) => { 101 | tendermint::get_transaction(&get_args.transaction_id, &cli.url).await 102 | } 103 | }; 104 | 105 | let (code, output) = match result { 106 | Ok(output) => (0, json!({"id": output.id,"hash": output.transaction_hash})), 107 | Err(err) => (1, json!({"error": err.to_string()})), 108 | }; 109 | 110 | println!("\n{output:#}"); 111 | std::process::exit(code); 112 | } 113 | 114 | async fn do_declare(args: DeclareArgs, url: &str) -> Result { 115 | let program = fs::read_to_string(args.contract)?; 116 | let transaction_type = TransactionType::Declare { program }; 117 | let transaction = Transaction::with_type(transaction_type)?; 118 | let transaction_serialized = bincode::serialize(&transaction)?; 119 | 120 | match tendermint::broadcast(transaction_serialized, url).await { 121 | Ok(_) => Ok(transaction), 122 | Err(e) => bail!("DECLARE: Error ocurred when sending out transaction: {e}"), 123 | } 124 | } 125 | 126 | async fn do_deploy(args: DeployArgs, url: &str) -> Result { 127 | let transaction_type = TransactionType::DeployAccount { 128 | class_hash: args.class_hash, 129 | salt: args.salt, 130 | inputs: args.inputs, 131 | }; 132 | 133 | let transaction = Transaction::with_type(transaction_type)?; 134 | let transaction_serialized = bincode::serialize(&transaction)?; 135 | 136 | match tendermint::broadcast(transaction_serialized, url).await { 137 | Ok(_) => Ok(transaction), 138 | Err(e) => bail!("DEPLOY: Error sending out transaction: {e}"), 139 | } 140 | } 141 | 142 | async fn do_invoke(args: InvokeArgs, url: &str) -> Result { 143 | let transaction_type = TransactionType::Invoke { 144 | address: args.address, 145 | function: args.function, 146 | inputs: args.inputs, 147 | }; 148 | 149 | let transaction = Transaction::with_type(transaction_type)?; 150 | let transaction_serialized = bincode::serialize(&transaction)?; 151 | 152 | match broadcast(transaction_serialized, url).await { 153 | Ok(_) => Ok(transaction), 154 | Err(e) => bail!("INVOKE: Error sending out transaction: {e}"), 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /sequencer/src/cli/tendermint.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{bail, ensure, Result}; 2 | use lib::Transaction; 3 | use tendermint_rpc::{query::Query, Client, HttpClient, Order}; 4 | use tracing::debug; 5 | 6 | pub async fn broadcast(transaction: Vec, url: &str) -> Result<()> { 7 | let client = HttpClient::new(url).unwrap(); 8 | 9 | let response = client.broadcast_tx_sync(transaction).await?; 10 | 11 | debug!("Response from CheckTx: {:?}", response); 12 | match response.code { 13 | tendermint::abci::Code::Ok => Ok(()), 14 | tendermint::abci::Code::Err(code) => { 15 | bail!("Error executing transaction {}: {}", code, response.log) 16 | } 17 | } 18 | } 19 | 20 | pub async fn get_transaction(tx_id: &str, url: &str) -> Result { 21 | let client = HttpClient::new(url)?; 22 | // todo: this index key might have to be a part of the shared lib so that both the CLI and the ABCI can be in sync 23 | let query = Query::contains("app.tx_id", tx_id); 24 | 25 | let response = client 26 | .tx_search(query, false, 1, 1, Order::Ascending) 27 | .await?; 28 | 29 | // early return with error if no transaction has been indexed for that tx id 30 | ensure!( 31 | response.total_count > 0, 32 | "Transaction ID {} is invalid or has not yet been committed to the blockchain", 33 | tx_id 34 | ); 35 | 36 | let tx_bytes = response.txs.into_iter().next().unwrap().tx; 37 | let transaction: Transaction = bincode::deserialize(&tx_bytes)?; 38 | 39 | Ok(transaction) 40 | } 41 | -------------------------------------------------------------------------------- /sequencer/src/lib/mod.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{ensure, Result}; 2 | use felt::Felt252; 3 | use num_traits::{Num, Zero}; 4 | use serde::{Deserialize, Serialize}; 5 | use starknet_rs::{ 6 | hash_utils::calculate_contract_address, services::api::contract_class::ContractClass, 7 | utils::Address, 8 | }; 9 | use uuid::Uuid; 10 | 11 | #[derive(Clone, Serialize, Deserialize, Debug)] 12 | pub struct Transaction { 13 | pub transaction_type: TransactionType, 14 | pub transaction_hash: String, 15 | pub id: String, 16 | } 17 | 18 | #[derive(Clone, Serialize, Deserialize, Debug)] 19 | pub enum TransactionType { 20 | /// Create new contract class. 21 | Declare { program: String }, 22 | 23 | /// Create an instance of a contract which will have storage assigned. (Accounts are a contract themselves) 24 | DeployAccount { 25 | class_hash: String, 26 | salt: i32, 27 | inputs: Option>, 28 | }, 29 | 30 | /// Execute a function from a deployed contract. 31 | Invoke { 32 | address: String, 33 | function: String, 34 | inputs: Option>, 35 | }, 36 | } 37 | 38 | impl Transaction { 39 | pub fn with_type(transaction_type: TransactionType) -> Result { 40 | Ok(Transaction { 41 | transaction_hash: transaction_type.compute_and_hash()?, 42 | transaction_type, 43 | id: Uuid::new_v4().to_string(), 44 | }) 45 | } 46 | 47 | /// Verify that the transaction id is consistent with its contents, by checking its sha256 hash. 48 | pub fn assert_integrity(&self) -> Result<()> { 49 | ensure!( 50 | self.transaction_hash == self.transaction_type.compute_and_hash()?, 51 | "Corrupted transaction: Inconsistent transaction id" 52 | ); 53 | 54 | Ok(()) 55 | } 56 | } 57 | 58 | impl TransactionType { 59 | // TODO: Rename this and/or structure the code differently 60 | pub fn compute_and_hash(&self) -> Result { 61 | match self { 62 | TransactionType::Declare { program } => { 63 | let contract_class = ContractClass::try_from(program.as_str())?; 64 | // This function requires cairo_programs/contracts.json to exist as it uses that cairo program to compute the hash 65 | let contract_hash = starknet_rs::core::contract_address::starknet_contract_address::compute_class_hash(&contract_class)?; 66 | Ok(format!( 67 | "{}{}", 68 | "0x", 69 | hex::encode(contract_hash.to_bytes_be()) 70 | )) 71 | } 72 | TransactionType::DeployAccount { 73 | class_hash, 74 | salt, 75 | inputs, 76 | } => { 77 | let constructor_calldata = match &inputs { 78 | Some(vec) => vec.iter().map(|&n| n.into()).collect(), 79 | None => Vec::new(), 80 | }; 81 | 82 | let contract_address = calculate_contract_address( 83 | &Address((*salt).into()), 84 | &felt::Felt252::from_str_radix(&class_hash[2..], 16).unwrap(), // TODO: Handle these errors better 85 | &constructor_calldata, 86 | Address(Felt252::zero()), // TODO: Deployer address is hardcoded to 0 in starknet-in-rust, ask why 87 | )?; 88 | 89 | Ok(format!( 90 | "{}{}", 91 | "0x", 92 | hex::encode(contract_address.to_bytes_be()) 93 | )) 94 | } 95 | TransactionType::Invoke { 96 | address, 97 | function, 98 | inputs, 99 | } => Ok(format!( 100 | "Invoked {function} with inputs {inputs:?} for contract in address {address}" 101 | )), 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /sequencer/tests/client.rs: -------------------------------------------------------------------------------- 1 | use assert_cmd::{assert::Assert, Command}; 2 | use serde::de::DeserializeOwned; 3 | 4 | #[test] 5 | fn deploy_fibonacci() { 6 | let output = client_command(&["declare", "cairo_programs/fibonacci.json"]); 7 | output.unwrap(); // if exit code was not successful, this will panic 8 | 9 | let output = client_command(&["deploy-account", "class_hash"]); 10 | output.unwrap(); 11 | 12 | // todo: attempt invoke 13 | // todo: decide whether: 14 | // - CLI returns class_hash on check_tx, 15 | // - deliver_tx indexes class hash for a specific tx and creates an event for it, 16 | // - deliver_tx indexes class hash for a specific tx and we are able to use the `query` hook, 17 | // 18 | } 19 | 20 | fn client_command(args: &[&str]) -> Result { 21 | let command = &mut Command::cargo_bin("cli").unwrap(); 22 | 23 | command 24 | .args(args) 25 | .assert() 26 | .try_success() 27 | .map(parse_output) 28 | .map_err(|e| e.to_string()) 29 | } 30 | 31 | /// Extract the command assert output and deserialize it as json 32 | fn parse_output(result: Assert) -> T { 33 | let output = String::from_utf8(result.get_output().stdout.to_vec()).unwrap(); 34 | serde_json::from_str(&output).unwrap() 35 | } 36 | --------------------------------------------------------------------------------