├── .github
└── workflows
│ ├── build.yaml
│ └── deploy-testnet.yaml
├── .gitignore
├── Cargo.lock
├── Cargo.toml
├── Makefile
├── README.md
├── cairo_programs
├── Account.json
├── account_without_validation.json
└── contracts.json
├── narwhal-abci
├── .gitignore
├── Cargo.lock
├── Cargo.toml
├── README.md
├── assets
│ └── architecture.png
├── demo
│ ├── .gitignore
│ ├── README.md
│ ├── benchmark
│ │ ├── __init__
│ │ ├── __pycache__
│ │ │ ├── logs.cpython-310.pyc
│ │ │ └── utils.cpython-310.pyc
│ │ ├── aggregate.py
│ │ ├── commands.py
│ │ ├── config.py
│ │ ├── instance.py
│ │ ├── local.py
│ │ ├── logs.py
│ │ ├── plot.py
│ │ ├── remote.py
│ │ ├── settings.py
│ │ └── utils.py
│ ├── cleanup-logs.sh
│ ├── fabfile.py
│ ├── node_params.json
│ ├── poetry.lock
│ └── pyproject.toml
├── narwhal-abci
│ ├── Cargo.toml
│ └── src
│ │ ├── abci_server.rs
│ │ ├── engine.rs
│ │ └── lib.rs
├── node
│ ├── Cargo.toml
│ └── src
│ │ └── main.rs
└── starknet-abci
│ ├── Cargo.toml
│ ├── programs
│ └── fibonacci.json
│ └── src
│ ├── app.rs
│ ├── bin
│ ├── client.rs
│ └── starknet-app.rs
│ ├── lib.rs
│ ├── transaction.rs
│ └── types.rs
└── sequencer
├── .gitignore
├── Cargo.toml
├── LICENSE
├── bench
├── cmd
│ └── load_test
│ │ └── main.go
├── go.mod
├── go.sum
└── pkg
│ └── abci
│ └── client.go
├── cairo_programs
└── contracts.json
├── docker-compose.yml
├── playbooks
└── deploy.yaml
├── programs
├── factorial.cairo
├── factorial.json
├── fibonacci.cairo
└── fibonacci.json
├── prometheus.yml
├── src
├── abci
│ ├── application.rs
│ └── main.rs
├── bench
│ └── main.rs
├── cli
│ ├── main.rs
│ └── tendermint.rs
└── lib
│ └── mod.rs
└── tests
└── client.rs
/.github/workflows/build.yaml:
--------------------------------------------------------------------------------
1 | name: Lambdaworks build checks
2 | on:
3 | push:
4 | branches: [ main ]
5 | pull_request:
6 | branches: [ '*' ]
7 |
8 | concurrency:
9 | group: ${{ github.head_ref || github.run_id }}
10 | cancel-in-progress: true
11 |
12 | jobs:
13 | check:
14 | name: Check
15 | runs-on: ubuntu-latest
16 | steps:
17 | - name: Checkout sources
18 | uses: actions/checkout@v3
19 |
20 | - name: Install stable toolchain
21 | uses: actions-rs/toolchain@v1
22 | with:
23 | profile: minimal
24 | toolchain: stable
25 | override: true
26 |
27 | - name: Run cargo check
28 | uses: actions-rs/cargo@v1
29 | with:
30 | command: check
31 |
32 | fmt:
33 | name: Lints
34 | runs-on: ubuntu-latest
35 | steps:
36 | - name: Checkout sources
37 | uses: actions/checkout@v3
38 |
39 | - name: Install stable toolchain
40 | uses: actions-rs/toolchain@v1
41 | with:
42 | profile: minimal
43 | toolchain: stable
44 | override: true
45 | components: rustfmt, clippy
46 |
47 | - name: Run cargo fmt
48 | uses: actions-rs/cargo@v1
49 | with:
50 | command: fmt
51 | args: --all -- --check
52 |
53 | clippy:
54 | runs-on: ubuntu-latest
55 | steps:
56 | - uses: actions/checkout@v3
57 | - uses: actions-rs/toolchain@v1
58 | with:
59 | toolchain: stable
60 | components: clippy
61 | - run: make clippy
62 |
--------------------------------------------------------------------------------
/.github/workflows/deploy-testnet.yaml:
--------------------------------------------------------------------------------
1 | name: Reset and deploy tendermint testnet
2 | on: workflow_dispatch
3 | jobs:
4 |
5 | deploy:
6 | name: Reset and deploy tendermint testnet
7 | environment:
8 | name: development
9 | runs-on: ubuntu-latest
10 | steps:
11 |
12 | - name: Checkout
13 | uses: actions/checkout@v3.3.0
14 |
15 | - name: Run deploy on all nodes
16 | uses: dawidd6/action-ansible-playbook@v2
17 | with:
18 | # Required, playbook filepath
19 | playbook: deploy.yaml
20 | # Optional, directory where playbooks live
21 | directory: ./ansible/playbooks/
22 | # Optional, SSH private key
23 | key: ${{ secrets.SSH_PRIVATE_KEY }}
24 | # Optional, literal inventory file contents
25 | inventory: |
26 | tendermint-nodes:
27 | hosts:
28 | starknet-0:
29 | ansible_host: "5.9.57.45"
30 | ansible_user: root
31 | ansible_python_interpreter: /usr/bin/python3
32 | ansible_ssh_common_args: '-o StrictHostKeyChecking=no'
33 | starknet-1:
34 | ansible_host: "5.9.57.44"
35 | ansible_user: root
36 | ansible_python_interpreter: /usr/bin/python3
37 | ansible_ssh_common_args: '-o StrictHostKeyChecking=no'
38 | starknet-2:
39 | ansible_host: "5.9.57.89"
40 | ansible_user: root
41 | ansible_python_interpreter: /usr/bin/python3
42 | ansible_ssh_common_args: '-o StrictHostKeyChecking=no'
43 | options: |
44 | --verbose
45 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /bin/
2 | /rollkit-node/rollkit-node
3 | /rollkit-node-bitcoin/rollkit-node-bitcoin
4 | /target
5 | .DS_Store
6 |
7 | abci.height
8 |
--------------------------------------------------------------------------------
/Cargo.toml:
--------------------------------------------------------------------------------
1 | [workspace]
2 | members = ["sequencer"]
3 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | .PHONY: reset abci cli consensus_config consensus_install rollkit_celestia bitcoin celestia
2 |
3 | OS := $(shell uname | tr '[:upper:]' '[:lower:]')
4 |
5 | ifeq ($(shell uname -p), arm)
6 | ARCH=arm64
7 | else
8 | ARCH=amd64
9 | endif
10 |
11 | # By default consensus protocol is CometBFT. Can be overriden with Tendermint
12 | CONSENSUS=cometbft
13 |
14 | ifeq ($(CONSENSUS), tendermint)
15 | CONSENSUS_VERSION=0.34.22
16 | CONSENSUS_HOME=~/.tendermint
17 | else
18 | CONSENSUS=cometbft
19 | CONSENSUS_VERSION=0.37.0
20 | CONSENSUS_HOME=~/.cometbft
21 | endif
22 |
23 | test_make:
24 | echo "CONSENSUS = $(CONSENSUS) version=$(CONSENSUS_VERSION) home=$(CONSENSUS_HOME)"
25 |
26 | # Build the client program and put it in bin/aleo
27 | cli:
28 | mkdir -p bin && cargo build --release && cp target/release/cli bin/cli
29 |
30 | # Installs tendermint for current OS and puts it in bin/
31 | bin/tendermint:
32 | make consensus_install CONSENSUS=tendermint
33 |
34 | # Installs cometbft for current OS and puts it in bin/
35 | bin/cometbft:
36 | make consensus_install CONSENSUS=cometbft
37 |
38 | # Internal phony target to install tendermint/cometbft for an arbitrary OS
39 | consensus_install:
40 | mkdir -p $(CONSENSUS)-install bin && cd $(CONSENSUS)-install &&\
41 | wget https://github.com/$(CONSENSUS)/$(CONSENSUS)/releases/download/v$(CONSENSUS_VERSION)/$(CONSENSUS)_$(CONSENSUS_VERSION)_$(OS)_$(ARCH).tar.gz &&\
42 | tar -xzvf $(CONSENSUS)_$(CONSENSUS_VERSION)_$(OS)_$(ARCH).tar.gz
43 | mv $(CONSENSUS)-install/$(CONSENSUS) bin/ && rm -rf $(CONSENSUS)-install
44 | bin/$(CONSENSUS) init
45 |
46 | # Run a consensus node, installing it if necessary
47 | node: bin/$(CONSENSUS) consensus_config
48 | bin/$(CONSENSUS) node --consensus.create_empty_blocks_interval="30s"
49 |
50 | # Override a tendermint/cometbft node's default configuration. NOTE: we should do something more declarative if we need to update more settings.
51 | consensus_config:
52 | sed -i.bak 's/max_body_bytes = 1000000/max_body_bytes = 12000000/g' $(CONSENSUS_HOME)/config/config.toml
53 | sed -i.bak 's/max_tx_bytes = 1048576/max_tx_bytes = 10485770/g' $(CONSENSUS_HOME)/config/config.toml
54 | sed -i.bak 's#laddr = "tcp://127.0.0.1:26657"#laddr = "tcp://0.0.0.0:26657"#g' $(CONSENSUS_HOME)/config/config.toml
55 | sed -i.bak 's/prometheus = false/prometheus = true/g' $(CONSENSUS_HOME)/config/config.toml
56 |
57 | # remove the blockchain data
58 | reset: bin/$(CONSENSUS)
59 | bin/$(CONSENSUS) unsafe_reset_all
60 | rm -rf abci.height
61 |
62 | # run the Cairo abci application
63 | abci:
64 | cargo run --release --bin abci
65 |
66 | # run tests on release mode (default VM backend) to ensure there is no extra printing to stdout
67 | test:
68 | RUST_BACKTRACE=full cargo test --release -- --nocapture --test-threads=4
69 |
70 | # Initialize the consensus configuration for a localnet of the given amount of validators
71 | localnet: VALIDATORS:=4
72 | localnet: ADDRESS:=127.0.0.1
73 | localnet: HOMEDIR:=localnet
74 | localnet: bin/consensus cli
75 | rm -rf $(HOMEDIR)/
76 | bin/$(CONSENSUS) testnet --v $(VALIDATORS) --o ./$(HOMEDIR) --starting-ip-address $(ADDRESS)
77 | for n in $$(seq 0 $$(($(VALIDATORS)-1))) ; do \
78 | make localnet_config CONSENSUS_HOME=$(HOMEDIR)/node$$n NODE=$$n VALIDATORS=$(VALIDATORS); \
79 | mkdir $(HOMEDIR)/node$$n/abci ; \
80 | done
81 | .PHONY: localnet
82 | # cargo run --bin genesis --release -- $(HOMEDIR)/*
83 |
84 | # run both the abci application and the consensus node
85 | # assumes config for each node has been done previously
86 | localnet_start: NODE:=0
87 | localnet_start: HOMEDIR:=localnet
88 | localnet_start:
89 | bin/$(CONSENSUS) node --home ./$(HOMEDIR)/node$(NODE) &
90 | cd ./$(HOMEDIR)/node$(NODE)/abci; cargo run --release --bin abci -- --port 26$(NODE)58
91 | .PHONY: localnet_start
92 |
93 |
94 | localnet_config:
95 | sed -i.bak 's/max_body_bytes = 1000000/max_body_bytes = 12000000/g' $(CONSENSUS_HOME)/config/config.toml
96 | sed -i.bak 's/max_tx_bytes = 1048576/max_tx_bytes = 10485770/g' $(CONSENSUS_HOME)/config/config.toml
97 | sed -i.bak 's/prometheus = false/prometheus = true/g' $(CONSENSUS_HOME)/config/config.toml
98 | for n in $$(seq 0 $$(($(VALIDATORS)-1))) ; do \
99 | eval "sed -i.bak 's/127.0.0.$$(($${n}+1)):26656/127.0.0.1:26$${n}56/g' $(CONSENSUS_HOME)/config/config.toml" ;\
100 | done
101 | sed -i.bak 's#laddr = "tcp://0.0.0.0:26656"#laddr = "tcp://0.0.0.0:26$(NODE)56"#g' $(CONSENSUS_HOME)/config/config.toml
102 | sed -i.bak 's#laddr = "tcp://127.0.0.1:26657"#laddr = "tcp://0.0.0.0:26$(NODE)57"#g' $(CONSENSUS_HOME)/config/config.toml
103 | sed -i.bak 's#proxy_app = "tcp://127.0.0.1:26658"#proxy_app = "tcp://127.0.0.1:26$(NODE)58"#g' $(CONSENSUS_HOME)/config/config.toml
104 | .PHONY: localnet_config
105 |
106 |
107 | localnet_reset:
108 | bin/$(CONSENSUS) unsafe_reset_all
109 | rm -rf localnet/node*/abci/abci.height;
110 | .PHONY: localnet_reset
111 |
112 | clippy:
113 | cargo clippy --all-targets --all-features -- -D warnings
114 | .PHONY: clippy
115 |
116 | celestia:
117 | (cd local-da; docker compose -f ./docker/test-docker-compose.yml up)
118 |
119 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | _Note: `main` branch contains the development version of the sequencer, for rollups integration please check branch [`rollups`](https://github.com/lambdaclass/starknet_tendermint_sequencer/tree/rollups)_
2 | # Tendermint-based Starknet sequencer
3 |
4 | Sequencer for Starknet based in CometBFT (formerly Tendermint Core) and [starknet-in-rust](https://github.com/lambdaclass/starknet_in_rust).
5 |
6 | ## Getting started
7 |
8 | First, install CometBFT:
9 |
10 | ````
11 | make consensus_install
12 | ````
13 | This will install CometBFT and leave the binary in `bin/comebft`. Additionally, it will run `cometbft init` which creates the required configuration file and stores it in `~/.cometbft/`.
14 |
15 | You can now run the sequencer ABCI with CometBFT by just running those two binaries.
16 |
17 | Run CometBFT node on a terminal:
18 |
19 | ```bash
20 | make node
21 | ```
22 | Run the ABCI sequencer application:
23 |
24 | ```bash
25 | make abci
26 | ```
27 | In order to reset CometBFT's state before rerunning it, make sure you run `make reset`
28 |
29 | ### Sending a transaction
30 |
31 | To send executions to the sequencer you need to have a compiled Cairo program (*.json files in the repo). Then you can send them like so:
32 |
33 | ```bash
34 | cargo run --release execute sequencer/programs/fibonacci.json main
35 | ```
36 |
37 | ### Running Tendermint Core instead of CometBFT
38 |
39 | Current code can be run with both Tendermint and CometBFT (up to version 0.34.27). In order to use Tendermint Core the make command should include the `CONSENSUS` variable:
40 |
41 | ```bash
42 | make node CONSENSUS=tendermint
43 | ```
44 |
45 | This will run Tendermint Core instead of CometBFT (and also will install and configure it if not present).
46 |
47 | ### Benchmark
48 |
49 | You can run a benchmark with
50 |
51 | ```bash
52 | cd sequencer
53 | cargo run --release --bin bench -- --nodes "{list-of-nodes}" --threads 4 --transactions-per-thread 1000
54 | ```
55 |
56 | Where `list-of-nodes` is a list of addresses that are part of the Tendermint network (in the form of `ipaddr:socket`).
57 | The benchmark runs `fibonacci.json` (`fib(500)`), where requests are sent with a round-robin fashion to the list of nodes, through the number of threads you specify, with the amount of transactions per thread you desire.
58 |
59 | #### Example run
60 |
61 | ```bash
62 | > cargo run --release --bin bench -- --nodes "127.0.0.1:26157 127.0.0.1:26057"
63 |
64 | Time it took for all transactions to be delivered: 1308 ms
65 | ```
66 |
67 | Note that this is the time for all transactions to return (ie; validation that they have entered the mempool), but no assumptions can be made in terms of transaction finality.
68 |
69 | ### Benchmarking with Tendermint Load Testing Framework
70 |
71 | There is an alternate way to benchmark the app: using [tm-load-test](https://github.com/informalsystems/tm-load-test). In order to do that there is a load_test command written in go in `/bench` directory. This needs Go v1.12 at least to be built.
72 |
73 | To build it:
74 |
75 | ```bash
76 | cd bench
77 | go build -o ./build/load_test ./cmd/load_test/main.go
78 | ```
79 |
80 | and once built move back to root directory and use
81 |
82 | ```bash
83 | ./bench/build/load_test -c 10 -T 10 -r 1000 -s 250 --broadcast-tx-method async --endpoints ws://localhost:26657/websocket --stats-output result.csv
84 | ```
85 |
86 | to run it against a local tendermint+abci node.
87 |
88 | `-c` is the amount of connectios per endpoint
89 |
90 | `-T` is the amount of seconds to run the test
91 |
92 | `-r` is the rate of tx per second to send
93 |
94 | `-s` is the maximum size of a transaction to be sent.
95 |
96 | Run
97 | ```bash
98 | ./bench/build/load_test -h
99 | ```
100 | to get further information.
101 |
102 | In result.csv there will be a summary of the operation. eg:
103 | ```bash
104 | > cat result.csv
105 | Parameter,Value,Units
106 | total_time,10.875,seconds
107 | total_txs,55249,count
108 | total_bytes,1242747923,bytes
109 | avg_tx_rate,5080.169436,transactions per second
110 | avg_data_rate,114271208.808144,bytes per second
111 | ```
112 |
113 | To run it against a cluster, several nodes can be provided in `--endpoints` parameter. eg:
114 | ```bash
115 | ./bench/build/load_test -c 5 -T 10 -r 1000 -s 250 --broadcast-tx-method async --endpoints ws://5.9.57.45:26657/websocket,ws://5.9.57.44:26657/websocket,ws://5.9.57.89:26657/websocket --stats-output result.csv
116 | ```
117 |
118 | Check [tm-load-test](https://github.com/informalsystems/tm-load-test) and [Tendermint Load Testing Framework](https://github.com/informalsystems/tm-load-test/tree/main/pkg/loadtest) and for more information.
119 |
120 | ## Reference links
121 | * [Starknet sequencer](https://www.starknet.io/de/posts/engineering/starknets-new-sequencer#:~:text=What%20does%20the%20sequencer%20do%3F)
122 | * [Papyrus Starknet full node](https://medium.com/starkware/papyrus-an-open-source-starknet-full-node-396f7cd90202)
123 | * [Blockifier](https://github.com/starkware-libs/blockifier)
124 | * [tendermint-rs](https://github.com/informalsystems/tendermint-rs)
125 | * [ABCI overview](https://docs.tendermint.com/v0.34/introduction/what-is-tendermint.html#abci-overview)
126 | * [ABCI v0.34 reference](https://github.com/tendermint/tendermint/blob/v0.34.x/spec/abci/abci.md)
127 | * [CometBFT](https://github.com/cometbft/cometbft)
128 | * [About why app hash is needed](https://github.com/tendermint/tendermint/issues/1179). Also [this](https://github.com/tendermint/tendermint/blob/v0.34.x/spec/abci/apps.md#query-proofs).
129 | * [About Tendermint 0.34's future](https://github.com/tendermint/tendermint/issues/9972)
130 | ### Starknet
131 | * [Starknet State](https://docs.starknet.io/documentation/architecture_and_concepts/State/starknet-state/)
132 | * [Starknet architecture](https://david-barreto.com/starknets-architecture-review/)
133 | * [Starknet transaction lifecylce](https://docs.starknet.io/documentation/architecture_and_concepts/Blocks/transaction-life-cycle/)
134 |
--------------------------------------------------------------------------------
/narwhal-abci/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 |
--------------------------------------------------------------------------------
/narwhal-abci/Cargo.toml:
--------------------------------------------------------------------------------
1 | [workspace]
2 | members = ["node", "narwhal-abci", "starknet-abci"]
3 |
--------------------------------------------------------------------------------
/narwhal-abci/README.md:
--------------------------------------------------------------------------------
1 | Based on [gakonst's narwhal-abci-evm implementation](https://github.com/gakonst/narwhal-abci-evm/blob/master/evm-abci/src/types.rs)
2 |
3 | # Narwhal & Bullshark x ABCI x Cairo
4 |
5 | Components:
6 | * Reliable stream of hashes of batches of transactions from Bullshark
7 | * Reconstruction of the ledger by querying Narwhal workers' stores for the confirmed batches of transactions
8 | * Delivery of the reconstructed ledger over ABCI to the application logic
9 | * Implementation of a Rust ABCI app using Cairo/Starknet-in-rust
10 |
11 | 
12 |
13 | ## Demo
14 |
15 | Setup/dependencies (from the main folder of the repository):
16 | * [Rust](https://www.rust-lang.org/)
17 | * [Python Poetry](https://python-poetry.org/)
18 | * [tmux](https://github.com/tmux/tmux)
19 | * `cd demo && poetry install`
20 |
21 | Run demo (from the main folder of the repository):
22 | 1. 1st terminal: `cd demo && cargo build && poetry run fab local`
23 | 2. 2nd terminal (after the testbed has started in 1st terminal): `cargo run --bin client`
24 |
25 | The second command will produce output like this:
26 |
27 |
28 | The demo consensus network is run by four nodes (each running on localhost), whose RPC endpoints are reachable on TCP ports 3002, 3009, 3016, and 3023, respectively. There are three accounts, Alice (initially 1.5 ETH), Bob (initially 0 ETH), and Charlie (initially 0 ETH). Alice performs a double spend, sending 1 ETH each to Bob and Charlie in two different transactions that get input to the nodes at ports 3009 and 3016, respectively. Note that only one transaction can make it. Eventually, nodes reach consensus on which transaction gets executed in Foundry's EVM, and the application state is updated in lockstep across all nodes. The update is reflected in subsequent balance queries.
29 |
30 | ## TODOs
31 |
32 | 1. Why does the state transition take a few seconds to get applied?
33 | 2. Can we make this work with Anvil instead of rebuilding a full EVM execution environment?
34 |
--------------------------------------------------------------------------------
/narwhal-abci/assets/architecture.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lambdaclass/starknet_tendermint_sequencer/a29f5ec2267b1caf837230e86283b7db1a92830d/narwhal-abci/assets/architecture.png
--------------------------------------------------------------------------------
/narwhal-abci/demo/.gitignore:
--------------------------------------------------------------------------------
1 | .committee.json
2 | .db-*/
3 | .node-*.json
4 | .parameters.json
5 | benchmark_client
6 | logs/
7 | node
8 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/README.md:
--------------------------------------------------------------------------------
1 | # Running Benchmarks
2 | This document explains how to benchmark the codebase and read benchmarks' results. It also provides a step-by-step tutorial to run benchmarks on [Amazon Web Services (AWS)](https://aws.amazon.com) accross multiple data centers (WAN).
3 |
4 | ## Local Benchmarks
5 | When running benchmarks, the codebase is automatically compiled with the feature flag `benchmark`. This enables the node to print some special log entries that are then read by the python scripts and used to compute performance. These special log entries are clearly indicated with comments in the code: make sure to not alter them (otherwise the benchmark scripts will fail to interpret the logs).
6 |
7 | ### Parametrize the benchmark
8 | After cloning the repo and [installing all dependencies](https://github.com/asonnino/narwhal#quick-start), you can use [Fabric](http://www.fabfile.org/) to run benchmarks on your local machine. Locate the task called `local` in the file [fabfile.py](https://github.com/asonnino/narwhal/blob/master/benchmark/fabfile.py):
9 | ```python
10 | @task
11 | def local(ctx):
12 | ...
13 | ```
14 | The task specifies two types of parameters, the *benchmark parameters* and the *nodes parameters*. The benchmark parameters look as follows:
15 | ```python
16 | bench_params = {
17 | 'nodes': 4,
18 | 'workers': 1,
19 | 'rate': 50_000,
20 | 'tx_size': 512,
21 | 'faults': 0,
22 | 'duration': 20,
23 | }
24 | ```
25 | They specify the number of primaries (`nodes`) and workers per primary (`workers`) to deploy, the input rate (tx/s) at which the clients submits transactions to the system (`rate`), the size of each transaction in bytes (`tx_size`), the number of faulty nodes ('faults), and the duration of the benchmark in seconds (`duration`). The minimum transaction size is 9 bytes, this ensure that the transactions of a client are all different. The benchmarking script will deploy as many clients as workers and divide the input rate equally amongst each client. For instance, if you configure the testbed with 4 nodes, 1 worker per node, and an input rate of 1,000 tx/s (as in the example above), the scripts will deploy 4 clients each submitting transactions to one node at a rate of 250 tx/s. When the parameters `faults` is set to `f > 0`, the last `f` nodes and clients are not booted; the system will thus run with `n-f` nodes (and `n-f` clients).
26 |
27 | The nodes parameters determine the configuration for the primaries and workers:
28 | ```python
29 | node_params = {
30 | 'header_size': 1_000,
31 | 'max_header_delay': 5_000,
32 | 'gc_depth': 50,
33 | 'sync_retry_delay': 10_000,
34 | 'sync_retry_nodes': 3,
35 | 'batch_size': 500_000,
36 | 'max_batch_delay': 100
37 | }
38 | ```
39 | They are defined as follows:
40 | * `header_size`: The preferred header size. The primary creates a new header when it has enough parents and enough batches' digests to reach `header_size`. Denominated in bytes.
41 | * `max_header_delay`: The maximum delay that the primary waits between generating two headers, even if the header did not reach `max_header_size`. Denominated in ms.
42 | * `gc_depth`: The depth of the garbage collection (Denominated in number of rounds).
43 | * `sync_retry_delay`: The delay after which the synchronizer retries to send sync requests. Denominated in ms.
44 | * `sync_retry_nodes`: Determine with how many nodes to sync when re-trying to send sync-request. These nodes are picked at random from the committee.
45 | * `batch_size`: The preferred batch size. The workers seal a batch of transactions when it reaches this size. Denominated in bytes.
46 | * `max_batch_delay`: The delay after which the workers seal a batch of transactions, even if `max_batch_size` is not reached. Denominated in ms.
47 |
48 | ### Run the benchmark
49 | Once you specified both `bench_params` and `node_params` as desired, run:
50 | ```
51 | $ fab local
52 | ```
53 | This command first recompiles your code in `release` mode (and with the `benchmark` feature flag activated), thus ensuring you always benchmark the latest version of your code. This may take a long time the first time you run it. It then generates the configuration files and keys for each node, and runs the benchmarks with the specified parameters. It finally parses the logs and displays a summary of the execution similarly to the one below. All the configuration and key files are hidden JSON files; i.e., their name starts with a dot (`.`), such as `.committee.json`.
54 | ```
55 | -----------------------------------------
56 | SUMMARY:
57 | -----------------------------------------
58 | + CONFIG:
59 | Faults: 0 node(s)
60 | Committee size: 4 node(s)
61 | Worker(s) per node: 1 worker(s)
62 | Collocate primary and workers: True
63 | Input rate: 50,000 tx/s
64 | Transaction size: 512 B
65 | Execution time: 19 s
66 |
67 | Header size: 1,000 B
68 | Max header delay: 100 ms
69 | GC depth: 50 round(s)
70 | Sync retry delay: 10,000 ms
71 | Sync retry nodes: 3 node(s)
72 | batch size: 500,000 B
73 | Max batch delay: 100 ms
74 |
75 | + RESULTS:
76 | Consensus TPS: 46,478 tx/s
77 | Consensus BPS: 23,796,531 B/s
78 | Consensus latency: 464 ms
79 |
80 | End-to-end TPS: 46,149 tx/s
81 | End-to-end BPS: 23,628,541 B/s
82 | End-to-end latency: 557 ms
83 | -----------------------------------------
84 | ```
85 | The 'Consensus TPS' and 'Consensus latency' respectively report the average throughput and latency without considering the client. The consensus latency thus refers to the time elapsed between the block's creation and its commit. In contrast, 'End-to-end TPS' and 'End-to-end latency' report the performance of the whole system, starting from when the client submits the transaction. The end-to-end latency is often called 'client-perceived latency'. To accurately measure this value without degrading performance, the client periodically submits 'sample' transactions that are tracked across all the modules until they get committed into a block; the benchmark scripts use sample transactions to estimate the end-to-end latency.
86 |
87 | ## AWS Benchmarks
88 | This repo integrates various python scripts to deploy and benchmark the codebase on [Amazon Web Services (AWS)](https://aws.amazon.com). They are particularly useful to run benchmarks in the WAN, across multiple data centers. This section provides a step-by-step tutorial explaining how to use them.
89 |
90 | ### Step 1. Set up your AWS credentials
91 | Set up your AWS credentials to enable programmatic access to your account from your local machine. These credentials will authorize your machine to create, delete, and edit instances on your AWS account programmatically. First of all, [find your 'access key id' and 'secret access key'](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-creds). Then, create a file `~/.aws/credentials` with the following content:
92 | ```
93 | [default]
94 | aws_access_key_id = YOUR_ACCESS_KEY_ID
95 | aws_secret_access_key = YOUR_SECRET_ACCESS_KEY
96 | ```
97 | Do not specify any AWS region in that file as the python scripts will allow you to handle multiple regions programmatically.
98 |
99 | ### Step 2. Add your SSH public key to your AWS account
100 | You must now [add your SSH public key to your AWS account](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html). This operation is manual (AWS exposes little APIs to manipulate keys) and needs to be repeated for each AWS region that you plan to use. Upon importing your key, AWS requires you to choose a 'name' for your key; ensure you set the same name on all AWS regions. This SSH key will be used by the python scripts to execute commands and upload/download files to your AWS instances.
101 | If you don't have an SSH key, you can create one using [ssh-keygen](https://www.ssh.com/ssh/keygen/):
102 | ```
103 | $ ssh-keygen -f ~/.ssh/aws
104 | ```
105 |
106 | ### Step 3. Configure the testbed
107 | The file [settings.json](https://github.com/asonnino/narwhal/blob/master/benchmark/settings.json) (located in [narwhal/benchmarks](https://github.com/asonnino/narwhal/blob/master/benchmark)) contains all the configuration parameters of the testbed to deploy. Its content looks as follows:
108 | ```json
109 | {
110 | "key": {
111 | "name": "aws",
112 | "path": "/absolute/key/path"
113 | },
114 | "port": 5000,
115 | "repo": {
116 | "name": "narwhal",
117 | "url": "https://github.com/asonnino/narwhal.git",
118 | "branch": "master"
119 | },
120 | "instances": {
121 | "type": "m5d.8xlarge",
122 | "regions": ["us-east-1", "eu-north-1", "ap-southeast-2", "us-west-1", "ap-northeast-1"]
123 | }
124 | }
125 | ```
126 | The first block (`key`) contains information regarding your SSH key:
127 | ```json
128 | "key": {
129 | "name": "aws",
130 | "path": "/absolute/key/path"
131 | },
132 | ```
133 | Enter the name of your SSH key; this is the name you specified in the AWS web console in step 2. Also, enter the absolute path of your SSH private key (using a relative path won't work).
134 |
135 |
136 | The second block (`ports`) specifies the TCP ports to use:
137 | ```json
138 | "port": 5000,
139 | ```
140 | Narwhal requires a number of TCP ports, depening on the number of workers per node, Each primary requires 2 ports (one to receive messages from other primaties and one to receive messages from its workers), and each worker requires 3 ports (one to receive client transactions, one to receive messages from its primary, and one to receive messages from other workers). Note that the script will open a large port range (5000-7000) to the WAN on all your AWS instances.
141 |
142 | The third block (`repo`) contains the information regarding the repository's name, the URL of the repo, and the branch containing the code to deploy:
143 | ```json
144 | "repo": {
145 | "name": "narwhal",
146 | "url": "https://github.com/asonnino/narwhal.git",
147 | "branch": "master"
148 | },
149 | ```
150 | Remember to update the `url` field to the name of your repo. Modifying the branch name is particularly useful when testing new functionalities without having to checkout the code locally.
151 |
152 | The the last block (`instances`) specifies the [AWS instance type](https://aws.amazon.com/ec2/instance-types) and the [AWS regions](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) to use:
153 | ```json
154 | "instances": {
155 | "type": "m5d.8xlarge",
156 | "regions": ["us-east-1", "eu-north-1", "ap-southeast-2", "us-west-1", "ap-northeast-1"]
157 | }
158 | ```
159 | The instance type selects the hardware on which to deploy the testbed. For example, `m5d.8xlarge` instances come with 32 vCPUs (16 physical cores), 128 GB of RAM, and guarantee 10 Gbps of bandwidth. The python scripts will configure each instance with 300 GB of SSD hard drive. The `regions` field specifies the data centers to use. If you require more nodes than data centers, the python scripts will distribute the nodes as equally as possible amongst the data centers. All machines run a fresh install of Ubuntu Server 20.04.
160 |
161 | ### Step 4. Create a testbed
162 | The AWS instances are orchestrated with [Fabric](http://www.fabfile.org) from the file [fabfile.py](https://github.com/asonnino/narwhal/blob/master/benchmark/fabfile.pyy) (located in [narwhal/benchmarks](https://github.com/asonnino/narwhal/blob/master/benchmark)); you can list all possible commands as follows:
163 | ```
164 | $ cd narwhal/benchmark
165 | $ fab --list
166 | ```
167 | The command `fab create` creates new AWS instances; open [fabfile.py](https://github.com/asonnino/narwhal/blob/master/benchmark/fabfile.py) and locate the `create` task:
168 | ```python
169 | @task
170 | def create(ctx, nodes=2):
171 | ...
172 | ```
173 | The parameter `nodes` determines how many instances to create in *each* AWS region. That is, if you specified 5 AWS regions as in the example of step 3, setting `nodes=2` will creates a total of 10 machines:
174 | ```
175 | $ fab create
176 |
177 | Creating 10 instances |██████████████████████████████| 100.0%
178 | Waiting for all instances to boot...
179 | Successfully created 10 new instances
180 | ```
181 | You can then clone the repo and install rust on the remote instances with `fab install`:
182 | ```
183 | $ fab install
184 |
185 | Installing rust and cloning the repo...
186 | Initialized testbed of 10 nodes
187 | ```
188 | This may take a long time as the command will first update all instances.
189 | The commands `fab stop` and `fab start` respectively stop and start the testbed without destroying it (it is good practice to stop the testbed when not in use as AWS can be quite expensive); and `fab destroy` terminates all instances and destroys the testbed. Note that, depending on the instance types, AWS instances may take up to several minutes to fully start or stop. The command `fab info` displays a nice summary of all available machines and information to manually connect to them (for debug).
190 |
191 | ### Step 5. Run a benchmark
192 | After setting up the testbed, running a benchmark on AWS is similar to running it locally (see [Run Local Benchmarks](https://github.com/asonnino/narwhal/tree/master/benchmark#local-benchmarks)). Locate the task `remote` in [fabfile.py](https://github.com/asonnino/narwhal/blob/master/benchmark/fabfile.py):
193 | ```python
194 | @task
195 | def remote(ctx):
196 | ...
197 | ```
198 | The benchmark parameters are similar to [local benchmarks](https://github.com/asonnino/narwhal/tree/master/benchmark#local-benchmarks) but allow to specify the number of nodes and the input rate as arrays to automate multiple benchmarks with a single command. The parameter `runs` specifies the number of times to repeat each benchmark (to later compute the average and stdev of the results), and the parameter `collocate` specifies whether to collocate all the node's workers and the primary on the same machine. If `collocate` is set to `False`, the script will run one node per data center (AWS region), with its primary and each of its worker running on a dedicated instance.
199 | ```python
200 | bench_params = {
201 | 'nodes': [10, 20, 30],
202 | 'workers: 2,
203 | 'collocate': True,
204 | 'rate': [20_000, 30_000, 40_000],
205 | 'tx_size': 512,
206 | 'faults': 0,
207 | 'duration': 300,
208 | 'runs': 2,
209 | }
210 | ```
211 | Similarly to local benchmarks, the scripts will deploy as many clients as workers and divide the input rate equally amongst each client. Each client is colocated with a worker, and only submit transactions to the worker with whom they share the machine.
212 |
213 | Once you specified both `bench_params` and `node_params` as desired, run:
214 | ```
215 | $ fab remote
216 | ```
217 | This command first updates all machines with the latest commit of the GitHub repo and branch specified in your file [settings.json](https://github.com/asonnino/narwhal/blob/master/benchmark/settings.json) (step 3); this ensures that benchmarks are always run with the latest version of the code. It then generates and uploads the configuration files to each machine, runs the benchmarks with the specified parameters, and downloads the logs. It finally parses the logs and prints the results into a folder called `results` (which is automatically created if it doesn't already exists). You can run `fab remote` multiple times without fearing to override previous results, the command either appends new results to a file containing existing ones or prints them in separate files. If anything goes wrong during a benchmark, you can always stop it by running `fab kill`.
218 |
219 | ### Step 6. Plot the results
220 | Once you have enough results, you can aggregate and plot them:
221 | ```
222 | $ fab plot
223 | ```
224 | This command creates a latency graph, a throughput graph, and a robustness graph in a folder called `plots` (which is automatically created if it doesn't already exists). You can adjust the plot parameters to filter which curves to add to the plot:
225 | ```python
226 | plot_params = {
227 | 'faults': [0],
228 | 'nodes': [10, 20, 50],
229 | 'workers': [1],
230 | 'collocate': True,
231 | 'tx_size': 512,
232 | 'max_latency': [3_500, 4_500]
233 | }
234 | ```
235 |
236 | The first graph ('latency') plots the latency versus the throughput. It shows that the latency is low until a fairly neat threshold after which it drastically increases. Determining this threshold is crucial to understand the limits of the system.
237 |
238 | Another challenge is comparing apples-to-apples between different deployments of the system. The challenge here is again that latency and throughput are interdependent, as a result a throughput/number of nodes chart could be tricky to produce fairly. The way to do it is to define a maximum latency and measure the throughput at this point instead of simply pushing every system to its peak throughput (where latency is meaningless). The second graph ('tps') plots the maximum achievable throughput under a maximum latency for different numbers of nodes.
239 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/benchmark/__init__:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lambdaclass/starknet_tendermint_sequencer/a29f5ec2267b1caf837230e86283b7db1a92830d/narwhal-abci/demo/benchmark/__init__
--------------------------------------------------------------------------------
/narwhal-abci/demo/benchmark/__pycache__/logs.cpython-310.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lambdaclass/starknet_tendermint_sequencer/a29f5ec2267b1caf837230e86283b7db1a92830d/narwhal-abci/demo/benchmark/__pycache__/logs.cpython-310.pyc
--------------------------------------------------------------------------------
/narwhal-abci/demo/benchmark/__pycache__/utils.cpython-310.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lambdaclass/starknet_tendermint_sequencer/a29f5ec2267b1caf837230e86283b7db1a92830d/narwhal-abci/demo/benchmark/__pycache__/utils.cpython-310.pyc
--------------------------------------------------------------------------------
/narwhal-abci/demo/benchmark/aggregate.py:
--------------------------------------------------------------------------------
1 | # Copyright(C) Facebook, Inc. and its affiliates.
2 | from re import search
3 | from collections import defaultdict
4 | from statistics import mean, stdev
5 | from glob import glob
6 | from copy import deepcopy
7 | from os.path import join
8 | import os
9 |
10 | from benchmark.utils import PathMaker
11 |
12 |
13 | class Setup:
14 | def __init__(self, faults, nodes, workers, collocate, rate, tx_size):
15 | self.nodes = nodes
16 | self.workers = workers
17 | self.collocate = collocate
18 | self.rate = rate
19 | self.tx_size = tx_size
20 | self.faults = faults
21 | self.max_latency = 'any'
22 |
23 | def __str__(self):
24 | return (
25 | f' Faults: {self.faults}\n'
26 | f' Committee size: {self.nodes}\n'
27 | f' Workers per node: {self.workers}\n'
28 | f' Collocate primary and workers: {self.collocate}\n'
29 | f' Input rate: {self.rate} tx/s\n'
30 | f' Transaction size: {self.tx_size} B\n'
31 | f' Max latency: {self.max_latency} ms\n'
32 | )
33 |
34 | def __eq__(self, other):
35 | return isinstance(other, Setup) and str(self) == str(other)
36 |
37 | def __hash__(self):
38 | return hash(str(self))
39 |
40 | @classmethod
41 | def from_str(cls, raw):
42 | faults = int(search(r'Faults: (\d+)', raw).group(1))
43 | nodes = int(search(r'Committee size: (\d+)', raw).group(1))
44 | workers = int(search(r'Worker\(s\) per node: (\d+)', raw).group(1))
45 | collocate = 'True' == search(
46 | r'Collocate primary and workers: (True|False)', raw
47 | ).group(1)
48 | rate = int(search(r'Input rate: (\d+)', raw).group(1))
49 | tx_size = int(search(r'Transaction size: (\d+)', raw).group(1))
50 | return cls(faults, nodes, workers, collocate, rate, tx_size)
51 |
52 |
53 | class Result:
54 | def __init__(self, mean_tps, mean_latency, std_tps=0, std_latency=0):
55 | self.mean_tps = mean_tps
56 | self.mean_latency = mean_latency
57 | self.std_tps = std_tps
58 | self.std_latency = std_latency
59 |
60 | def __str__(self):
61 | return(
62 | f' TPS: {self.mean_tps} +/- {self.std_tps} tx/s\n'
63 | f' Latency: {self.mean_latency} +/- {self.std_latency} ms\n'
64 | )
65 |
66 | @classmethod
67 | def from_str(cls, raw):
68 | tps = int(search(r'End-to-end TPS: (\d+)', raw).group(1))
69 | latency = int(search(r'End-to-end latency: (\d+)', raw).group(1))
70 | return cls(tps, latency)
71 |
72 | @classmethod
73 | def aggregate(cls, results):
74 | if len(results) == 1:
75 | return results[0]
76 |
77 | mean_tps = round(mean([x.mean_tps for x in results]))
78 | mean_latency = round(mean([x.mean_latency for x in results]))
79 | std_tps = round(stdev([x.mean_tps for x in results]))
80 | std_latency = round(stdev([x.mean_latency for x in results]))
81 | return cls(mean_tps, mean_latency, std_tps, std_latency)
82 |
83 |
84 | class LogAggregator:
85 | def __init__(self, max_latencies):
86 | assert isinstance(max_latencies, list)
87 | assert all(isinstance(x, int) for x in max_latencies)
88 |
89 | self.max_latencies = max_latencies
90 |
91 | data = ''
92 | for filename in glob(join(PathMaker.results_path(), '*.txt')):
93 | with open(filename, 'r') as f:
94 | data += f.read()
95 |
96 | records = defaultdict(list)
97 | for chunk in data.replace(',', '').split('SUMMARY')[1:]:
98 | if chunk:
99 | records[Setup.from_str(chunk)] += [Result.from_str(chunk)]
100 |
101 | self.records = {k: Result.aggregate(v) for k, v in records.items()}
102 |
103 | def print(self):
104 | if not os.path.exists(PathMaker.plots_path()):
105 | os.makedirs(PathMaker.plots_path())
106 |
107 | results = [
108 | self._print_latency(),
109 | self._print_tps(scalability=False),
110 | self._print_tps(scalability=True),
111 | ]
112 | for name, records in results:
113 | for setup, values in records.items():
114 | data = '\n'.join(
115 | f' Variable value: X={x}\n{y}' for x, y in values
116 | )
117 | string = (
118 | '\n'
119 | '-----------------------------------------\n'
120 | ' RESULTS:\n'
121 | '-----------------------------------------\n'
122 | f'{setup}'
123 | '\n'
124 | f'{data}'
125 | '-----------------------------------------\n'
126 | )
127 |
128 | max_lat = setup.max_latency
129 | filename = PathMaker.agg_file(
130 | name,
131 | setup.faults,
132 | setup.nodes,
133 | setup.workers,
134 | setup.collocate,
135 | setup.rate,
136 | setup.tx_size,
137 | max_latency=None if max_lat == 'any' else max_lat,
138 | )
139 | with open(filename, 'w') as f:
140 | f.write(string)
141 |
142 | def _print_latency(self):
143 | records = deepcopy(self.records)
144 | organized = defaultdict(list)
145 | for setup, result in records.items():
146 | rate = setup.rate
147 | setup.rate = 'any'
148 | organized[setup] += [(result.mean_tps, result, rate)]
149 |
150 | for setup, results in list(organized.items()):
151 | results.sort(key=lambda x: x[2])
152 | organized[setup] = [(x, y) for x, y, _ in results]
153 |
154 | return 'latency', organized
155 |
156 | def _print_tps(self, scalability):
157 | records = deepcopy(self.records)
158 | organized = defaultdict(list)
159 | for max_latency in self.max_latencies:
160 | for setup, result in records.items():
161 | setup = deepcopy(setup)
162 | if result.mean_latency <= max_latency:
163 | setup.rate = 'any'
164 | setup.max_latency = max_latency
165 | if scalability:
166 | variable = setup.workers
167 | setup.workers = 'x'
168 | else:
169 | variable = setup.nodes
170 | setup.nodes = 'x'
171 |
172 | new_point = all(variable != x[0] for x in organized[setup])
173 | highest_tps = False
174 | for v, r in organized[setup]:
175 | if result.mean_tps > r.mean_tps and variable == v:
176 | organized[setup].remove((v, r))
177 | highest_tps = True
178 | if new_point or highest_tps:
179 | organized[setup] += [(variable, result)]
180 |
181 | [v.sort(key=lambda x: x[0]) for v in organized.values()]
182 | return 'tps', organized
183 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/benchmark/commands.py:
--------------------------------------------------------------------------------
1 | # Copyright(C) Facebook, Inc. and its affiliates.
2 | from os.path import join
3 | import os
4 |
5 | from benchmark.utils import PathMaker
6 |
7 | NODE = "./../target/debug/node"
8 |
9 |
10 | class CommandMaker:
11 |
12 | @staticmethod
13 | def cleanup():
14 | return (
15 | f'rm -r .db-* ; rm .*.json ; mkdir -p {PathMaker.results_path()}'
16 | )
17 |
18 | @staticmethod
19 | def clean_logs():
20 | return f'rm -r {PathMaker.logs_path()} ; mkdir -p {PathMaker.logs_path()}'
21 |
22 | @staticmethod
23 | def compile():
24 | return 'cargo build'
25 | # return 'cargo build --quiet --release --features benchmark'
26 |
27 | @staticmethod
28 | def generate_key(filename):
29 | assert isinstance(filename, str)
30 | return f'{NODE} generate_keys --filename {filename}'
31 |
32 | @staticmethod
33 | def run_primary(keys, committee, store, parameters, app_api, abci_api, debug=False):
34 | print(store, keys)
35 | assert isinstance(keys, str)
36 | assert isinstance(committee, str)
37 | assert isinstance(parameters, str)
38 | assert isinstance(debug, bool)
39 | v = '-vvv' if debug else '-vv'
40 | return (f'{NODE} {v} run --keys {keys} --committee {committee} '
41 | f'--store {store} --parameters {parameters} primary --app-api {app_api} --abci-api {abci_api} ')
42 |
43 | @staticmethod
44 | def run_worker(keys, committee, store, parameters, id, debug=False):
45 | assert isinstance(keys, str)
46 | assert isinstance(committee, str)
47 | assert isinstance(parameters, str)
48 | assert isinstance(debug, bool)
49 | v = '-vvv' if debug else '-vv'
50 | return (f'{NODE} {v} run --keys {keys} --committee {committee} '
51 | f'--store {store} --parameters {parameters} worker --id {id}')
52 |
53 | @staticmethod
54 | def run_client(address, size, rate, nodes):
55 | assert isinstance(address, str)
56 | assert isinstance(size, int) and size > 0
57 | assert isinstance(rate, int) and rate >= 0
58 | assert isinstance(nodes, list)
59 | assert all(isinstance(x, str) for x in nodes)
60 | nodes = f'--nodes {" ".join(nodes)}' if nodes else ''
61 | return f'./../target/debug/benchmark_client {address} --size {size} --rate {rate} {nodes}'
62 |
63 | @staticmethod
64 | def run_app(listen_on):
65 | assert isinstance(listen_on, str)
66 | return f'../target/debug/starknet-app --demo {listen_on}'
67 |
68 | @staticmethod
69 | def kill():
70 | print("os.getenv('TMUX'):", os.getenv('TMUX'))
71 | if os.getenv('TMUX'):
72 | # running within tmux (Georgios' config)
73 | # kill all other sessions
74 | return "tmux kill-session -a"
75 | else:
76 | # running without tmux (Joachim's config)
77 | # This does not work when running in Tmux
78 | return 'tmux kill-server'
79 |
80 | @staticmethod
81 | def alias_binaries(origin):
82 | assert isinstance(origin, str)
83 | # This is aliasing only the release
84 | # print('Origin', origin)
85 | node, client = join(origin, 'node'), join(origin, 'benchmark_client')
86 | return f'rm node ; rm benchmark_client ; ln -s {node} . ; ln -s {client} .'
87 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/benchmark/config.py:
--------------------------------------------------------------------------------
1 | # Copyright(C) Facebook, Inc. and its affiliates.
2 | from json import dump, load
3 | from collections import OrderedDict
4 |
5 |
6 | class ConfigError(Exception):
7 | pass
8 |
9 |
10 | class Key:
11 | def __init__(self, name, secret):
12 | self.name = name
13 | self.secret = secret
14 |
15 | @classmethod
16 | def from_file(cls, filename):
17 | assert isinstance(filename, str)
18 | with open(filename, 'r') as f:
19 | data = load(f)
20 | return cls(data['name'], data['secret'])
21 |
22 |
23 | class Committee:
24 | ''' The committee looks as follows:
25 | "authorities: {
26 | "name": {
27 | "stake": 1,
28 | "primary: {
29 | "primary_to_primary": x.x.x.x:x,
30 | "worker_to_primary": x.x.x.x:x,
31 | },
32 | "workers": {
33 | "0": {
34 | "primary_to_worker": x.x.x.x:x,
35 | "worker_to_worker": x.x.x.x:x,
36 | "transactions": x.x.x.x:x
37 | },
38 | ...
39 | }
40 | },
41 | ...
42 | }
43 | '''
44 |
45 | def __init__(self, addresses, base_port):
46 | ''' The `addresses` field looks as follows:
47 | {
48 | "name": ["host", "host", ...],
49 | ...
50 | }
51 | '''
52 | assert isinstance(addresses, OrderedDict)
53 | assert all(isinstance(x, str) for x in addresses.keys())
54 | assert all(
55 | isinstance(x, list) and len(x) > 1 for x in addresses.values()
56 | )
57 | assert all(
58 | isinstance(x, str) for y in addresses.values() for x in y
59 | )
60 | assert len({len(x) for x in addresses.values()}) == 1
61 | assert isinstance(base_port, int) and base_port > 1024
62 |
63 | port = base_port
64 | self.json = {'authorities': OrderedDict()}
65 | for name, hosts in addresses.items():
66 | host = hosts.pop(0)
67 | primary_addr = {
68 | 'primary_to_primary': f'{host}:{port}',
69 | 'worker_to_primary': f'{host}:{port + 1}',
70 |
71 | # new addresses for APIs
72 | 'api_rpc': f'{host}:{port + 2}',
73 | 'api_abci': f'{host}:{port + 3}',
74 | }
75 | port += 4
76 |
77 | workers_addr = OrderedDict()
78 | for j, host in enumerate(hosts):
79 | workers_addr[j] = {
80 | 'primary_to_worker': f'{host}:{port}',
81 | 'transactions': f'{host}:{port + 1}',
82 | 'worker_to_worker': f'{host}:{port + 2}',
83 | }
84 | port += 3
85 |
86 | self.json['authorities'][name] = {
87 | 'stake': 1,
88 | 'primary': primary_addr,
89 | 'workers': workers_addr
90 | }
91 |
92 | def primary_addresses(self, faults=0):
93 | ''' Returns an ordered list of primaries' addresses. '''
94 | assert faults < self.size()
95 | addresses = []
96 | good_nodes = self.size() - faults
97 | for authority in list(self.json['authorities'].values())[:good_nodes]:
98 | addresses += [authority['primary']['primary_to_primary']]
99 | return addresses
100 |
101 | def rpc_addresses(self, faults=0):
102 | ''' Returns an ordered list of rpcs' addresses. '''
103 | assert faults < self.size()
104 | addresses = []
105 | good_nodes = self.size() - faults
106 | for authority in list(self.json['authorities'].values())[:good_nodes]:
107 | addresses += [authority['primary']['api_rpc']]
108 | return addresses
109 |
110 | def app_addresses(self, faults=0):
111 | ''' Returns an ordered list of apps' addresses. '''
112 | assert faults < self.size()
113 | addresses = []
114 | good_nodes = self.size() - faults
115 | for authority in list(self.json['authorities'].values())[:good_nodes]:
116 | addresses += [authority['primary']['api_abci']]
117 | return addresses
118 |
119 | def workers_addresses(self, faults=0):
120 | ''' Returns an ordered list of list of workers' addresses. '''
121 | assert faults < self.size()
122 | addresses = []
123 | good_nodes = self.size() - faults
124 | for authority in list(self.json['authorities'].values())[:good_nodes]:
125 | authority_addresses = []
126 | for id, worker in authority['workers'].items():
127 | authority_addresses += [(id, worker['transactions'])]
128 | addresses.append(authority_addresses)
129 | return addresses
130 |
131 | def ips(self, name=None):
132 | ''' Returns all the ips associated with an authority (in any order). '''
133 | if name is None:
134 | names = list(self.json['authorities'].keys())
135 | else:
136 | names = [name]
137 |
138 | ips = set()
139 | for name in names:
140 | addresses = self.json['authorities'][name]['primary']
141 | ips.add(self.ip(addresses['primary_to_primary']))
142 | ips.add(self.ip(addresses['worker_to_primary']))
143 |
144 | for worker in self.json['authorities'][name]['workers'].values():
145 | ips.add(self.ip(worker['primary_to_worker']))
146 | ips.add(self.ip(worker['worker_to_worker']))
147 | ips.add(self.ip(worker['transactions']))
148 |
149 | return list(ips)
150 |
151 | def remove_nodes(self, nodes):
152 | ''' remove the `nodes` last nodes from the committee. '''
153 | assert nodes < self.size()
154 | for _ in range(nodes):
155 | self.json['authorities'].popitem()
156 |
157 | def size(self):
158 | ''' Returns the number of authorities. '''
159 | return len(self.json['authorities'])
160 |
161 | def workers(self):
162 | ''' Returns the total number of workers (all authorities altogether). '''
163 | return sum(len(x['workers']) for x in self.json['authorities'].values())
164 |
165 | def print(self, filename):
166 | print(filename)
167 | assert isinstance(filename, str)
168 | with open(filename, 'w') as f:
169 | dump(self.json, f, indent=4, sort_keys=True)
170 |
171 | @staticmethod
172 | def ip(address):
173 | assert isinstance(address, str)
174 | return address.split(':')[0]
175 |
176 |
177 | class LocalCommittee(Committee):
178 | def __init__(self, names, port, workers):
179 | assert isinstance(names, list)
180 | assert all(isinstance(x, str) for x in names)
181 | assert isinstance(port, int)
182 | assert isinstance(workers, int) and workers > 0
183 | addresses = OrderedDict((x, ['127.0.0.1']*(1+workers)) for x in names)
184 | super().__init__(addresses, port)
185 |
186 |
187 | class NodeParameters:
188 | def __init__(self, json):
189 | inputs = []
190 | try:
191 | inputs += [json['header_size']]
192 | inputs += [json['max_header_delay']]
193 | inputs += [json['gc_depth']]
194 | inputs += [json['sync_retry_delay']]
195 | inputs += [json['sync_retry_nodes']]
196 | inputs += [json['batch_size']]
197 | inputs += [json['max_batch_delay']]
198 | except KeyError as e:
199 | raise ConfigError(f'Malformed parameters: missing key {e}')
200 |
201 | if not all(isinstance(x, int) for x in inputs):
202 | raise ConfigError('Invalid parameters type')
203 |
204 | self.json = json
205 |
206 | def print(self, filename):
207 | assert isinstance(filename, str)
208 | with open(filename, 'w') as f:
209 | dump(self.json, f, indent=4, sort_keys=True)
210 |
211 |
212 | class BenchParameters:
213 | def __init__(self, json):
214 | try:
215 | self.faults = int(json['faults'])
216 |
217 | nodes = json['nodes']
218 | nodes = nodes if isinstance(nodes, list) else [nodes]
219 | if not nodes or any(x <= 1 for x in nodes):
220 | raise ConfigError('Missing or invalid number of nodes')
221 | self.nodes = [int(x) for x in nodes]
222 |
223 | rate = json['rate']
224 | rate = rate if isinstance(rate, list) else [rate]
225 | if not rate:
226 | raise ConfigError('Missing input rate')
227 | self.rate = [int(x) for x in rate]
228 |
229 |
230 | self.workers = int(json['workers'])
231 |
232 | if 'collocate' in json:
233 | self.collocate = bool(json['collocate'])
234 | else:
235 | self.collocate = True
236 |
237 | self.tx_size = int(json['tx_size'])
238 |
239 | self.duration = int(json['duration'])
240 |
241 | self.runs = int(json['runs']) if 'runs' in json else 1
242 | except KeyError as e:
243 | raise ConfigError(f'Malformed bench parameters: missing key {e}')
244 |
245 | except ValueError:
246 | raise ConfigError('Invalid parameters type')
247 |
248 | if min(self.nodes) <= self.faults:
249 | raise ConfigError('There should be more nodes than faults')
250 |
251 |
252 | class PlotParameters:
253 | def __init__(self, json):
254 | try:
255 | faults = json['faults']
256 | faults = faults if isinstance(faults, list) else [faults]
257 | self.faults = [int(x) for x in faults] if faults else [0]
258 |
259 | nodes = json['nodes']
260 | nodes = nodes if isinstance(nodes, list) else [nodes]
261 | if not nodes:
262 | raise ConfigError('Missing number of nodes')
263 | self.nodes = [int(x) for x in nodes]
264 |
265 | workers = json['workers']
266 | workers = workers if isinstance(workers, list) else [workers]
267 | if not workers:
268 | raise ConfigError('Missing number of workers')
269 | self.workers = [int(x) for x in workers]
270 |
271 | if 'collocate' in json:
272 | self.collocate = bool(json['collocate'])
273 | else:
274 | self.collocate = True
275 |
276 | self.tx_size = int(json['tx_size'])
277 |
278 | max_lat = json['max_latency']
279 | max_lat = max_lat if isinstance(max_lat, list) else [max_lat]
280 | if not max_lat:
281 | raise ConfigError('Missing max latency')
282 | self.max_latency = [int(x) for x in max_lat]
283 |
284 | except KeyError as e:
285 | raise ConfigError(f'Malformed bench parameters: missing key {e}')
286 |
287 | except ValueError:
288 | raise ConfigError('Invalid parameters type')
289 |
290 | if len(self.nodes) > 1 and len(self.workers) > 1:
291 | raise ConfigError(
292 | 'Either the "nodes" or the "workers can be a list (not both)'
293 | )
294 |
295 | def scalability(self):
296 | return len(self.workers) > 1
297 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/benchmark/instance.py:
--------------------------------------------------------------------------------
1 | # Copyright(C) Facebook, Inc. and its affiliates.
2 | import boto3
3 | from botocore.exceptions import ClientError
4 | from collections import defaultdict, OrderedDict
5 | from time import sleep
6 |
7 | from benchmark.utils import Print, BenchError, progress_bar
8 | from benchmark.settings import Settings, SettingsError
9 |
10 |
11 | class AWSError(Exception):
12 | def __init__(self, error):
13 | assert isinstance(error, ClientError)
14 | self.message = error.response['Error']['Message']
15 | self.code = error.response['Error']['Code']
16 | super().__init__(self.message)
17 |
18 |
19 | class InstanceManager:
20 | INSTANCE_NAME = 'dag-node'
21 | SECURITY_GROUP_NAME = 'dag'
22 |
23 | def __init__(self, settings):
24 | assert isinstance(settings, Settings)
25 | self.settings = settings
26 | self.clients = OrderedDict()
27 | for region in settings.aws_regions:
28 | self.clients[region] = boto3.client('ec2', region_name=region)
29 |
30 | @classmethod
31 | def make(cls, settings_file='settings.json'):
32 | try:
33 | return cls(Settings.load(settings_file))
34 | except SettingsError as e:
35 | raise BenchError('Failed to load settings', e)
36 |
37 | def _get(self, state):
38 | # Possible states are: 'pending', 'running', 'shutting-down',
39 | # 'terminated', 'stopping', and 'stopped'.
40 | ids, ips = defaultdict(list), defaultdict(list)
41 | for region, client in self.clients.items():
42 | r = client.describe_instances(
43 | Filters=[
44 | {
45 | 'Name': 'tag:Name',
46 | 'Values': [self.INSTANCE_NAME]
47 | },
48 | {
49 | 'Name': 'instance-state-name',
50 | 'Values': state
51 | }
52 | ]
53 | )
54 | instances = [y for x in r['Reservations'] for y in x['Instances']]
55 | for x in instances:
56 | ids[region] += [x['InstanceId']]
57 | if 'PublicIpAddress' in x:
58 | ips[region] += [x['PublicIpAddress']]
59 | return ids, ips
60 |
61 | def _wait(self, state):
62 | # Possible states are: 'pending', 'running', 'shutting-down',
63 | # 'terminated', 'stopping', and 'stopped'.
64 | while True:
65 | sleep(1)
66 | ids, _ = self._get(state)
67 | if sum(len(x) for x in ids.values()) == 0:
68 | break
69 |
70 | def _create_security_group(self, client):
71 | client.create_security_group(
72 | Description='HotStuff node',
73 | GroupName=self.SECURITY_GROUP_NAME,
74 | )
75 |
76 | client.authorize_security_group_ingress(
77 | GroupName=self.SECURITY_GROUP_NAME,
78 | IpPermissions=[
79 | {
80 | 'IpProtocol': 'tcp',
81 | 'FromPort': 22,
82 | 'ToPort': 22,
83 | 'IpRanges': [{
84 | 'CidrIp': '0.0.0.0/0',
85 | 'Description': 'Debug SSH access',
86 | }],
87 | 'Ipv6Ranges': [{
88 | 'CidrIpv6': '::/0',
89 | 'Description': 'Debug SSH access',
90 | }],
91 | },
92 | {
93 | 'IpProtocol': 'tcp',
94 | 'FromPort': self.settings.base_port,
95 | 'ToPort': self.settings.base_port + 2_000,
96 | 'IpRanges': [{
97 | 'CidrIp': '0.0.0.0/0',
98 | 'Description': 'Dag port',
99 | }],
100 | 'Ipv6Ranges': [{
101 | 'CidrIpv6': '::/0',
102 | 'Description': 'Dag port',
103 | }],
104 | }
105 | ]
106 | )
107 |
108 | def _get_ami(self, client):
109 | # The AMI changes with regions.
110 | response = client.describe_images(
111 | Filters=[{
112 | 'Name': 'description',
113 | 'Values': ['Canonical, Ubuntu, 20.04 LTS, amd64 focal image build on 2020-10-26']
114 | }]
115 | )
116 | return response['Images'][0]['ImageId']
117 |
118 | def create_instances(self, instances):
119 | assert isinstance(instances, int) and instances > 0
120 |
121 | # Create the security group in every region.
122 | for client in self.clients.values():
123 | try:
124 | self._create_security_group(client)
125 | except ClientError as e:
126 | error = AWSError(e)
127 | if error.code != 'InvalidGroup.Duplicate':
128 | raise BenchError('Failed to create security group', error)
129 |
130 | try:
131 | # Create all instances.
132 | size = instances * len(self.clients)
133 | progress = progress_bar(
134 | self.clients.values(), prefix=f'Creating {size} instances'
135 | )
136 | for client in progress:
137 | client.run_instances(
138 | ImageId=self._get_ami(client),
139 | InstanceType=self.settings.instance_type,
140 | KeyName=self.settings.key_name,
141 | MaxCount=instances,
142 | MinCount=instances,
143 | SecurityGroups=[self.SECURITY_GROUP_NAME],
144 | TagSpecifications=[{
145 | 'ResourceType': 'instance',
146 | 'Tags': [{
147 | 'Key': 'Name',
148 | 'Value': self.INSTANCE_NAME
149 | }]
150 | }],
151 | EbsOptimized=True,
152 | BlockDeviceMappings=[{
153 | 'DeviceName': '/dev/sda1',
154 | 'Ebs': {
155 | 'VolumeType': 'gp2',
156 | 'VolumeSize': 200,
157 | 'DeleteOnTermination': True
158 | }
159 | }],
160 | )
161 |
162 | # Wait for the instances to boot.
163 | Print.info('Waiting for all instances to boot...')
164 | self._wait(['pending'])
165 | Print.heading(f'Successfully created {size} new instances')
166 | except ClientError as e:
167 | raise BenchError('Failed to create AWS instances', AWSError(e))
168 |
169 | def terminate_instances(self):
170 | try:
171 | ids, _ = self._get(['pending', 'running', 'stopping', 'stopped'])
172 | size = sum(len(x) for x in ids.values())
173 | if size == 0:
174 | Print.heading(f'All instances are shut down')
175 | return
176 |
177 | # Terminate instances.
178 | for region, client in self.clients.items():
179 | if ids[region]:
180 | client.terminate_instances(InstanceIds=ids[region])
181 |
182 | # Wait for all instances to properly shut down.
183 | Print.info('Waiting for all instances to shut down...')
184 | self._wait(['shutting-down'])
185 | for client in self.clients.values():
186 | client.delete_security_group(
187 | GroupName=self.SECURITY_GROUP_NAME
188 | )
189 |
190 | Print.heading(f'Testbed of {size} instances destroyed')
191 | except ClientError as e:
192 | raise BenchError('Failed to terminate instances', AWSError(e))
193 |
194 | def start_instances(self, max):
195 | size = 0
196 | try:
197 | ids, _ = self._get(['stopping', 'stopped'])
198 | for region, client in self.clients.items():
199 | if ids[region]:
200 | target = ids[region]
201 | target = target if len(target) < max else target[:max]
202 | size += len(target)
203 | client.start_instances(InstanceIds=target)
204 | Print.heading(f'Starting {size} instances')
205 | except ClientError as e:
206 | raise BenchError('Failed to start instances', AWSError(e))
207 |
208 | def stop_instances(self):
209 | try:
210 | ids, _ = self._get(['pending', 'running'])
211 | for region, client in self.clients.items():
212 | if ids[region]:
213 | client.stop_instances(InstanceIds=ids[region])
214 | size = sum(len(x) for x in ids.values())
215 | Print.heading(f'Stopping {size} instances')
216 | except ClientError as e:
217 | raise BenchError(AWSError(e))
218 |
219 | def hosts(self, flat=False):
220 | try:
221 | _, ips = self._get(['pending', 'running'])
222 | return [x for y in ips.values() for x in y] if flat else ips
223 | except ClientError as e:
224 | raise BenchError('Failed to gather instances IPs', AWSError(e))
225 |
226 | def print_info(self):
227 | hosts = self.hosts()
228 | key = self.settings.key_path
229 | text = ''
230 | for region, ips in hosts.items():
231 | text += f'\n Region: {region.upper()}\n'
232 | for i, ip in enumerate(ips):
233 | new_line = '\n' if (i+1) % 6 == 0 else ''
234 | text += f'{new_line} {i}\tssh -i {key} ubuntu@{ip}\n'
235 | print(
236 | '\n'
237 | '----------------------------------------------------------------\n'
238 | ' INFO:\n'
239 | '----------------------------------------------------------------\n'
240 | f' Available machines: {sum(len(x) for x in hosts.values())}\n'
241 | f'{text}'
242 | '----------------------------------------------------------------\n'
243 | )
244 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/benchmark/local.py:
--------------------------------------------------------------------------------
1 | # Copyright(C) Facebook, Inc. and its affiliates.
2 | import subprocess
3 | from math import ceil
4 | from os.path import basename, splitext
5 | from time import sleep
6 |
7 | from benchmark.commands import CommandMaker
8 | from benchmark.config import Key, LocalCommittee, NodeParameters, BenchParameters, ConfigError
9 | from benchmark.logs import LogParser, ParseError
10 | from benchmark.utils import Print, BenchError, PathMaker
11 |
12 |
13 | class LocalBench:
14 | BASE_PORT = 3000
15 |
16 | def __init__(self, bench_parameters_dict, node_parameters_dict):
17 | print("bench params dict", bench_parameters_dict)
18 | print("node params dict", node_parameters_dict)
19 | try:
20 | self.bench_parameters = BenchParameters(bench_parameters_dict)
21 | self.node_parameters = NodeParameters(node_parameters_dict)
22 | except ConfigError as e:
23 | raise BenchError('Invalid nodes or bench parameters', e)
24 |
25 | def __getattr__(self, attr):
26 | return getattr(self.bench_parameters, attr)
27 |
28 | def _background_run(self, command, log_file):
29 | name = splitext(basename(log_file))[0]
30 | cmd = f'{command} &> {log_file}'
31 | # cmd = f'{command}'
32 | print("Background run:", ['tmux', 'new', '-d', '-s', name, cmd])
33 | subprocess.run(['tmux', 'new', '-d', '-s', name, cmd], check=True)
34 |
35 | def _kill_nodes(self):
36 | # try:
37 | cmd = CommandMaker.kill().split()
38 | subprocess.run(cmd)#, stderr=subprocess.DEVNULL)
39 | # except subprocess.SubprocessError as e:
40 | # raise BenchError('Failed to kill testbed', e)
41 |
42 | def run(self, debug=False):
43 | assert isinstance(debug, bool)
44 | Print.heading('Starting local benchmark')
45 |
46 | # Kill any previous testbed.
47 | self._kill_nodes()
48 |
49 | try:
50 | Print.info('Setting up testbed...')
51 | nodes, rate = self.nodes[0], self.rate[0]
52 |
53 | # Cleanup all files.
54 | cmd = f'{CommandMaker.clean_logs()} ; {CommandMaker.cleanup()}'
55 | subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL)
56 | # sleep(0.5) # Removing the store may take time.
57 |
58 | print(cmd)
59 |
60 | # Recompile the latest code.
61 | cmd = CommandMaker.compile().split()
62 | print(cmd)
63 | subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path())
64 |
65 | # Create alias for the client and nodes binary.
66 | cmd = CommandMaker.alias_binaries(PathMaker.binary_path())
67 | print(cmd)
68 | subprocess.run([cmd], shell=True)
69 |
70 | # Generate configuration files.
71 | keys = []
72 | key_files = [PathMaker.key_file(i) for i in range(nodes)]
73 | for filename in key_files:
74 | cmd = CommandMaker.generate_key(filename).split()
75 | subprocess.run(cmd, check=True)
76 | keys += [Key.from_file(filename)]
77 |
78 | print(key_files, keys)
79 |
80 | names = [x.name for x in keys]
81 | committee = LocalCommittee(names, self.BASE_PORT, self.workers)
82 | # prints to .committee.json
83 | # print(PathMaker.committee_file())
84 | committee.print(PathMaker.committee_file())
85 |
86 | print(names, committee)
87 |
88 |
89 | self.node_parameters.print(PathMaker.parameters_file())
90 |
91 | # Run the clients (they will wait for the nodes to be ready).
92 | # Worker transaction endpoint (3003, 3008 etc.)
93 | # Probably the TPU equivalent?
94 | workers_addresses = committee.workers_addresses(self.faults)
95 |
96 |
97 | print("[+] Spinning up apps")
98 | # Run the apps
99 | for i, address in enumerate(committee.app_addresses(self.faults)):
100 | cmd = CommandMaker.run_app(address)
101 | log_file = PathMaker.app_log_file(i)
102 | # Each one of these starts a new tmux session
103 | self._background_run(cmd, log_file)
104 |
105 | sleep(1)
106 |
107 | # print("[+] Spinning up clients")
108 | # # The benchmark clients connect to the worker addresses to submit transactions
109 | # # Starts 1 client for each worker process.
110 | # rate_share = ceil(rate / committee.workers())
111 | # for i, addresses in enumerate(workers_addresses):
112 | # for (id, address) in addresses:
113 | # cmd = CommandMaker.run_client(
114 | # address,
115 | # self.tx_size,
116 | # rate_share,
117 | # [x for y in workers_addresses for _, x in y]
118 | # )
119 | # log_file = PathMaker.client_log_file(i, id)
120 | # print("--> [+] Running", cmd, log_file)
121 | # self._background_run(cmd, log_file)
122 |
123 |
124 | print("[+] Spinning up primaries")
125 | # Run the primaries (except the faulty ones).
126 | for i, address in enumerate(committee.primary_addresses(self.faults)):
127 | cmd = CommandMaker.run_primary(
128 | PathMaker.key_file(i),
129 | PathMaker.committee_file(),
130 | PathMaker.db_path(i),
131 | PathMaker.parameters_file(),
132 | app_api = committee.app_addresses(self.faults)[i],
133 | abci_api = committee.rpc_addresses(self.faults)[i],
134 | debug=debug
135 | )
136 | log_file = PathMaker.primary_log_file(i)
137 | # Each one of these starts a new tmux session
138 | self._background_run(cmd, log_file)
139 |
140 |
141 | print("[+] Spinning up workers")
142 | # Run the workers (except the faulty ones).
143 | for i, addresses in enumerate(workers_addresses):
144 | for (id, address) in addresses:
145 | cmd = CommandMaker.run_worker(
146 | PathMaker.key_file(i),
147 | PathMaker.committee_file(),
148 | PathMaker.db_path(i, id),
149 | PathMaker.parameters_file(),
150 | id, # The worker's id.
151 | debug=debug
152 | )
153 | log_file = PathMaker.worker_log_file(i, id)
154 | self._background_run(cmd, log_file)
155 |
156 |
157 | # Wait for all transactions to be processed.
158 | Print.info(f'Running benchmark ({self.duration} sec)...')
159 | sleep(self.duration)
160 | self._kill_nodes()
161 |
162 | # # Parse logs and return the parser.
163 | # Print.info('Parsing logs...')
164 | # return LogParser.process(PathMaker.logs_path(), faults=self.faults)
165 |
166 | except Exception as e:
167 | self._kill_nodes()
168 | raise BenchError('Failed to run benchmark', e)
169 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/benchmark/logs.py:
--------------------------------------------------------------------------------
1 | # Copyright(C) Facebook, Inc. and its affiliates.
2 | from datetime import datetime
3 | from glob import glob
4 | from multiprocessing import Pool
5 | from os.path import join
6 | from re import findall, search
7 | from statistics import mean
8 |
9 | from benchmark.utils import Print
10 |
11 |
12 | class ParseError(Exception):
13 | pass
14 |
15 |
16 | class LogParser:
17 | def __init__(self, clients, primaries, workers, faults=0):
18 | inputs = [clients, primaries, workers]
19 | assert all(isinstance(x, list) for x in inputs)
20 | assert all(isinstance(x, str) for y in inputs for x in y)
21 | assert all(x for x in inputs)
22 |
23 | self.faults = faults
24 | if isinstance(faults, int):
25 | self.committee_size = len(primaries) + int(faults)
26 | self.workers = len(workers) // len(primaries)
27 | else:
28 | self.committee_size = '?'
29 | self.workers = '?'
30 |
31 | # Parse the clients logs.
32 | try:
33 | with Pool() as p:
34 | results = p.map(self._parse_clients, clients)
35 | except (ValueError, IndexError, AttributeError) as e:
36 | raise ParseError(f'Failed to parse clients\' logs: {e}')
37 | self.size, self.rate, self.start, misses, self.sent_samples \
38 | = zip(*results)
39 | self.misses = sum(misses)
40 |
41 | # Parse the primaries logs.
42 | try:
43 | with Pool() as p:
44 | results = p.map(self._parse_primaries, primaries)
45 | except (ValueError, IndexError, AttributeError) as e:
46 | raise ParseError(f'Failed to parse nodes\' logs: {e}')
47 | proposals, commits, self.configs, primary_ips = zip(*results)
48 | self.proposals = self._merge_results([x.items() for x in proposals])
49 | self.commits = self._merge_results([x.items() for x in commits])
50 |
51 | # Parse the workers logs.
52 | try:
53 | with Pool() as p:
54 | results = p.map(self._parse_workers, workers)
55 | except (ValueError, IndexError, AttributeError) as e:
56 | raise ParseError(f'Failed to parse workers\' logs: {e}')
57 | sizes, self.received_samples, workers_ips = zip(*results)
58 | self.sizes = {
59 | k: v for x in sizes for k, v in x.items() if k in self.commits
60 | }
61 |
62 | # Determine whether the primary and the workers are collocated.
63 | self.collocate = set(primary_ips) == set(workers_ips)
64 |
65 | # Check whether clients missed their target rate.
66 | if self.misses != 0:
67 | Print.warn(
68 | f'Clients missed their target rate {self.misses:,} time(s)'
69 | )
70 |
71 | def _merge_results(self, input):
72 | # Keep the earliest timestamp.
73 | merged = {}
74 | for x in input:
75 | for k, v in x:
76 | if not k in merged or merged[k] > v:
77 | merged[k] = v
78 | return merged
79 |
80 | def _parse_clients(self, log):
81 | if search(r'Error', log) is not None:
82 | raise ParseError('Client(s) panicked')
83 |
84 | size = int(search(r'Transactions size: (\d+)', log).group(1))
85 | rate = int(search(r'Transactions rate: (\d+)', log).group(1))
86 |
87 | tmp = search(r'\[(.*Z) .* Start ', log).group(1)
88 | start = self._to_posix(tmp)
89 |
90 | misses = len(findall(r'rate too high', log))
91 |
92 | tmp = findall(r'\[(.*Z) .* sample transaction (\d+)', log)
93 | samples = {int(s): self._to_posix(t) for t, s in tmp}
94 |
95 | return size, rate, start, misses, samples
96 |
97 | def _parse_primaries(self, log):
98 | if search(r'(?:panicked|Error)', log) is not None:
99 | raise ParseError('Primary(s) panicked')
100 |
101 | tmp = findall(r'\[(.*Z) .* Created B\d+\([^ ]+\) -> ([^ ]+=)', log)
102 | tmp = [(d, self._to_posix(t)) for t, d in tmp]
103 | proposals = self._merge_results([tmp])
104 |
105 | tmp = findall(r'\[(.*Z) .* Committed B\d+\([^ ]+\) -> ([^ ]+=)', log)
106 | tmp = [(d, self._to_posix(t)) for t, d in tmp]
107 | commits = self._merge_results([tmp])
108 |
109 | configs = {
110 | 'header_size': int(
111 | search(r'Header size .* (\d+)', log).group(1)
112 | ),
113 | 'max_header_delay': int(
114 | search(r'Max header delay .* (\d+)', log).group(1)
115 | ),
116 | 'gc_depth': int(
117 | search(r'Garbage collection depth .* (\d+)', log).group(1)
118 | ),
119 | 'sync_retry_delay': int(
120 | search(r'Sync retry delay .* (\d+)', log).group(1)
121 | ),
122 | 'sync_retry_nodes': int(
123 | search(r'Sync retry nodes .* (\d+)', log).group(1)
124 | ),
125 | 'batch_size': int(
126 | search(r'Batch size .* (\d+)', log).group(1)
127 | ),
128 | 'max_batch_delay': int(
129 | search(r'Max batch delay .* (\d+)', log).group(1)
130 | ),
131 | }
132 |
133 | ip = search(r'booted on (\d+.\d+.\d+.\d+)', log).group(1)
134 |
135 | return proposals, commits, configs, ip
136 |
137 | def _parse_workers(self, log):
138 | if search(r'(?:panic|Error)', log) is not None:
139 | raise ParseError('Worker(s) panicked')
140 |
141 | tmp = findall(r'Batch ([^ ]+) contains (\d+) B', log)
142 | sizes = {d: int(s) for d, s in tmp}
143 |
144 | tmp = findall(r'Batch ([^ ]+) contains sample tx (\d+)', log)
145 | samples = {int(s): d for d, s in tmp}
146 |
147 | ip = search(r'booted on (\d+.\d+.\d+.\d+)', log).group(1)
148 |
149 | return sizes, samples, ip
150 |
151 | def _to_posix(self, string):
152 | x = datetime.fromisoformat(string.replace('Z', '+00:00'))
153 | return datetime.timestamp(x)
154 |
155 | def _consensus_throughput(self):
156 | if not self.commits:
157 | return 0, 0, 0
158 | start, end = min(self.proposals.values()), max(self.commits.values())
159 | duration = end - start
160 | bytes = sum(self.sizes.values())
161 | bps = bytes / duration
162 | tps = bps / self.size[0]
163 | return tps, bps, duration
164 |
165 | def _consensus_latency(self):
166 | latency = [c - self.proposals[d] for d, c in self.commits.items()]
167 | return mean(latency) if latency else 0
168 |
169 | def _end_to_end_throughput(self):
170 | if not self.commits:
171 | return 0, 0, 0
172 | start, end = min(self.start), max(self.commits.values())
173 | duration = end - start
174 | bytes = sum(self.sizes.values())
175 | bps = bytes / duration
176 | tps = bps / self.size[0]
177 | return tps, bps, duration
178 |
179 | def _end_to_end_latency(self):
180 | latency = []
181 | for sent, received in zip(self.sent_samples, self.received_samples):
182 | for tx_id, batch_id in received.items():
183 | if batch_id in self.commits:
184 | assert tx_id in sent # We receive txs that we sent.
185 | start = sent[tx_id]
186 | end = self.commits[batch_id]
187 | latency += [end-start]
188 | return mean(latency) if latency else 0
189 |
190 | def result(self):
191 | header_size = self.configs[0]['header_size']
192 | max_header_delay = self.configs[0]['max_header_delay']
193 | gc_depth = self.configs[0]['gc_depth']
194 | sync_retry_delay = self.configs[0]['sync_retry_delay']
195 | sync_retry_nodes = self.configs[0]['sync_retry_nodes']
196 | batch_size = self.configs[0]['batch_size']
197 | max_batch_delay = self.configs[0]['max_batch_delay']
198 |
199 | consensus_latency = self._consensus_latency() * 1_000
200 | consensus_tps, consensus_bps, _ = self._consensus_throughput()
201 | end_to_end_tps, end_to_end_bps, duration = self._end_to_end_throughput()
202 | end_to_end_latency = self._end_to_end_latency() * 1_000
203 |
204 | return (
205 | '\n'
206 | '-----------------------------------------\n'
207 | ' SUMMARY:\n'
208 | '-----------------------------------------\n'
209 | ' + CONFIG:\n'
210 | f' Faults: {self.faults} node(s)\n'
211 | f' Committee size: {self.committee_size} node(s)\n'
212 | f' Worker(s) per node: {self.workers} worker(s)\n'
213 | f' Collocate primary and workers: {self.collocate}\n'
214 | f' Input rate: {sum(self.rate):,} tx/s\n'
215 | f' Transaction size: {self.size[0]:,} B\n'
216 | f' Execution time: {round(duration):,} s\n'
217 | '\n'
218 | f' Header size: {header_size:,} B\n'
219 | f' Max header delay: {max_header_delay:,} ms\n'
220 | f' GC depth: {gc_depth:,} round(s)\n'
221 | f' Sync retry delay: {sync_retry_delay:,} ms\n'
222 | f' Sync retry nodes: {sync_retry_nodes:,} node(s)\n'
223 | f' batch size: {batch_size:,} B\n'
224 | f' Max batch delay: {max_batch_delay:,} ms\n'
225 | '\n'
226 | ' + RESULTS:\n'
227 | f' Consensus TPS: {round(consensus_tps):,} tx/s\n'
228 | f' Consensus BPS: {round(consensus_bps):,} B/s\n'
229 | f' Consensus latency: {round(consensus_latency):,} ms\n'
230 | '\n'
231 | f' End-to-end TPS: {round(end_to_end_tps):,} tx/s\n'
232 | f' End-to-end BPS: {round(end_to_end_bps):,} B/s\n'
233 | f' End-to-end latency: {round(end_to_end_latency):,} ms\n'
234 | '-----------------------------------------\n'
235 | )
236 |
237 | def print(self, filename):
238 | assert isinstance(filename, str)
239 | with open(filename, 'a') as f:
240 | f.write(self.result())
241 |
242 | @classmethod
243 | def process(cls, directory, faults=0):
244 | assert isinstance(directory, str)
245 |
246 | clients = []
247 | for filename in sorted(glob(join(directory, 'client-*.log'))):
248 | with open(filename, 'r') as f:
249 | clients += [f.read()]
250 | primaries = []
251 | for filename in sorted(glob(join(directory, 'primary-*.log'))):
252 | with open(filename, 'r') as f:
253 | primaries += [f.read()]
254 | workers = []
255 | for filename in sorted(glob(join(directory, 'worker-*.log'))):
256 | with open(filename, 'r') as f:
257 | workers += [f.read()]
258 |
259 | return cls(clients, primaries, workers, faults=faults)
260 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/benchmark/plot.py:
--------------------------------------------------------------------------------
1 | # Copyright(C) Facebook, Inc. and its affiliates.
2 | from collections import defaultdict
3 | from re import findall, search, split
4 | import matplotlib.pyplot as plt
5 | import matplotlib.ticker as tick
6 | from glob import glob
7 | from itertools import cycle
8 |
9 | from benchmark.utils import PathMaker
10 | from benchmark.config import PlotParameters
11 | from benchmark.aggregate import LogAggregator
12 |
13 |
14 | @tick.FuncFormatter
15 | def default_major_formatter(x, pos):
16 | if pos is None:
17 | return
18 | if x >= 1_000:
19 | return f'{x/1000:.0f}k'
20 | else:
21 | return f'{x:.0f}'
22 |
23 |
24 | @tick.FuncFormatter
25 | def sec_major_formatter(x, pos):
26 | if pos is None:
27 | return
28 | return f'{float(x)/1000:.1f}'
29 |
30 |
31 | @tick.FuncFormatter
32 | def mb_major_formatter(x, pos):
33 | if pos is None:
34 | return
35 | return f'{x:,.0f}'
36 |
37 |
38 | class PlotError(Exception):
39 | pass
40 |
41 |
42 | class Ploter:
43 | def __init__(self, filenames):
44 | if not filenames:
45 | raise PlotError('No data to plot')
46 |
47 | self.results = []
48 | try:
49 | for filename in filenames:
50 | with open(filename, 'r') as f:
51 | self.results += [f.read().replace(',', '')]
52 | except OSError as e:
53 | raise PlotError(f'Failed to load log files: {e}')
54 |
55 | def _natural_keys(self, text):
56 | def try_cast(text): return int(text) if text.isdigit() else text
57 | return [try_cast(c) for c in split('(\d+)', text)]
58 |
59 | def _tps(self, data):
60 | values = findall(r' TPS: (\d+) \+/- (\d+)', data)
61 | values = [(int(x), int(y)) for x, y in values]
62 | return list(zip(*values))
63 |
64 | def _latency(self, data, scale=1):
65 | values = findall(r' Latency: (\d+) \+/- (\d+)', data)
66 | values = [(float(x)/scale, float(y)/scale) for x, y in values]
67 | return list(zip(*values))
68 |
69 | def _variable(self, data):
70 | return [int(x) for x in findall(r'Variable value: X=(\d+)', data)]
71 |
72 | def _tps2bps(self, x):
73 | data = self.results[0]
74 | size = int(search(r'Transaction size: (\d+)', data).group(1))
75 | return x * size / 10**6
76 |
77 | def _bps2tps(self, x):
78 | data = self.results[0]
79 | size = int(search(r'Transaction size: (\d+)', data).group(1))
80 | return x * 10**6 / size
81 |
82 | def _plot(self, x_label, y_label, y_axis, z_axis, type):
83 | plt.figure()
84 | markers = cycle(['o', 'v', 's', 'p', 'D', 'P'])
85 | self.results.sort(key=self._natural_keys, reverse=(type == 'tps'))
86 | for result in self.results:
87 | y_values, y_err = y_axis(result)
88 | x_values = self._variable(result)
89 | if len(y_values) != len(y_err) or len(y_err) != len(x_values):
90 | raise PlotError('Unequal number of x, y, and y_err values')
91 |
92 | plt.errorbar(
93 | x_values, y_values, yerr=y_err, label=z_axis(result),
94 | linestyle='dotted', marker=next(markers), capsize=3
95 | )
96 |
97 | plt.legend(loc='lower center', bbox_to_anchor=(0.5, 1), ncol=3)
98 | plt.xlim(xmin=0)
99 | plt.ylim(bottom=0)
100 | plt.xlabel(x_label, fontweight='bold')
101 | plt.ylabel(y_label[0], fontweight='bold')
102 | plt.xticks(weight='bold')
103 | plt.yticks(weight='bold')
104 | plt.grid()
105 | ax = plt.gca()
106 | ax.xaxis.set_major_formatter(default_major_formatter)
107 | ax.yaxis.set_major_formatter(default_major_formatter)
108 | if 'latency' in type:
109 | ax.yaxis.set_major_formatter(sec_major_formatter)
110 | if len(y_label) > 1:
111 | secaxy = ax.secondary_yaxis(
112 | 'right', functions=(self._tps2bps, self._bps2tps)
113 | )
114 | secaxy.set_ylabel(y_label[1])
115 | secaxy.yaxis.set_major_formatter(mb_major_formatter)
116 |
117 | for x in ['pdf', 'png']:
118 | plt.savefig(PathMaker.plot_file(type, x), bbox_inches='tight')
119 |
120 | @staticmethod
121 | def nodes(data):
122 | x = search(r'Committee size: (\d+)', data).group(1)
123 | f = search(r'Faults: (\d+)', data).group(1)
124 | faults = f'({f} faulty)' if f != '0' else ''
125 | return f'{x} nodes {faults}'
126 |
127 | @staticmethod
128 | def workers(data):
129 | x = search(r'Workers per node: (\d+)', data).group(1)
130 | f = search(r'Faults: (\d+)', data).group(1)
131 | faults = f'({f} faulty)' if f != '0' else ''
132 | return f'{x} workers {faults}'
133 |
134 | @staticmethod
135 | def max_latency(data):
136 | x = search(r'Max latency: (\d+)', data).group(1)
137 | f = search(r'Faults: (\d+)', data).group(1)
138 | faults = f'({f} faulty)' if f != '0' else ''
139 | return f'Max latency: {float(x) / 1000:,.1f} s {faults}'
140 |
141 | @classmethod
142 | def plot_latency(cls, files, scalability):
143 | assert isinstance(files, list)
144 | assert all(isinstance(x, str) for x in files)
145 | z_axis = cls.workers if scalability else cls.nodes
146 | x_label = 'Throughput (tx/s)'
147 | y_label = ['Latency (s)']
148 | ploter = cls(files)
149 | ploter._plot(x_label, y_label, ploter._latency, z_axis, 'latency')
150 |
151 | @classmethod
152 | def plot_tps(cls, files, scalability):
153 | assert isinstance(files, list)
154 | assert all(isinstance(x, str) for x in files)
155 | z_axis = cls.max_latency
156 | x_label = 'Workers per node' if scalability else 'Committee size'
157 | y_label = ['Throughput (tx/s)', 'Throughput (MB/s)']
158 | ploter = cls(files)
159 | ploter._plot(x_label, y_label, ploter._tps, z_axis, 'tps')
160 |
161 | @classmethod
162 | def plot(cls, params_dict):
163 | try:
164 | params = PlotParameters(params_dict)
165 | except PlotError as e:
166 | raise PlotError('Invalid nodes or bench parameters', e)
167 |
168 | # Aggregate the logs.
169 | LogAggregator(params.max_latency).print()
170 |
171 | # Make the latency, tps, and robustness graphs.
172 | iterator = params.workers if params.scalability() else params.nodes
173 | latency_files, tps_files = [], []
174 | for f in params.faults:
175 | for x in iterator:
176 | latency_files += glob(
177 | PathMaker.agg_file(
178 | 'latency',
179 | f,
180 | x if not params.scalability() else params.nodes[0],
181 | x if params.scalability() else params.workers[0],
182 | params.collocate,
183 | 'any',
184 | params.tx_size,
185 | )
186 | )
187 |
188 | for l in params.max_latency:
189 | tps_files += glob(
190 | PathMaker.agg_file(
191 | 'tps',
192 | f,
193 | 'x' if not params.scalability() else params.nodes[0],
194 | 'x' if params.scalability() else params.workers[0],
195 | params.collocate,
196 | 'any',
197 | params.tx_size,
198 | max_latency=l
199 | )
200 | )
201 |
202 | cls.plot_latency(latency_files, params.scalability())
203 | cls.plot_tps(tps_files, params.scalability())
204 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/benchmark/remote.py:
--------------------------------------------------------------------------------
1 | # Copyright(C) Facebook, Inc. and its affiliates.
2 | from collections import OrderedDict
3 | from fabric import Connection, ThreadingGroup as Group
4 | from fabric.exceptions import GroupException
5 | from paramiko import RSAKey
6 | from paramiko.ssh_exception import PasswordRequiredException, SSHException
7 | from os.path import basename, splitext
8 | from time import sleep
9 | from math import ceil
10 | from copy import deepcopy
11 | import subprocess
12 |
13 | from benchmark.config import Committee, Key, NodeParameters, BenchParameters, ConfigError
14 | from benchmark.utils import BenchError, Print, PathMaker, progress_bar
15 | from benchmark.commands import CommandMaker
16 | from benchmark.logs import LogParser, ParseError
17 | from benchmark.instance import InstanceManager
18 |
19 |
20 | class FabricError(Exception):
21 | ''' Wrapper for Fabric exception with a meaningfull error message. '''
22 |
23 | def __init__(self, error):
24 | assert isinstance(error, GroupException)
25 | message = list(error.result.values())[-1]
26 | super().__init__(message)
27 |
28 |
29 | class ExecutionError(Exception):
30 | pass
31 |
32 |
33 | class Bench:
34 | def __init__(self, ctx):
35 | self.manager = InstanceManager.make()
36 | self.settings = self.manager.settings
37 | try:
38 | ctx.connect_kwargs.pkey = RSAKey.from_private_key_file(
39 | self.manager.settings.key_path
40 | )
41 | self.connect = ctx.connect_kwargs
42 | except (IOError, PasswordRequiredException, SSHException) as e:
43 | raise BenchError('Failed to load SSH key', e)
44 |
45 | def _check_stderr(self, output):
46 | if isinstance(output, dict):
47 | for x in output.values():
48 | if x.stderr:
49 | raise ExecutionError(x.stderr)
50 | else:
51 | if output.stderr:
52 | raise ExecutionError(output.stderr)
53 |
54 | def install(self):
55 | Print.info('Installing rust and cloning the repo...')
56 | cmd = [
57 | 'sudo apt-get update',
58 | 'sudo apt-get -y upgrade',
59 | 'sudo apt-get -y autoremove',
60 |
61 | # The following dependencies prevent the error: [error: linker `cc` not found].
62 | 'sudo apt-get -y install build-essential',
63 | 'sudo apt-get -y install cmake',
64 |
65 | # Install rust (non-interactive).
66 | 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y',
67 | 'source $HOME/.cargo/env',
68 | 'rustup default stable',
69 |
70 | # This is missing from the Rocksdb installer (needed for Rocksdb).
71 | 'sudo apt-get install -y clang',
72 |
73 | # Clone the repo.
74 | f'(git clone {self.settings.repo_url} || (cd {self.settings.repo_name} ; git pull))'
75 | ]
76 | hosts = self.manager.hosts(flat=True)
77 | try:
78 | g = Group(*hosts, user='ubuntu', connect_kwargs=self.connect)
79 | g.run(' && '.join(cmd), hide=True)
80 | Print.heading(f'Initialized testbed of {len(hosts)} nodes')
81 | except (GroupException, ExecutionError) as e:
82 | e = FabricError(e) if isinstance(e, GroupException) else e
83 | raise BenchError('Failed to install repo on testbed', e)
84 |
85 | def kill(self, hosts=[], delete_logs=False):
86 | assert isinstance(hosts, list)
87 | assert isinstance(delete_logs, bool)
88 | hosts = hosts if hosts else self.manager.hosts(flat=True)
89 | delete_logs = CommandMaker.clean_logs() if delete_logs else 'true'
90 | cmd = [delete_logs, f'({CommandMaker.kill()} || true)']
91 | try:
92 | g = Group(*hosts, user='ubuntu', connect_kwargs=self.connect)
93 | g.run(' && '.join(cmd), hide=True)
94 | except GroupException as e:
95 | raise BenchError('Failed to kill nodes', FabricError(e))
96 |
97 | def _select_hosts(self, bench_parameters):
98 | # Collocate the primary and its workers on the same machine.
99 | if bench_parameters.collocate:
100 | nodes = max(bench_parameters.nodes)
101 |
102 | # Ensure there are enough hosts.
103 | hosts = self.manager.hosts()
104 | if sum(len(x) for x in hosts.values()) < nodes:
105 | return []
106 |
107 | # Select the hosts in different data centers.
108 | ordered = zip(*hosts.values())
109 | ordered = [x for y in ordered for x in y]
110 | return ordered[:nodes]
111 |
112 | # Spawn the primary and each worker on a different machine. Each
113 | # authority runs in a single data center.
114 | else:
115 | primaries = max(bench_parameters.nodes)
116 |
117 | # Ensure there are enough hosts.
118 | hosts = self.manager.hosts()
119 | if len(hosts.keys()) < primaries:
120 | return []
121 | for ips in hosts.values():
122 | if len(ips) < bench_parameters.workers + 1:
123 | return []
124 |
125 | # Ensure the primary and its workers are in the same region.
126 | selected = []
127 | for region in list(hosts.keys())[:primaries]:
128 | ips = list(hosts[region])[:bench_parameters.workers + 1]
129 | selected.append(ips)
130 | return selected
131 |
132 | def _background_run(self, host, command, log_file):
133 | name = splitext(basename(log_file))[0]
134 | cmd = f'tmux new -d -s "{name}" "{command} |& tee {log_file}"'
135 | c = Connection(host, user='ubuntu', connect_kwargs=self.connect)
136 | output = c.run(cmd, hide=True)
137 | self._check_stderr(output)
138 |
139 | def _update(self, hosts, collocate):
140 | if collocate:
141 | ips = list(set(hosts))
142 | else:
143 | ips = list(set([x for y in hosts for x in y]))
144 |
145 | Print.info(
146 | f'Updating {len(ips)} machines (branch "{self.settings.branch}")...'
147 | )
148 | cmd = [
149 | f'(cd {self.settings.repo_name} && git fetch -f)',
150 | f'(cd {self.settings.repo_name} && git checkout -f {self.settings.branch})',
151 | f'(cd {self.settings.repo_name} && git pull -f)',
152 | 'source $HOME/.cargo/env',
153 | f'(cd {self.settings.repo_name}/node && {CommandMaker.compile()})',
154 | CommandMaker.alias_binaries(
155 | f'./{self.settings.repo_name}/target/release/'
156 | )
157 | ]
158 | g = Group(*ips, user='ubuntu', connect_kwargs=self.connect)
159 | g.run(' && '.join(cmd), hide=True)
160 |
161 | def _config(self, hosts, node_parameters, bench_parameters):
162 | Print.info('Generating configuration files...')
163 |
164 | # Cleanup all local configuration files.
165 | cmd = CommandMaker.cleanup()
166 | subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL)
167 |
168 | # Recompile the latest code.
169 | cmd = CommandMaker.compile().split()
170 | subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path())
171 |
172 | # Create alias for the client and nodes binary.
173 | cmd = CommandMaker.alias_binaries(PathMaker.binary_path())
174 | subprocess.run([cmd], shell=True)
175 |
176 | # Generate configuration files.
177 | keys = []
178 | key_files = [PathMaker.key_file(i) for i in range(len(hosts))]
179 | for filename in key_files:
180 | cmd = CommandMaker.generate_key(filename).split()
181 | subprocess.run(cmd, check=True)
182 | keys += [Key.from_file(filename)]
183 |
184 | names = [x.name for x in keys]
185 |
186 | if bench_parameters.collocate:
187 | workers = bench_parameters.workers
188 | addresses = OrderedDict(
189 | (x, [y] * (workers + 1)) for x, y in zip(names, hosts)
190 | )
191 | else:
192 | addresses = OrderedDict(
193 | (x, y) for x, y in zip(names, hosts)
194 | )
195 | committee = Committee(addresses, self.settings.base_port)
196 | committee.print(PathMaker.committee_file())
197 |
198 | node_parameters.print(PathMaker.parameters_file())
199 |
200 | # Cleanup all nodes and upload configuration files.
201 | names = names[:len(names)-bench_parameters.faults]
202 | progress = progress_bar(names, prefix='Uploading config files:')
203 | for i, name in enumerate(progress):
204 | for ip in committee.ips(name):
205 | c = Connection(ip, user='ubuntu', connect_kwargs=self.connect)
206 | c.run(f'{CommandMaker.cleanup()} || true', hide=True)
207 | c.put(PathMaker.committee_file(), '.')
208 | c.put(PathMaker.key_file(i), '.')
209 | c.put(PathMaker.parameters_file(), '.')
210 |
211 | return committee
212 |
213 | def _run_single(self, rate, committee, bench_parameters, debug=False):
214 | faults = bench_parameters.faults
215 |
216 | # Kill any potentially unfinished run and delete logs.
217 | hosts = committee.ips()
218 | self.kill(hosts=hosts, delete_logs=True)
219 |
220 | # Run the clients (they will wait for the nodes to be ready).
221 | # Filter all faulty nodes from the client addresses (or they will wait
222 | # for the faulty nodes to be online).
223 | Print.info('Booting clients...')
224 | workers_addresses = committee.workers_addresses(faults)
225 | rate_share = ceil(rate / committee.workers())
226 | for i, addresses in enumerate(workers_addresses):
227 | for (id, address) in addresses:
228 | host = Committee.ip(address)
229 | cmd = CommandMaker.run_client(
230 | address,
231 | bench_parameters.tx_size,
232 | rate_share,
233 | [x for y in workers_addresses for _, x in y]
234 | )
235 | log_file = PathMaker.client_log_file(i, id)
236 | self._background_run(host, cmd, log_file)
237 |
238 | # Run the primaries (except the faulty ones).
239 | Print.info('Booting primaries...')
240 | for i, address in enumerate(committee.primary_addresses(faults)):
241 | host = Committee.ip(address)
242 | cmd = CommandMaker.run_primary(
243 | PathMaker.key_file(i),
244 | PathMaker.committee_file(),
245 | PathMaker.db_path(i),
246 | PathMaker.parameters_file(),
247 | debug=debug
248 | )
249 | log_file = PathMaker.primary_log_file(i)
250 | self._background_run(host, cmd, log_file)
251 |
252 | # Run the workers (except the faulty ones).
253 | Print.info('Booting workers...')
254 | for i, addresses in enumerate(workers_addresses):
255 | for (id, address) in addresses:
256 | host = Committee.ip(address)
257 | cmd = CommandMaker.run_worker(
258 | PathMaker.key_file(i),
259 | PathMaker.committee_file(),
260 | PathMaker.db_path(i, id),
261 | PathMaker.parameters_file(),
262 | id, # The worker's id.
263 | debug=debug
264 | )
265 | log_file = PathMaker.worker_log_file(i, id)
266 | self._background_run(host, cmd, log_file)
267 |
268 | # Wait for all transactions to be processed.
269 | duration = bench_parameters.duration
270 | for _ in progress_bar(range(20), prefix=f'Running benchmark ({duration} sec):'):
271 | sleep(ceil(duration / 20))
272 | self.kill(hosts=hosts, delete_logs=False)
273 |
274 | def _logs(self, committee, faults):
275 | # Delete local logs (if any).
276 | cmd = CommandMaker.clean_logs()
277 | subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL)
278 |
279 | # Download log files.
280 | workers_addresses = committee.workers_addresses(faults)
281 | progress = progress_bar(workers_addresses, prefix='Downloading workers logs:')
282 | for i, addresses in enumerate(progress):
283 | for id, address in addresses:
284 | host = Committee.ip(address)
285 | c = Connection(host, user='ubuntu', connect_kwargs=self.connect)
286 | c.get(
287 | PathMaker.client_log_file(i, id),
288 | local=PathMaker.client_log_file(i, id)
289 | )
290 | c.get(
291 | PathMaker.worker_log_file(i, id),
292 | local=PathMaker.worker_log_file(i, id)
293 | )
294 |
295 | primary_addresses = committee.primary_addresses(faults)
296 | progress = progress_bar(primary_addresses, prefix='Downloading primaries logs:')
297 | for i, address in enumerate(progress):
298 | host = Committee.ip(address)
299 | c = Connection(host, user='ubuntu', connect_kwargs=self.connect)
300 | c.get(
301 | PathMaker.primary_log_file(i),
302 | local=PathMaker.primary_log_file(i)
303 | )
304 |
305 | # Parse logs and return the parser.
306 | Print.info('Parsing logs and computing performance...')
307 | return LogParser.process(PathMaker.logs_path(), faults=faults)
308 |
309 | def run(self, bench_parameters_dict, node_parameters_dict, debug=False):
310 | assert isinstance(debug, bool)
311 | Print.heading('Starting remote benchmark')
312 | try:
313 | bench_parameters = BenchParameters(bench_parameters_dict)
314 | node_parameters = NodeParameters(node_parameters_dict)
315 | except ConfigError as e:
316 | raise BenchError('Invalid nodes or bench parameters', e)
317 |
318 | # Select which hosts to use.
319 | selected_hosts = self._select_hosts(bench_parameters)
320 | if not selected_hosts:
321 | Print.warn('There are not enough instances available')
322 | return
323 |
324 | # Update nodes.
325 | try:
326 | self._update(selected_hosts, bench_parameters.collocate)
327 | except (GroupException, ExecutionError) as e:
328 | e = FabricError(e) if isinstance(e, GroupException) else e
329 | raise BenchError('Failed to update nodes', e)
330 |
331 | # Upload all configuration files.
332 | try:
333 | committee = self._config(
334 | selected_hosts, node_parameters, bench_parameters
335 | )
336 | except (subprocess.SubprocessError, GroupException) as e:
337 | e = FabricError(e) if isinstance(e, GroupException) else e
338 | raise BenchError('Failed to configure nodes', e)
339 |
340 | # Run benchmarks.
341 | for n in bench_parameters.nodes:
342 | committee_copy = deepcopy(committee)
343 | committee_copy.remove_nodes(committee.size() - n)
344 |
345 | for r in bench_parameters.rate:
346 | Print.heading(f'\nRunning {n} nodes (input rate: {r:,} tx/s)')
347 |
348 | # Run the benchmark.
349 | for i in range(bench_parameters.runs):
350 | Print.heading(f'Run {i+1}/{bench_parameters.runs}')
351 | try:
352 | self._run_single(
353 | r, committee_copy, bench_parameters, debug
354 | )
355 |
356 | faults = bench_parameters.faults
357 | logger = self._logs(committee_copy, faults)
358 | logger.print(PathMaker.result_file(
359 | faults,
360 | n,
361 | bench_parameters.workers,
362 | bench_parameters.collocate,
363 | r,
364 | bench_parameters.tx_size,
365 | ))
366 | except (subprocess.SubprocessError, GroupException, ParseError) as e:
367 | self.kill(hosts=selected_hosts)
368 | if isinstance(e, GroupException):
369 | e = FabricError(e)
370 | Print.error(BenchError('Benchmark failed', e))
371 | continue
372 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/benchmark/settings.py:
--------------------------------------------------------------------------------
1 | # Copyright(C) Facebook, Inc. and its affiliates.
2 | from json import load, JSONDecodeError
3 |
4 |
5 | class SettingsError(Exception):
6 | pass
7 |
8 |
9 | class Settings:
10 | def __init__(self, key_name, key_path, base_port, repo_name, repo_url,
11 | branch, instance_type, aws_regions):
12 | inputs_str = [
13 | key_name, key_path, repo_name, repo_url, branch, instance_type
14 | ]
15 | if isinstance(aws_regions, list):
16 | regions = aws_regions
17 | else:
18 | regions = [aws_regions]
19 | inputs_str += regions
20 | ok = all(isinstance(x, str) for x in inputs_str)
21 | ok &= isinstance(base_port, int)
22 | ok &= len(regions) > 0
23 | if not ok:
24 | raise SettingsError('Invalid settings types')
25 |
26 | self.key_name = key_name
27 | self.key_path = key_path
28 |
29 | self.base_port = base_port
30 |
31 | self.repo_name = repo_name
32 | self.repo_url = repo_url
33 | self.branch = branch
34 |
35 | self.instance_type = instance_type
36 | self.aws_regions = regions
37 |
38 | @classmethod
39 | def load(cls, filename):
40 | try:
41 | with open(filename, 'r') as f:
42 | data = load(f)
43 |
44 | return cls(
45 | data['key']['name'],
46 | data['key']['path'],
47 | data['port'],
48 | data['repo']['name'],
49 | data['repo']['url'],
50 | data['repo']['branch'],
51 | data['instances']['type'],
52 | data['instances']['regions'],
53 | )
54 | except (OSError, JSONDecodeError) as e:
55 | raise SettingsError(str(e))
56 |
57 | except KeyError as e:
58 | raise SettingsError(f'Malformed settings: missing key {e}')
59 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/benchmark/utils.py:
--------------------------------------------------------------------------------
1 | # Copyright(C) Facebook, Inc. and its affiliates.
2 | from os.path import join
3 |
4 |
5 | class BenchError(Exception):
6 | def __init__(self, message, error):
7 | assert isinstance(error, Exception)
8 | self.message = message
9 | self.cause = error
10 | super().__init__(message)
11 |
12 |
13 | class PathMaker:
14 | @staticmethod
15 | def binary_path():
16 | return join('..', 'target', 'release')
17 |
18 | @staticmethod
19 | def node_crate_path():
20 | return join('..', 'node')
21 |
22 | @staticmethod
23 | def committee_file():
24 | return '.committee.json'
25 |
26 | @staticmethod
27 | def parameters_file():
28 | return '.parameters.json'
29 |
30 | @staticmethod
31 | def key_file(i):
32 | assert isinstance(i, int) and i >= 0
33 | return f'.node-{i}.json'
34 |
35 | @staticmethod
36 | def db_path(i, j=None):
37 | assert isinstance(i, int) and i >= 0
38 | assert (isinstance(j, int) and i >= 0) or j is None
39 | worker_id = f'-{j}' if j is not None else ''
40 | return f'.db-{i}{worker_id}'
41 |
42 | @staticmethod
43 | def logs_path():
44 | return 'logs'
45 |
46 | @staticmethod
47 | def primary_log_file(i):
48 | assert isinstance(i, int) and i >= 0
49 | return join(PathMaker.logs_path(), f'primary-{i}.log')
50 |
51 | @staticmethod
52 | def app_log_file(i):
53 | assert isinstance(i, int) and i >= 0
54 | return join(PathMaker.logs_path(), f'app-{i}.log')
55 |
56 | @staticmethod
57 | def worker_log_file(i, j):
58 | assert isinstance(i, int) and i >= 0
59 | assert isinstance(j, int) and i >= 0
60 | return join(PathMaker.logs_path(), f'worker-{i}-{j}.log')
61 |
62 | @staticmethod
63 | def client_log_file(i, j):
64 | assert isinstance(i, int) and i >= 0
65 | assert isinstance(j, int) and i >= 0
66 | return join(PathMaker.logs_path(), f'client-{i}-{j}.log')
67 |
68 | @staticmethod
69 | def results_path():
70 | return 'results'
71 |
72 | @staticmethod
73 | def result_file(faults, nodes, workers, collocate, rate, tx_size):
74 | return join(
75 | PathMaker.results_path(),
76 | f'bench-{faults}-{nodes}-{workers}-{collocate}-{rate}-{tx_size}.txt'
77 | )
78 |
79 | @staticmethod
80 | def plots_path():
81 | return 'plots'
82 |
83 | @staticmethod
84 | def agg_file(type, faults, nodes, workers, collocate, rate, tx_size, max_latency=None):
85 | if max_latency is None:
86 | name = f'{type}-bench-{faults}-{nodes}-{workers}-{collocate}-{rate}-{tx_size}.txt'
87 | else:
88 | name = f'{type}-{max_latency}-bench-{faults}-{nodes}-{workers}-{collocate}-{rate}-{tx_size}.txt'
89 | return join(PathMaker.plots_path(), name)
90 |
91 | @staticmethod
92 | def plot_file(name, ext):
93 | return join(PathMaker.plots_path(), f'{name}.{ext}')
94 |
95 |
96 | class Color:
97 | HEADER = '\033[95m'
98 | OK_BLUE = '\033[94m'
99 | OK_GREEN = '\033[92m'
100 | WARNING = '\033[93m'
101 | FAIL = '\033[91m'
102 | END = '\033[0m'
103 | BOLD = '\033[1m'
104 | UNDERLINE = '\033[4m'
105 |
106 |
107 | class Print:
108 | @staticmethod
109 | def heading(message):
110 | assert isinstance(message, str)
111 | print(f'{Color.OK_GREEN}{message}{Color.END}')
112 |
113 | @staticmethod
114 | def info(message):
115 | assert isinstance(message, str)
116 | print(message)
117 |
118 | @staticmethod
119 | def warn(message):
120 | assert isinstance(message, str)
121 | print(f'{Color.BOLD}{Color.WARNING}WARN{Color.END}: {message}')
122 |
123 | @staticmethod
124 | def error(e):
125 | assert isinstance(e, BenchError)
126 | print(f'\n{Color.BOLD}{Color.FAIL}ERROR{Color.END}: {e}\n')
127 | causes, current_cause = [], e.cause
128 | while isinstance(current_cause, BenchError):
129 | causes += [f' {len(causes)}: {e.cause}\n']
130 | current_cause = current_cause.cause
131 | causes += [f' {len(causes)}: {type(current_cause)}\n']
132 | causes += [f' {len(causes)}: {current_cause}\n']
133 | print(f'Caused by: \n{"".join(causes)}\n')
134 |
135 |
136 | def progress_bar(iterable, prefix='', suffix='', decimals=1, length=30, fill='█', print_end='\r'):
137 | total = len(iterable)
138 |
139 | def printProgressBar(iteration):
140 | formatter = '{0:.'+str(decimals)+'f}'
141 | percent = formatter.format(100 * (iteration / float(total)))
142 | filledLength = int(length * iteration // total)
143 | bar = fill * filledLength + '-' * (length - filledLength)
144 | print(f'\r{prefix} |{bar}| {percent}% {suffix}', end=print_end)
145 |
146 | printProgressBar(0)
147 | for i, item in enumerate(iterable):
148 | yield item
149 | printProgressBar(i + 1)
150 | print()
151 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/cleanup-logs.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash -ve
2 |
3 | cat logs/primary-0.log | grep -i LEDGER | awk '{ print $7; }' > logs/primary-0-parsed.log
4 | cat logs/primary-1.log | grep -i LEDGER | awk '{ print $7; }' > logs/primary-1-parsed.log
5 | cat logs/primary-2.log | grep -i LEDGER | awk '{ print $7; }' > logs/primary-2-parsed.log
6 | cat logs/primary-3.log | grep -i LEDGER | awk '{ print $7; }' > logs/primary-3-parsed.log
7 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/fabfile.py:
--------------------------------------------------------------------------------
1 | # Copyright(C) Facebook, Inc. and its affiliates.
2 | from fabric import task
3 |
4 | from benchmark.local import LocalBench
5 | from benchmark.logs import ParseError, LogParser
6 | from benchmark.utils import Print
7 | from benchmark.plot import Ploter, PlotError
8 | from benchmark.instance import InstanceManager
9 | from benchmark.remote import Bench, BenchError
10 |
11 |
12 | @task
13 | def local(ctx, debug=True):
14 | ''' Run benchmarks on localhost '''
15 | bench_params = {
16 | 'faults': 0,
17 | 'nodes': 4, # only start 2 nodes
18 | 'workers': 1,
19 | 'rate': 0,
20 | 'tx_size': 512,
21 | 'duration': 5*60,
22 | }
23 | node_params = {
24 | 'header_size': 50, # bytes
25 | 'max_header_delay': 1_000, # ms
26 | 'gc_depth': 50, # rounds
27 | 'sync_retry_delay': 10_000, # ms
28 | 'sync_retry_nodes': 3, # number of nodes
29 | 'batch_size': 500_000, # bytes
30 | 'max_batch_delay': 200 # ms
31 | }
32 |
33 | bencher = LocalBench(bench_params, node_params)
34 | print(bencher)
35 | bencher.run()
36 |
37 | # try:
38 | # ret = LocalBench(bench_params, node_params).run(debug)
39 | # print(ret.result())
40 | # except BenchError as e:
41 | # Print.error(e)
42 |
43 |
44 | @task
45 | def create(ctx, nodes=10):
46 | ''' Create a testbed'''
47 | try:
48 | InstanceManager.make().create_instances(nodes)
49 | except BenchError as e:
50 | Print.error(e)
51 |
52 |
53 | @task
54 | def destroy(ctx):
55 | ''' Destroy the testbed '''
56 | try:
57 | InstanceManager.make().terminate_instances()
58 | except BenchError as e:
59 | Print.error(e)
60 |
61 |
62 | @task
63 | def start(ctx, max=2):
64 | ''' Start at most `max` machines per data center '''
65 | try:
66 | InstanceManager.make().start_instances(max)
67 | except BenchError as e:
68 | Print.error(e)
69 |
70 |
71 | @task
72 | def stop(ctx):
73 | ''' Stop all machines '''
74 | try:
75 | InstanceManager.make().stop_instances()
76 | except BenchError as e:
77 | Print.error(e)
78 |
79 |
80 | @task
81 | def info(ctx):
82 | ''' Display connect information about all the available machines '''
83 | try:
84 | InstanceManager.make().print_info()
85 | except BenchError as e:
86 | Print.error(e)
87 |
88 |
89 | @task
90 | def install(ctx):
91 | ''' Install the codebase on all machines '''
92 | try:
93 | Bench(ctx).install()
94 | except BenchError as e:
95 | Print.error(e)
96 |
97 |
98 | @task
99 | def remote(ctx, debug=False):
100 | ''' Run benchmarks on AWS '''
101 | bench_params = {
102 | 'faults': 0,
103 | 'nodes': [10, 20],
104 | 'workers': 1,
105 | 'collocate': True,
106 | 'rate': [10_000, 50_000],
107 | 'tx_size': 512,
108 | 'duration': 300,
109 | 'runs': 2,
110 | }
111 | node_params = {
112 | 'header_size': 50, # bytes
113 | 'max_header_delay': 5_000, # ms
114 | 'gc_depth': 50, # rounds
115 | 'sync_retry_delay': 10_000, # ms
116 | 'sync_retry_nodes': 3, # number of nodes
117 | 'batch_size': 500_000, # bytes
118 | 'max_batch_delay': 200 # ms
119 | }
120 | try:
121 | Bench(ctx).run(bench_params, node_params, debug)
122 | except BenchError as e:
123 | Print.error(e)
124 |
125 |
126 | @task
127 | def plot(ctx):
128 | ''' Plot performance using the logs generated by "fab remote" '''
129 | plot_params = {
130 | 'faults': [0],
131 | 'nodes': [10, 20, 50],
132 | 'workers': [1],
133 | 'collocate': True,
134 | 'tx_size': 512,
135 | 'max_latency': [2_500, 4_500]
136 | }
137 | try:
138 | Ploter.plot(plot_params)
139 | except PlotError as e:
140 | Print.error(BenchError('Failed to plot performance', e))
141 |
142 |
143 | @task
144 | def kill(ctx):
145 | ''' Stop execution on all machines '''
146 | try:
147 | Bench(ctx).kill()
148 | except BenchError as e:
149 | Print.error(e)
150 |
151 |
152 | @task
153 | def logs(ctx):
154 | ''' Print a summary of the logs '''
155 | try:
156 | print(LogParser.process('./logs', faults='?').result())
157 | except ParseError as e:
158 | Print.error(BenchError('Failed to parse logs', e))
159 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/node_params.json:
--------------------------------------------------------------------------------
1 | {
2 | 'header_size': 50, # bytes
3 | 'max_header_delay': 5_000, # ms
4 | 'gc_depth': 50, # rounds
5 | 'sync_retry_delay': 10_000, # ms
6 | 'sync_retry_nodes': 3, # number of nodes
7 | 'batch_size': 500_000, # bytes
8 | 'max_batch_delay': 200 # ms
9 | }
10 |
11 |
--------------------------------------------------------------------------------
/narwhal-abci/demo/pyproject.toml:
--------------------------------------------------------------------------------
1 | [tool.poetry]
2 | name = "benchmark"
3 | version = "0.1.0"
4 | description = ""
5 | authors = ["Georgios Konstantopoulos "]
6 |
7 | [tool.poetry.dependencies]
8 | python = "^3.10"
9 | boto3 = "1.16.0"
10 | Fabric = "2.6.0"
11 | matplotlib = "3.3.4"
12 |
13 | [tool.poetry.dev-dependencies]
14 |
15 | [build-system]
16 | requires = ["poetry-core>=1.0.0"]
17 | build-backend = "poetry.core.masonry.api"
18 |
--------------------------------------------------------------------------------
/narwhal-abci/narwhal-abci/Cargo.toml:
--------------------------------------------------------------------------------
1 | [package]
2 | name = "narwhal-abci"
3 | version = "0.1.0"
4 | edition = "2021"
5 |
6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
7 |
8 | [dependencies]
9 | tokio = { version = "1.19.2", features = ["full"] }
10 | tokio-util = { version = "0.6.2", features= ["codec"] }
11 | warp = "0.3.2"
12 | rocksdb = "0.16.0"
13 | serde = { version = "1.0", features = ["derive"] }
14 | log = "0.4.11"
15 | futures = "0.3.15"
16 | eyre = "0.6.8"
17 |
18 | bincode = "1.3.3"
19 | hex = "0.4.3"
20 |
21 | tendermint-abci = { version = "0.23.7", features = ["client"] }
22 | tendermint-proto = "0.23.7"
23 |
24 | narwhal_primary = { package = "primary", git = "https://github.com/asonnino/narwhal/" }
25 | narwhal_crypto = { package = "crypto", git = "https://github.com/asonnino/narwhal/" }
26 | serde_json = "1.0.82"
27 |
--------------------------------------------------------------------------------
/narwhal-abci/narwhal-abci/src/abci_server.rs:
--------------------------------------------------------------------------------
1 | use crate::{AbciQueryQuery, BroadcastTxQuery};
2 |
3 | use eyre::WrapErr;
4 | use futures::SinkExt;
5 | use tendermint_proto::abci::ResponseQuery;
6 | use tokio::sync::mpsc::Sender;
7 | use tokio::sync::oneshot::{channel as oneshot_channel, Sender as OneShotSender};
8 |
9 | use tokio::net::TcpStream;
10 | use tokio_util::codec::{Framed, LengthDelimitedCodec};
11 | use warp::{Filter, Rejection};
12 |
13 | use std::net::SocketAddr;
14 |
15 | /// Simple HTTP API server which listens to messages on:
16 | /// * `broadcast_tx`: forwards them to Narwhal's mempool/worker socket, which will proceed to put
17 | /// it in the consensus process and eventually forward it to the application.
18 | /// * `abci_query`: forwards them over a channel to a handler (typically the application).
19 | pub struct AbciApi {
20 | mempool_address: SocketAddr,
21 | tx: Sender<(OneShotSender, AbciQueryQuery)>,
22 | }
23 |
24 | impl AbciApi {
25 | pub fn new(
26 | mempool_address: SocketAddr,
27 | tx: Sender<(OneShotSender, AbciQueryQuery)>,
28 | ) -> Self {
29 | Self {
30 | mempool_address,
31 | tx,
32 | }
33 | }
34 | }
35 |
36 | impl AbciApi {
37 | pub fn routes(self) -> impl Filter + Clone {
38 | let route_broadcast_tx = warp::path("broadcast_tx")
39 | .and(warp::query::())
40 | .and_then(move |req: BroadcastTxQuery| async move {
41 | log::warn!("broadcast_tx: {:?}", req);
42 |
43 | let stream = TcpStream::connect(self.mempool_address)
44 | .await
45 | .wrap_err(format!(
46 | "ROUTE_BROADCAST_TX failed to connect to {}",
47 | self.mempool_address
48 | ))
49 | .unwrap();
50 | let mut transport = Framed::new(stream, LengthDelimitedCodec::new());
51 |
52 | if let Err(e) = transport.send(req.tx.clone().into()).await {
53 | Ok::<_, Rejection>(format!("ERROR IN: broadcast_tx: {:?}. Err: {}", req, e))
54 | } else {
55 | Ok::<_, Rejection>(format!("broadcast_tx: {:?}", req))
56 | }
57 | });
58 |
59 | let route_abci_query = warp::path("abci_query")
60 | .and(warp::query::())
61 | .and_then(move |req: AbciQueryQuery| {
62 | let tx_abci_queries = self.tx.clone();
63 | async move {
64 | log::warn!("abci_query: {:?}", req);
65 |
66 | let (tx, rx) = oneshot_channel();
67 | match tx_abci_queries.send((tx, req.clone())).await {
68 | Ok(_) => {}
69 | Err(err) => log::error!("Error forwarding abci query: {}", err),
70 | };
71 | let resp = rx.await.unwrap();
72 | // Return the value
73 | Ok::<_, Rejection>(resp.value)
74 | }
75 | });
76 |
77 | route_broadcast_tx.or(route_abci_query)
78 | }
79 | }
80 |
--------------------------------------------------------------------------------
/narwhal-abci/narwhal-abci/src/engine.rs:
--------------------------------------------------------------------------------
1 | use crate::AbciQueryQuery;
2 | use std::net::SocketAddr;
3 | use std::time::Instant;
4 | use tokio::sync::mpsc::Receiver;
5 | use tokio::sync::oneshot::Sender as OneShotSender;
6 |
7 | // Tendermint Types
8 | use tendermint_abci::{Client as AbciClient, ClientBuilder};
9 | use tendermint_proto::abci::{
10 | RequestBeginBlock, RequestDeliverTx, RequestEndBlock, RequestInfo, RequestInitChain,
11 | RequestQuery, ResponseQuery,
12 | };
13 | use tendermint_proto::types::Header;
14 |
15 | // Narwhal types
16 | use narwhal_crypto::Digest;
17 | use narwhal_primary::Certificate;
18 |
19 | /// The engine drives the ABCI Application by concurrently polling for:
20 | /// 1. Calling the BeginBlock -> DeliverTx -> EndBlock -> Commit event loop on the ABCI App on each Bullshark
21 | /// certificate received. It will also call Info and InitChain to initialize the ABCI App if
22 | /// necessary.
23 | /// 2. Processing Query & Broadcast Tx messages received from the Primary's ABCI Server API and forwarding them to the
24 | /// ABCI App via a Tendermint protobuf client.
25 | pub struct Engine {
26 | /// The address of the ABCI app
27 | pub app_address: SocketAddr,
28 | /// The path to the Primary's store, so that the Engine can query each of the Primary's workers
29 | /// for the data corresponding to a Certificate
30 | pub store_path: String,
31 | /// Messages received from the ABCI Server to be forwarded to the engine.
32 | pub rx_abci_queries: Receiver<(OneShotSender, AbciQueryQuery)>,
33 | /// The last block height, initialized to the application's latest block by default
34 | pub last_block_height: i64,
35 | pub client: AbciClient,
36 | pub req_client: AbciClient,
37 | }
38 |
39 | impl Engine {
40 | pub fn new(
41 | app_address: SocketAddr,
42 | store_path: &str,
43 | rx_abci_queries: Receiver<(OneShotSender, AbciQueryQuery)>,
44 | ) -> Self {
45 | let mut client = ClientBuilder::default().connect(&app_address).unwrap();
46 |
47 | let last_block_height = client
48 | .info(RequestInfo::default())
49 | .map(|res| res.last_block_height)
50 | .unwrap_or_default();
51 |
52 | // Instantiate a new client to not be locked in an Info connection
53 | let client = ClientBuilder::default().connect(&app_address).unwrap();
54 | let req_client = ClientBuilder::default().connect(&app_address).unwrap();
55 | Self {
56 | app_address,
57 | store_path: store_path.to_string(),
58 | rx_abci_queries,
59 | last_block_height,
60 | client,
61 | req_client,
62 | }
63 | }
64 |
65 | /// Receives an ordered list of certificates and apply any application-specific logic.
66 | pub async fn run(&mut self, mut rx_output: Receiver) -> eyre::Result<()> {
67 | self.init_chain()?;
68 |
69 | loop {
70 | tokio::select! {
71 | Some(certificate) = rx_output.recv() => {
72 | self.handle_cert(certificate)?;
73 | },
74 | Some((tx, req)) = self.rx_abci_queries.recv() => {
75 | self.handle_abci_query(tx, req)?;
76 | }
77 | else => break,
78 | }
79 | }
80 |
81 | Ok(())
82 | }
83 |
84 | /// On each new certificate, increment the block height to proposed and run through the
85 | /// BeginBlock -> DeliverTx for each tx in the certificate -> EndBlock -> Commit event loop.
86 | fn handle_cert(&mut self, certificate: Certificate) -> eyre::Result<()> {
87 | // increment block
88 | let proposed_block_height = self.last_block_height + 1;
89 |
90 | // save it for next time
91 | self.last_block_height = proposed_block_height;
92 |
93 | // drive the app through the event loop
94 | let now = Instant::now();
95 | self.begin_block(proposed_block_height)?;
96 | self.reconstruct_and_deliver_txs(certificate)?;
97 | self.end_block(proposed_block_height)?;
98 | self.commit()?;
99 | log::info!("time in block: {} ms", now.elapsed().as_millis());
100 |
101 | Ok(())
102 | }
103 |
104 | /// Handles ABCI queries coming to the primary and forwards them to the ABCI App. Each
105 | /// handle call comes with a Sender channel which is used to send the response back to the
106 | /// Primary and then to the client.
107 | ///
108 | /// Client => Primary => handle_cert => ABCI App => Primary => Client
109 | fn handle_abci_query(
110 | &mut self,
111 | tx: OneShotSender,
112 | req: AbciQueryQuery,
113 | ) -> eyre::Result<()> {
114 | let req_height = req.height.unwrap_or(0);
115 | let req_prove = req.prove.unwrap_or(false);
116 |
117 | let resp = self.req_client.query(RequestQuery {
118 | data: req.data.into(),
119 | path: req.path,
120 | height: req_height as i64,
121 | prove: req_prove,
122 | })?;
123 |
124 | if let Err(err) = tx.send(resp) {
125 | eyre::bail!("{:?}", err);
126 | }
127 | Ok(())
128 | }
129 |
130 | /// Opens a RocksDB handle to a Worker's database and tries to read the batch
131 | /// stored at the provided certificate's digest.
132 | fn reconstruct_batch(&self, digest: Digest, worker_id: u32) -> eyre::Result> {
133 | // Open the database to each worker
134 | // TODO: Figure out if this is expensive
135 | let db = rocksdb::DB::open_for_read_only(
136 | &rocksdb::Options::default(),
137 | self.worker_db(worker_id),
138 | true,
139 | )?;
140 |
141 | // Query the db
142 | let key = digest.to_vec();
143 | match db.get(&key) {
144 | Ok(Some(res)) => Ok(res),
145 | Ok(None) => eyre::bail!("digest {} not found", digest),
146 | Err(err) => eyre::bail!(err),
147 | }
148 | }
149 |
150 | /// Calls DeliverTx on the ABCI app
151 | /// Deserializes a raw abtch as `WorkerMesssage::Batch` and proceeds to deliver
152 | /// each transaction over the DeliverTx API.
153 | fn deliver_batch(&mut self, batch: Vec) -> eyre::Result<()> {
154 | // Deserialize and parse the message.
155 | match bincode::deserialize(&batch) {
156 | Ok(WorkerMessage::Batch(batch)) => {
157 | batch.into_iter().try_for_each(|tx| {
158 | self.deliver_tx(tx)?;
159 | Ok::<_, eyre::Error>(())
160 | })?;
161 | }
162 | _ => eyre::bail!("unrecognized message format"),
163 | };
164 | Ok(())
165 | }
166 |
167 | /// Reconstructs the batch corresponding to the provided Primary's certificate from the Workers' stores
168 | /// and proceeds to deliver each tx to the App over ABCI's DeliverTx endpoint.
169 | fn reconstruct_and_deliver_txs(&mut self, certificate: Certificate) -> eyre::Result<()> {
170 | // Try reconstructing the batches from the cert digests
171 | //
172 | // NB:
173 | // This is maybe a false positive by Clippy, without the `collect` the Iterator fails
174 | // iterator fails to compile because we're mutably borrowing in the `try_for_each`
175 | // when we've already immutably borrowed in the `.map`.
176 | #[allow(clippy::needless_collect)]
177 | let batches = certificate
178 | .header
179 | .payload
180 | .into_iter()
181 | .map(|(digest, worker_id)| self.reconstruct_batch(digest, worker_id))
182 | .collect::>();
183 |
184 | // Deliver
185 | batches.into_iter().try_for_each(|batch| {
186 | // this will throw an error if the deserialization failed anywhere
187 | let batch = batch?;
188 | self.deliver_batch(batch)?;
189 | Ok::<_, eyre::Error>(())
190 | })?;
191 |
192 | Ok(())
193 | }
194 |
195 | /// Helper function for getting the database handle to a worker associated
196 | /// with a primary (e.g. Primary db-0 -> Worker-0 db-0-0, Wroekr-1 db-0-1 etc.)
197 | fn worker_db(&self, id: u32) -> String {
198 | format!("{}-{}", self.store_path, id)
199 | }
200 | }
201 |
202 | // Tendermint Lifecycle Helpers
203 | impl Engine {
204 | /// Calls the `InitChain` hook on the app, ignores "already initialized" errors.
205 | pub fn init_chain(&mut self) -> eyre::Result<()> {
206 | let mut client = ClientBuilder::default().connect(&self.app_address)?;
207 | match client.init_chain(RequestInitChain::default()) {
208 | Ok(_) => {}
209 | Err(err) => {
210 | // ignore errors about the chain being uninitialized
211 | if err.to_string().contains("already initialized") {
212 | log::warn!("chain was already initialized: {}", err);
213 | return Ok(());
214 | }
215 | eyre::bail!(err)
216 | }
217 | };
218 | Ok(())
219 | }
220 |
221 | /// Calls the `BeginBlock` hook on the ABCI app. For now, it just makes a request with
222 | /// the new block height.
223 | // If we wanted to, we could add additional arguments to be forwarded from the Consensus
224 | // to the App logic on the beginning of each block.
225 | fn begin_block(&mut self, height: i64) -> eyre::Result<()> {
226 | let req = RequestBeginBlock {
227 | header: Some(Header {
228 | height,
229 | ..Default::default()
230 | }),
231 | ..Default::default()
232 | };
233 |
234 | self.client.begin_block(req)?;
235 | Ok(())
236 | }
237 |
238 | /// Calls the `DeliverTx` hook on the ABCI app.
239 | fn deliver_tx(&mut self, tx: Transaction) -> eyre::Result<()> {
240 | self.client.deliver_tx(RequestDeliverTx { tx })?;
241 | Ok(())
242 | }
243 |
244 | /// Calls the `EndBlock` hook on the ABCI app. For now, it just makes a request with
245 | /// the proposed block height.
246 | // If we wanted to, we could add additional arguments to be forwarded from the Consensus
247 | // to the App logic on the end of each block.
248 | fn end_block(&mut self, height: i64) -> eyre::Result<()> {
249 | let req = RequestEndBlock { height };
250 | self.client.end_block(req)?;
251 | Ok(())
252 | }
253 |
254 | /// Calls the `Commit` hook on the ABCI app.
255 | fn commit(&mut self) -> eyre::Result<()> {
256 | self.client.commit()?;
257 | Ok(())
258 | }
259 | }
260 |
261 | // Helpers for deserializing batches, because `narwhal::worker` is not part
262 | // of the public API. TODO -> make a PR to expose it.
263 | pub type Transaction = Vec;
264 | pub type Batch = Vec;
265 | #[derive(serde::Deserialize)]
266 | pub enum WorkerMessage {
267 | Batch(Batch),
268 | }
269 |
--------------------------------------------------------------------------------
/narwhal-abci/narwhal-abci/src/lib.rs:
--------------------------------------------------------------------------------
1 | mod abci_server;
2 | pub use abci_server::AbciApi;
3 |
4 | mod engine;
5 | pub use engine::Engine;
6 |
7 | use serde::{Deserialize, Serialize};
8 |
9 | #[derive(Serialize, Deserialize, Debug, Clone)]
10 | pub struct BroadcastTxQuery {
11 | tx: String,
12 | }
13 |
14 | #[derive(Serialize, Deserialize, Debug, Clone)]
15 | pub struct AbciQueryQuery {
16 | path: String,
17 | data: String,
18 | height: Option,
19 | prove: Option,
20 | }
21 |
--------------------------------------------------------------------------------
/narwhal-abci/node/Cargo.toml:
--------------------------------------------------------------------------------
1 | [package]
2 | name = "node"
3 | version = "0.1.0"
4 | edition = "2021"
5 |
6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
7 |
8 | [dependencies]
9 | tokio = { version = "1.19.2", features = ["full"] }
10 | tokio-util = { version = "0.6.2", features= ["codec"] }
11 | clap = "2.33.3"
12 | env_logger = "0.7.1"
13 | log = "0.4.11"
14 | bytes = "1.0.1"
15 | bincode = "1.3.1"
16 | rand = "0.7.3"
17 | futures = "0.3.15"
18 | tendermint-abci = { version = "0.23.7", features = ["client"] }
19 | tendermint-proto = "0.23.7"
20 |
21 | config = { git = "https://github.com/asonnino/narwhal/" }
22 | store = { git = "https://github.com/asonnino/narwhal/" }
23 | crypto = { git = "https://github.com/asonnino/narwhal/" }
24 | primary = { git = "https://github.com/asonnino/narwhal/" }
25 | worker = { git = "https://github.com/asonnino/narwhal/" }
26 | consensus = { git = "https://github.com/asonnino/narwhal/" }
27 |
28 | hex = "0.4.3"
29 | ethers = { git = "https://github.com/gakonst/ethers-rs " }
30 | serde_json = "1.0.82"
31 | reqwest = "0.11.11"
32 |
33 | rocksdb = "0.16.0"
34 | warp = "0.3.2"
35 | serde = { version = "1.0", features = ["derive"] }
36 |
37 | narwhal-abci = { path = "../narwhal-abci" }
38 | eyre = "0.6.8"
39 |
--------------------------------------------------------------------------------
/narwhal-abci/node/src/main.rs:
--------------------------------------------------------------------------------
1 | use crypto::PublicKey;
2 | use eyre::{Result, WrapErr};
3 | use std::net::SocketAddr;
4 |
5 | // Copyright(C) Facebook, Inc. and its affiliates.
6 | use clap::{crate_name, crate_version, App, AppSettings, ArgMatches, SubCommand};
7 | use config::Export as _;
8 | use config::Import as _;
9 | use config::{Committee, KeyPair, Parameters, WorkerId};
10 | use consensus::Consensus;
11 | use env_logger::Env;
12 | use primary::Primary;
13 | use store::Store;
14 | use tokio::sync::mpsc::{channel, Receiver};
15 | use worker::Worker;
16 |
17 | use narwhal_abci::{AbciApi, Engine};
18 |
19 | /// The default channel capacity.
20 | pub const CHANNEL_CAPACITY: usize = 1_000;
21 |
22 | #[tokio::main]
23 | async fn main() -> Result<()> {
24 | let matches = App::new(crate_name!())
25 | .version(crate_version!())
26 | .about("A research implementation of Narwhal and Tusk.")
27 | .args_from_usage("-v... 'Sets the level of verbosity'")
28 | .subcommand(
29 | SubCommand::with_name("generate_keys")
30 | .about("Print a fresh key pair to file")
31 | .args_from_usage("--filename= 'The file where to print the new key pair'"),
32 | )
33 | .subcommand(
34 | SubCommand::with_name("run")
35 | .about("Run a node")
36 | .args_from_usage("--keys= 'The file containing the node keys'")
37 | .args_from_usage("--committee= 'The file containing committee information'")
38 | .args_from_usage("--parameters=[FILE] 'The file containing the node parameters'")
39 | .args_from_usage("--store= 'The path where to create the data store'")
40 | .subcommand(
41 | SubCommand::with_name("primary")
42 | .about("Run a single primary")
43 | .args_from_usage(
44 | "--app-api= 'The host of the ABCI app receiving transactions'",
45 | )
46 | .args_from_usage(
47 | "--abci-api= 'The address to receive ABCI connections to'",
48 | ),
49 | )
50 | .subcommand(
51 | SubCommand::with_name("worker")
52 | .about("Run a single worker")
53 | .args_from_usage("--id= 'The worker id'"),
54 | )
55 | .setting(AppSettings::SubcommandRequiredElseHelp),
56 | )
57 | .setting(AppSettings::SubcommandRequiredElseHelp)
58 | .get_matches();
59 |
60 | let log_level = match matches.occurrences_of("v") {
61 | 0 => "error",
62 | 1 => "warn",
63 | 2 => "info",
64 | 3 => "debug",
65 | _ => "trace",
66 | };
67 | let mut logger = env_logger::Builder::from_env(Env::default().default_filter_or(log_level));
68 | #[cfg(feature = "benchmark")]
69 | logger.format_timestamp_millis();
70 | logger.init();
71 |
72 | match matches.subcommand() {
73 | ("generate_keys", Some(sub_matches)) => KeyPair::new()
74 | .export(sub_matches.value_of("filename").unwrap())
75 | .context("Failed to generate key pair")?,
76 | ("run", Some(sub_matches)) => run(sub_matches).await?,
77 | _ => unreachable!(),
78 | }
79 | Ok(())
80 | }
81 |
82 | // Runs either a worker or a primary.
83 | async fn run(matches: &ArgMatches<'_>) -> Result<()> {
84 | let key_file = matches.value_of("keys").unwrap();
85 | let committee_file = matches.value_of("committee").unwrap();
86 | let parameters_file = matches.value_of("parameters");
87 | let store_path = matches.value_of("store").unwrap();
88 |
89 | // Read the committee and node's keypair from file.
90 | let keypair = KeyPair::import(key_file).context("Failed to load the node's keypair")?;
91 | let committee =
92 | Committee::import(committee_file).context("Failed to load the committee information")?;
93 |
94 | // Load default parameters if none are specified.
95 | let parameters = match parameters_file {
96 | Some(filename) => {
97 | Parameters::import(filename).context("Failed to load the node's parameters")?
98 | }
99 | None => Parameters::default(),
100 | };
101 |
102 | // Make the data store.
103 | let store = Store::new(store_path).context("Failed to create a store")?;
104 |
105 | // Channels the sequence of certificates.
106 | let (tx_output, mut rx_output) = channel(CHANNEL_CAPACITY);
107 |
108 | // Check whether to run a primary, a worker, or an entire authority.
109 | match matches.subcommand() {
110 | // Spawn the primary and consensus core.
111 | ("primary", Some(sub_matches)) => {
112 | let (tx_new_certificates, rx_new_certificates) = channel(CHANNEL_CAPACITY);
113 | let (tx_feedback, rx_feedback) = channel(CHANNEL_CAPACITY);
114 |
115 | let keypair_name = keypair.name;
116 |
117 | let app_api = sub_matches.value_of("app-api").unwrap().to_string();
118 | let abci_api = sub_matches.value_of("abci-api").unwrap().to_string();
119 |
120 | Primary::spawn(
121 | keypair,
122 | committee.clone(),
123 | parameters.clone(),
124 | store.clone(),
125 | /* tx_consensus */ tx_new_certificates,
126 | /* rx_consensus */ rx_feedback,
127 | );
128 | Consensus::spawn(
129 | committee.clone(),
130 | parameters.gc_depth,
131 | /* rx_primary */ rx_new_certificates,
132 | /* tx_primary */ tx_feedback,
133 | tx_output,
134 | );
135 |
136 | process(
137 | rx_output,
138 | store_path,
139 | keypair_name,
140 | committee,
141 | abci_api,
142 | app_api,
143 | )
144 | .await?;
145 | }
146 |
147 | // Spawn a single worker.
148 | ("worker", Some(sub_matches)) => {
149 | let id = sub_matches
150 | .value_of("id")
151 | .unwrap()
152 | .parse::()
153 | .context("The worker id must be a positive integer")?;
154 |
155 | Worker::spawn(
156 | keypair.name,
157 | id,
158 | committee.clone(),
159 | parameters,
160 | store.clone(),
161 | );
162 |
163 | // for a worker there is nothing coming here ...
164 | rx_output.recv().await;
165 | }
166 |
167 | _ => unreachable!(),
168 | }
169 |
170 | // If this expression is reached, the program ends and all other tasks terminate.
171 | unreachable!();
172 | }
173 |
174 | async fn process(
175 | rx_output: Receiver,
176 | store_path: &str,
177 | keypair_name: PublicKey,
178 | committee: Committee,
179 | abci_api: String,
180 | app_api: String,
181 | ) -> eyre::Result<()> {
182 | // address of mempool
183 | let mempool_address = committee
184 | .worker(&keypair_name.clone(), &0)
185 | .expect("Our public key or worker id is not in the committee")
186 | .transactions;
187 |
188 | // ABCI queries will be sent using this from the RPC to the ABCI client
189 | let (tx_abci_queries, rx_abci_queries) = channel(CHANNEL_CAPACITY);
190 |
191 | tokio::spawn(async move {
192 | let api = AbciApi::new(mempool_address, tx_abci_queries);
193 | // let tx_abci_queries = tx_abci_queries.clone();
194 | // Spawn the ABCI RPC endpoint
195 | let mut address = abci_api.parse::().unwrap();
196 | address.set_ip("0.0.0.0".parse().unwrap());
197 | warp::serve(api.routes()).run(address).await
198 | });
199 |
200 | // Analyze the consensus' output.
201 | // Spawn the network receiver listening to messages from the other primaries.
202 | let mut app_address = app_api.parse::().unwrap();
203 | app_address.set_ip("0.0.0.0".parse().unwrap());
204 | let mut engine = Engine::new(app_address, store_path, rx_abci_queries);
205 | engine.run(rx_output).await?;
206 |
207 | Ok(())
208 | }
209 |
--------------------------------------------------------------------------------
/narwhal-abci/starknet-abci/Cargo.toml:
--------------------------------------------------------------------------------
1 | [package]
2 | name = "starknet-abci"
3 | version = "0.1.0"
4 | edition = "2021"
5 |
6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
7 |
8 | [dependencies]
9 | abci-rs = { version = "0.11.3", features = ["async-api" ] }
10 | async-trait = "0.1.56"
11 | eyre = "0.6.8"
12 | hex = "0.4.3"
13 | serde_json = "1.0.82"
14 | tendermint-abci = { version = "0.23.7", features = ["client"] }
15 | tendermint-proto = "0.23.7"
16 | tokio = { version = "1.19.2", features = ["macros"] }
17 | clap = { version = "3.0.10", features = [
18 | "derive",
19 | "env",
20 | "unicode",
21 | "wrap_help",
22 | ] }
23 | clap_complete = "3.0.4"
24 | serde = { version = "1.0.138", features = ["derive"] }
25 | reqwest = "0.11.11"
26 | sha2 = "0.10.6"
27 | tracing = "0.1.35"
28 | tracing-subscriber = { version = "0.3", features = ["registry", "env-filter", "fmt"] }
29 | tracing-error = "0.2.0"
30 | yansi = "0.5.1"
31 | uuid = { version = "1.2.1", features = ["v4"] }
32 | anyhow = "1.0.66"
33 | once_cell = "1.13.0"
34 | bincode = "1.3.3"
35 | starknet-rs = { git = "https://github.com/lambdaclass/starknet_in_rust", branch= "publish-structs" }
36 | # This was copied from starkent_in_rust/Cargo.toml as it seems it is missing an export for it
37 | felt = { git = "https://github.com/lambdaclass/cairo-rs", package = "cairo-felt", rev = "8dba86dbec935fa04a255e2edf3d5d184950fa22" }
38 |
--------------------------------------------------------------------------------
/narwhal-abci/starknet-abci/src/app.rs:
--------------------------------------------------------------------------------
1 |
2 | use crate::{Consensus, Info, Mempool, Snapshot, State};
3 | use std::sync::{Arc, Mutex};
4 |
5 | pub struct App {
6 | pub mempool: Mempool,
7 | pub snapshot: Snapshot,
8 | pub consensus: Consensus,
9 | pub info: Info,
10 | }
11 |
12 | impl Default for App {
13 | fn default() -> Self {
14 | Self::new()
15 | }
16 | }
17 |
18 | // demo
19 | impl App {
20 | pub fn new() -> Self {
21 | let state = State {
22 | block_height: Default::default(),
23 | app_hash: Default::default(),
24 | };
25 |
26 | let committed_state = Arc::new(Mutex::new(state.clone()));
27 |
28 | let consensus = Consensus::new(state);
29 | let mempool = Mempool::default();
30 |
31 | let info = Info {
32 | state: committed_state,
33 | };
34 | let snapshot = Snapshot::default();
35 |
36 | App {
37 | consensus,
38 | mempool,
39 | info,
40 | snapshot,
41 | }
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/narwhal-abci/starknet-abci/src/bin/client.rs:
--------------------------------------------------------------------------------
1 | use eyre::Result;
2 | use starknet_abci::transaction::{Transaction, TransactionType};
3 |
4 | async fn send_transaction(host: &str) -> Result<()> {
5 | let transaction_type = TransactionType::FunctionExecution {
6 | function: "main".to_string(),
7 | program_name: "fibonacci.json".to_string(),
8 | };
9 |
10 | let tx = Transaction::with_type(transaction_type).unwrap();
11 |
12 | let tx = serde_json::to_string(&tx)?;
13 |
14 | let client = reqwest::Client::new();
15 | client
16 | .get(format!("{}/broadcast_tx", host))
17 | .query(&[("tx", tx)])
18 | .send()
19 | .await?;
20 |
21 | Ok(())
22 | }
23 |
24 | #[tokio::main]
25 | async fn main() -> Result<()> {
26 | // the ABCI port on the various narwhal primaries
27 | let hosts = ["http://127.0.0.1:3002", "http://127.0.0.1:3009", "http://127.0.0.1:3016"];
28 |
29 | unsafe{
30 | for i in 0..200 {
31 | let tx_result = send_transaction(hosts.get_unchecked(i%3)).await;
32 | match tx_result
33 | {
34 | Ok(_) => println!("transaction committed to {}", hosts.get_unchecked(i%3)),
35 | Err(e) => println!("error: {}", e),
36 | }
37 | std::thread::sleep(std::time::Duration::from_millis(5));
38 | }
39 | }
40 | Ok(())
41 | }
42 |
--------------------------------------------------------------------------------
/narwhal-abci/starknet-abci/src/bin/starknet-app.rs:
--------------------------------------------------------------------------------
1 | use abci::async_api::Server;
2 | use starknet_abci::App;
3 | use std::net::SocketAddr;
4 |
5 | use clap::Parser;
6 |
7 | #[derive(Debug, Clone, Parser)]
8 | struct Args {
9 | #[clap(default_value = "0.0.0.0:26658")]
10 | host: String,
11 | #[clap(long, short)]
12 | demo: bool,
13 | }
14 |
15 | use tracing_error::ErrorLayer;
16 |
17 | use tracing_subscriber::prelude::*;
18 |
19 | /// Initializes a tracing Subscriber for logging
20 | #[allow(dead_code)]
21 | pub fn subscriber() {
22 | tracing_subscriber::Registry::default()
23 | .with(tracing_subscriber::EnvFilter::new("starknet-app=info"))
24 | .with(ErrorLayer::default())
25 | .with(tracing_subscriber::fmt::layer())
26 | .init()
27 | }
28 |
29 | #[tokio::main]
30 | async fn main() -> eyre::Result<()> {
31 | let args = Args::parse();
32 | subscriber();
33 |
34 | let App {
35 | consensus,
36 | mempool,
37 | info,
38 | snapshot,
39 | } = App::new();
40 | let server = Server::new(consensus, mempool, info, snapshot);
41 |
42 | dbg!(&args.host);
43 | let addr = args.host.parse::().unwrap();
44 |
45 | server.run(addr).await?;
46 |
47 | Ok(())
48 | }
49 |
--------------------------------------------------------------------------------
/narwhal-abci/starknet-abci/src/lib.rs:
--------------------------------------------------------------------------------
1 | mod app;
2 | pub use app::App;
3 |
4 | pub mod types;
5 | pub use types::{Consensus, Info, Mempool, Snapshot, State};
6 | pub mod transaction;
--------------------------------------------------------------------------------
/narwhal-abci/starknet-abci/src/transaction.rs:
--------------------------------------------------------------------------------
1 | use std::collections::HashMap;
2 |
3 | use anyhow::{ensure, Result};
4 | use felt::Felt;
5 | use serde::{Deserialize, Serialize};
6 | use sha2::{Digest, Sha256};
7 | use starknet_rs::business_logic::execution::execution_entry_point::ExecutionEntryPoint;
8 | use starknet_rs::business_logic::execution::objects::{CallType, TransactionExecutionContext};
9 | use starknet_rs::business_logic::fact_state::contract_state::ContractState;
10 | use starknet_rs::business_logic::fact_state::in_memory_state_reader::InMemoryStateReader;
11 | use starknet_rs::business_logic::fact_state::state::ExecutionResourcesManager;
12 | use starknet_rs::business_logic::state::cached_state::CachedState;
13 | use starknet_rs::definitions::general_config::StarknetGeneralConfig;
14 | use starknet_rs::services::api::contract_class::{ContractClass, EntryPointType};
15 | use starknet_rs::utils::Address;
16 | use uuid::Uuid;
17 |
18 | #[derive(Clone, Serialize, Deserialize, Debug)]
19 | pub struct Transaction {
20 | pub id: String,
21 | pub transaction_hash: String, // this acts
22 | pub transaction_type: TransactionType,
23 | }
24 |
25 | #[derive(Clone, Serialize, Deserialize, Debug)]
26 | pub enum TransactionType {
27 | /// Create new contract class.
28 | Declare,
29 |
30 | /// Create an instance of a contract which will have storage assigned. (Accounts are a contract themselves)
31 | Deploy,
32 |
33 | /// Execute a function from a deployed contract.
34 | Invoke,
35 |
36 | // TODO: Remove this when other transactions are implemented
37 | FunctionExecution {
38 | function: String,
39 | program_name: String,
40 | },
41 | }
42 |
43 | impl Transaction {
44 | pub fn with_type(transaction_type: TransactionType) -> Result {
45 | Ok(Transaction {
46 | transaction_hash: transaction_type.compute_and_hash()?,
47 | transaction_type,
48 | id: Uuid::new_v4().to_string(),
49 | })
50 | }
51 |
52 | /// Verify that the transaction id is consistent with its contents, by checking its sha256 hash.
53 | pub fn verify(&self) -> Result<()> {
54 | ensure!(
55 | self.transaction_hash == self.transaction_type.compute_and_hash()?,
56 | "Corrupted transaction: Inconsistent transaction id"
57 | );
58 |
59 | Ok(())
60 | }
61 | }
62 |
63 | impl TransactionType {
64 | pub fn compute_and_hash(&self) -> Result {
65 | match self {
66 | TransactionType::FunctionExecution {
67 | function,
68 | program_name: _,
69 | } => {
70 | let general_config = StarknetGeneralConfig::default();
71 |
72 | let tx_execution_context = TransactionExecutionContext::create_for_testing(
73 | Address(0.into()),
74 | 10,
75 | 0.into(),
76 | general_config.invoke_tx_max_n_steps(),
77 | 1,
78 | );
79 |
80 | let contract_address = Address(1111.into());
81 | let class_hash = [1; 32];
82 | let program = include_str!("../programs/fibonacci.json");
83 | let contract_class = ContractClass::try_from(program.to_string())
84 | .expect("Could not load contract from JSON");
85 |
86 | let contract_state = ContractState::new(
87 | class_hash,
88 | tx_execution_context.nonce().clone(),
89 | Default::default(),
90 | );
91 | let mut state_reader = InMemoryStateReader::new(HashMap::new(), HashMap::new());
92 | state_reader
93 | .contract_states_mut()
94 | .insert(contract_address.clone(), contract_state);
95 |
96 | let mut state = CachedState::new(
97 | state_reader,
98 | Some([(class_hash, contract_class)].into_iter().collect()),
99 | );
100 |
101 | let entry_point = ExecutionEntryPoint::new(
102 | contract_address,
103 | vec![],
104 | Felt::from_bytes_be(&starknet_rs::utils::calculate_sn_keccak(
105 | function.as_bytes(),
106 | )),
107 | Address(0.into()),
108 | EntryPointType::External,
109 | CallType::Delegate.into(),
110 | class_hash.into(),
111 | );
112 |
113 | let mut resources_manager = ExecutionResourcesManager::default();
114 |
115 | entry_point
116 | .execute(
117 | &mut state,
118 | &general_config,
119 | &mut resources_manager,
120 | &tx_execution_context,
121 | )
122 | .expect("Could not execute contract");
123 |
124 | let mut hasher = Sha256::new();
125 | hasher.update(function);
126 | let hash = hasher.finalize().as_slice().to_owned();
127 | Ok(hex::encode(hash))
128 | }
129 | TransactionType::Declare => todo!(),
130 | TransactionType::Deploy => todo!(),
131 | TransactionType::Invoke => todo!(),
132 | }
133 | }
134 | }
135 |
--------------------------------------------------------------------------------
/narwhal-abci/starknet-abci/src/types.rs:
--------------------------------------------------------------------------------
1 | use once_cell::sync::Lazy;
2 | use starknet_rs::testing::starknet_state::StarknetState;
3 | use tracing::{debug, info};
4 | use std::{sync::Arc, sync::Mutex,time::Instant};
5 | use sha2::{Digest, Sha256};
6 | use crate::transaction::{Transaction, TransactionType};
7 |
8 | use abci::{
9 | async_api::{
10 | Consensus as ConsensusTrait, Info as InfoTrait, Mempool as MempoolTrait,
11 | Snapshot as SnapshotTrait,
12 | },
13 | async_trait,
14 | types::*,
15 | };
16 |
17 |
18 | /// The app's state, containing a Revm DB.
19 | // TODO: Should we instead try to replace this with Anvil and implement traits for it?
20 | #[derive(Clone, Debug)]
21 | pub struct State {
22 | pub block_height: i64,
23 | pub app_hash: Vec,
24 | }
25 |
26 | impl Default for State {
27 | fn default() -> Self {
28 | Self {
29 | block_height: 0,
30 | app_hash: Vec::new(),
31 | }
32 | }
33 | }
34 |
35 | // because we don't get a `&mut self` in the ABCI API, we opt to have a mod-level variable
36 | // and because beginblock, endblock and deliver_tx all happen in the same thread, this is safe to do
37 | // an alternative would be Arc>, but we want to avoid extra-overhead of locks for the benchmark's sake
38 | static mut TRANSACTIONS: usize = 0;
39 | static mut TIMER: Lazy = Lazy::new(Instant::now);
40 |
41 | #[derive(serde::Serialize, serde::Deserialize, Debug)]
42 | pub struct TransactionResult {
43 | gas: u64,
44 | }
45 |
46 | pub struct Consensus {
47 | pub committed_state: Arc>,
48 | pub current_state: Arc>,
49 | hasher: Arc>,
50 | starknet_state: StarknetState,
51 | }
52 |
53 | impl Consensus {
54 | pub fn new(state: State) -> Self {
55 | let committed_state = Arc::new(Mutex::new(state.clone()));
56 | let current_state = Arc::new(Mutex::new(state));
57 |
58 | Consensus {
59 | committed_state,
60 | current_state,
61 | hasher: Arc::new(Mutex::new(Sha256::new())),
62 | starknet_state: StarknetState::new(None),
63 | }
64 | }
65 | }
66 |
67 | #[async_trait]
68 | impl ConsensusTrait for Consensus {
69 | #[tracing::instrument(skip(self))]
70 | async fn init_chain(&self, _init_chain_request: RequestInitChain) -> ResponseInitChain {
71 | ResponseInitChain::default()
72 | }
73 |
74 | #[tracing::instrument(skip(self))]
75 | async fn begin_block(&self, _begin_block_request: RequestBeginBlock) -> ResponseBeginBlock {
76 | // because begin_block, [deliver_tx] and end_block/commit are on the same thread, this is safe to do (see declaration of statics)
77 | unsafe {
78 | info!(
79 | "{} ms passed between begin_block() calls. {} transactions, {} tps",
80 | (*TIMER).elapsed().as_millis(),
81 | TRANSACTIONS,
82 | (TRANSACTIONS * 1000) as f32 / ((*TIMER).elapsed().as_millis() as f32)
83 | );
84 | TRANSACTIONS = 0;
85 |
86 | *TIMER = Instant::now();
87 | }
88 |
89 | Default::default()
90 | }
91 |
92 | #[tracing::instrument(skip(self))]
93 | async fn deliver_tx(&self, request: RequestDeliverTx) -> ResponseDeliverTx {
94 | tracing::trace!("delivering tx");
95 |
96 | let tx: Transaction = serde_json::from_slice(&request.tx).unwrap();
97 |
98 | // Validation consists of getting the hash and checking whether it is equal
99 | // to the tx id. The hash executes the program and hashes the trace.
100 |
101 | let tx_hash = tx
102 | .transaction_type
103 | .compute_and_hash()
104 | .map(|x| x == tx.transaction_hash);
105 |
106 | // because begin_block, [deliver_tx] and end_block/commit are on the same thread, this is safe to do (see declaration of statics)
107 | unsafe {
108 | TRANSACTIONS += 1;
109 | }
110 |
111 | match tx_hash {
112 | Ok(true) => {
113 | let _ = self
114 | .hasher
115 | .lock()
116 | .map(|mut hash| hash.update(tx.transaction_hash.clone()));
117 |
118 | // prepare this transaction to be queried by app.tx_id
119 | let index_event = Event {
120 | r#type: "app".to_string(),
121 | attributes: vec![EventAttribute {
122 | key: "tx_id".to_string().into_bytes(),
123 | value: tx.transaction_hash.to_string().into_bytes(),
124 | index: true,
125 | }],
126 | };
127 | let mut events = vec![index_event];
128 |
129 | match tx.transaction_type {
130 | TransactionType::FunctionExecution {
131 | function,
132 | program_name: _,
133 | } => {
134 | let function_event = Event {
135 | r#type: "function".to_string(),
136 | attributes: vec![EventAttribute {
137 | key: "function".to_string().into_bytes(),
138 | value: function.into_bytes(),
139 | index: true,
140 | }],
141 | };
142 | events.push(function_event);
143 | }
144 | TransactionType::Declare => todo!(),
145 | TransactionType::Deploy => todo!(),
146 | TransactionType::Invoke => todo!(),
147 | }
148 |
149 | ResponseDeliverTx {
150 | events,
151 | data: tx.transaction_hash.into_bytes(),
152 | ..Default::default()
153 | }
154 | }
155 | Ok(false) => ResponseDeliverTx {
156 | code: 1,
157 | log: "Error delivering transaction. Integrity check failed.".to_string(),
158 | info: "Error delivering transaction. Integrity check failed.".to_string(),
159 | ..Default::default()
160 | },
161 | Err(e) => ResponseDeliverTx {
162 | code: 1,
163 | log: format!("Error delivering transaction: {e}"),
164 | info: format!("Error delivering transaction: {e}"),
165 | ..Default::default()
166 | },
167 | }
168 | }
169 |
170 | #[tracing::instrument(skip(self))]
171 | async fn end_block(&self, _end_block_request: RequestEndBlock) -> ResponseEndBlock {
172 | // because begin_block, [deliver_tx] and end_block/commit are on the same thread, this is safe to do (see declaration of statics)
173 | unsafe {
174 | info!(
175 | "Committing block with {} transactions in {} ms. TPS: {}",
176 | TRANSACTIONS,
177 | (*TIMER).elapsed().as_millis(),
178 | (TRANSACTIONS * 1000) as f32 / ((*TIMER).elapsed().as_millis() as f32)
179 | );
180 | }
181 | ResponseEndBlock {
182 | ..Default::default()
183 | }
184 | }
185 |
186 | #[tracing::instrument(skip(self))]
187 | async fn commit(&self, _commit_request: RequestCommit) -> ResponseCommit {
188 |
189 | let app_hash: Result, String> = Ok(vec![]);
190 |
191 | // because begin_block, [deliver_tx] and end_block/commit are on the same thread, this is safe to do (see declaration of statics)
192 | unsafe {
193 | info!(
194 | "Committing block with {} transactions in {} ms. TPS: {}",
195 | TRANSACTIONS,
196 | (*TIMER).elapsed().as_millis(),
197 | (TRANSACTIONS * 1000) as f32 / ((*TIMER).elapsed().as_millis() as f32)
198 | );
199 | }
200 |
201 | match app_hash {
202 | Ok(hash) => ResponseCommit {
203 | data: hash,
204 | retain_height: 0,
205 | },
206 | // error should be handled here
207 | _ => ResponseCommit {
208 | data: vec![],
209 | retain_height: 0,
210 | },
211 | }
212 | }
213 | }
214 |
215 | #[derive(Debug, Clone, Default)]
216 | pub struct Mempool;
217 |
218 | #[async_trait]
219 | impl MempoolTrait for Mempool {
220 | async fn check_tx(&self, _check_tx_request: RequestCheckTx) -> ResponseCheckTx {
221 | ResponseCheckTx::default()
222 | }
223 | }
224 |
225 | #[derive(Debug, Clone)]
226 | pub struct Info {
227 | pub state: Arc>,
228 | }
229 |
230 | #[async_trait]
231 | impl InfoTrait for Info {
232 | // replicate the eth_call interface
233 | async fn query(&self, query_request: RequestQuery) -> ResponseQuery {
234 | ResponseQuery {
235 | key: query_request.data,
236 | ..Default::default()
237 | }
238 | }
239 |
240 | async fn info(&self, info_request: RequestInfo) -> ResponseInfo {
241 | debug!(
242 | "Got info request. Tendermint version: {}; Block version: {}; P2P version: {}",
243 | info_request.version, info_request.block_version, info_request.p2p_version
244 | );
245 |
246 | ResponseInfo {
247 | data: "cairo-app".to_string(),
248 | version: "0.1.0".to_string(),
249 | app_version: 1,
250 | last_block_height: 0i64,
251 |
252 | // using a fixed hash, see the commit() hook
253 | last_block_app_hash: vec![],
254 | }
255 | }
256 | }
257 |
258 | #[derive(Debug, Clone, Default)]
259 | pub struct Snapshot;
260 |
261 | impl SnapshotTrait for Snapshot {}
262 |
263 |
264 | /*
265 | #[cfg(test)]
266 | mod tests {
267 | use super::*;
268 | // use ethers::prelude::*;
269 |
270 | #[tokio::test]
271 | async fn run_and_query_tx() {
272 | let val = ethers::utils::parse_units(1, 18).unwrap();
273 | let alice = Address::random();
274 | let bob = Address::random();
275 |
276 | let mut state = State::default();
277 |
278 | // give alice some money
279 | state.db.insert_account_info(
280 | alice,
281 | revm::AccountInfo {
282 | balance: val,
283 | ..Default::default()
284 | },
285 | );
286 |
287 | // make the tx
288 | let tx = TransactionRequest::new()
289 | .from(alice)
290 | .to(bob)
291 | .gas_price(0)
292 | .data(vec![1, 2, 3, 4, 5])
293 | .gas(31000)
294 | .value(val);
295 |
296 | // Send it over an ABCI message
297 |
298 | let consensus = Consensus::new(state);
299 |
300 | let req = RequestDeliverTx {
301 | tx: serde_json::to_vec(&tx).unwrap(),
302 | };
303 | let res = consensus.deliver_tx(req).await;
304 | let res: TransactionResult = serde_json::from_slice(&res.data).unwrap();
305 | // tx passed
306 | assert_eq!(res.exit, Return::Stop);
307 |
308 | // now we query the state for bob's balance
309 | let info = Info {
310 | state: consensus.current_state.clone(),
311 | };
312 | let res = info
313 | .query(RequestQuery {
314 | data: serde_json::to_vec(&Query::Balance(bob)).unwrap(),
315 | ..Default::default()
316 | })
317 | .await;
318 | let res: QueryResponse = serde_json::from_slice(&res.value).unwrap();
319 | let balance = res.as_balance();
320 | assert_eq!(balance, val);
321 | }
322 | } */
323 |
--------------------------------------------------------------------------------
/sequencer/.gitignore:
--------------------------------------------------------------------------------
1 | # Generated by Cargo
2 | # will have compiled files and executables
3 | /.DS_Store
4 | **/target/
5 | target/
6 |
7 | .vscode/
8 |
9 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
10 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
11 | Cargo.lock
12 |
13 | hello/build
14 | bin/
15 | abci.*
16 | testnet/
17 | localnet/
18 | ./tendermint-install
19 |
20 | # Grafana local storage
21 | grafana-storage/
22 |
--------------------------------------------------------------------------------
/sequencer/Cargo.toml:
--------------------------------------------------------------------------------
1 | [package]
2 | name = "starknet-sequencer"
3 | version = "0.1.0"
4 | edition = "2021"
5 | default-run = "cli"
6 |
7 | [[bin]]
8 | path = "src/cli/main.rs"
9 | doctest = false
10 | name = "cli"
11 |
12 | [[bin]]
13 | path = "src/abci/main.rs"
14 | name = "abci"
15 |
16 | [[bin]]
17 | path = "src/bench/main.rs"
18 | name = "bench"
19 |
20 | [lib]
21 | path = "src/lib/mod.rs"
22 | doctest = false
23 | name = "lib"
24 |
25 | [profile.test]
26 | opt-level = 3
27 | debug-assertions = true
28 |
29 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
30 |
31 | [dependencies]
32 | anyhow = "1.0.66"
33 | bincode = "1.3.3"
34 | clap = { version = "4.0.5", features = ["derive", "env"] }
35 | once_cell = "*"
36 | futures = "0.3.26"
37 | hex = "0.4.3"
38 | sha2 = "0.10.6"
39 | serde = "1.0"
40 | serde_json = { version = "1.0", features = ["raw_value"] }
41 | tendermint = "0.30.0"
42 | tendermint-abci = "0.30.0"
43 | tendermint-proto = { version = "0.30.0", default-features = false }
44 | tendermint-rpc = { version = "0.30.0", features = ["http-client"] }
45 | tracing = "0.1"
46 | tracing-subscriber = {version = "0.3", features = ["env-filter", "fmt", "std"]}
47 | tokio = { version = "1.15.0", features = ["full"] }
48 | uuid = { version = "1.2.1", features = ["v4"] }
49 | starknet-rs = { git = "https://github.com/lambdaclass/starknet_in_rust", rev = "4ab3433c51df485cd205142ce96a92559b21a2e2" }
50 | # This was copied from starkent_in_rust/Cargo.toml as it seems it is missing an export for it
51 | felt = { git = "https://github.com/lambdaclass/cairo-rs", package = "cairo-felt", rev="77fe09ebbf72710935b455b1c5ff56b0bad7a4b8" }
52 | num-traits = "0.2.15"
53 |
54 | [dev-dependencies]
55 | assert_fs = "1.0.9"
56 | assert_cmd = "2.0.6"
57 | retry = "2.0.0"
58 | serial_test = "1.0.0"
59 | ctor = "0.1.23"
--------------------------------------------------------------------------------
/sequencer/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/sequencer/bench/cmd/load_test/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "github.com/informalsystems/tm-load-test/pkg/loadtest"
5 | "github.com/lambdaclass/load_tester/pkg/abci"
6 | )
7 |
8 | func main() {
9 | if err := loadtest.RegisterClientFactory("my-abci-app-name", &abci.MyABCIAppClientFactory{}); err != nil {
10 | panic(err)
11 | }
12 | // The loadtest.Run method will handle CLI argument parsing, errors,
13 | // configuration, instantiating the load test and/or coordinator/worker
14 | // operations, etc. All it needs is to know which client factory to use for
15 | // its load testing.
16 | loadtest.Run(&loadtest.CLIConfig{
17 | AppName: "my-load-tester",
18 | AppShortDesc: "Load testing application for My ABCI App (TM)",
19 | AppLongDesc: "Some long description on how to use the tool",
20 | DefaultClientFactory: "my-abci-app-name",
21 | })
22 | }
--------------------------------------------------------------------------------
/sequencer/bench/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/lambdaclass/load_tester
2 |
3 | go 1.19
4 |
5 | require github.com/informalsystems/tm-load-test v1.3.0
6 |
7 | require (
8 | github.com/beorn7/perks v1.0.1 // indirect
9 | github.com/cespare/xxhash/v2 v2.1.2 // indirect
10 | github.com/golang/protobuf v1.5.2 // indirect
11 | github.com/google/uuid v1.3.0 // indirect
12 | github.com/gorilla/websocket v1.5.0 // indirect
13 | github.com/inconshreveable/mousetrap v1.0.1 // indirect
14 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
15 | github.com/prometheus/client_golang v1.14.0 // indirect
16 | github.com/prometheus/client_model v0.3.0 // indirect
17 | github.com/prometheus/common v0.37.0 // indirect
18 | github.com/prometheus/procfs v0.8.0 // indirect
19 | github.com/satori/go.uuid v1.2.0 // indirect
20 | github.com/sirupsen/logrus v1.9.0 // indirect
21 | github.com/spf13/cobra v1.6.1 // indirect
22 | github.com/spf13/pflag v1.0.5 // indirect
23 | golang.org/x/sys v0.4.0 // indirect
24 | google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 // indirect
25 | )
26 |
--------------------------------------------------------------------------------
/sequencer/bench/pkg/abci/client.go:
--------------------------------------------------------------------------------
1 | package abci
2 |
3 | import (
4 | "github.com/informalsystems/tm-load-test/pkg/loadtest"
5 | "github.com/google/uuid"
6 | "log"
7 | "os/exec"
8 | "bytes"
9 | )
10 |
11 | // MyABCIAppClientFactory creates instances of MyABCIAppClient
12 | type MyABCIAppClientFactory struct {}
13 |
14 | // MyABCIAppClientFactory implements loadtest.ClientFactory
15 | var _ loadtest.ClientFactory = (*MyABCIAppClientFactory)(nil)
16 |
17 | // MyABCIAppClient is responsible for generating transactions. Only one client
18 | // will be created per connection to the remote Tendermint RPC endpoint, and
19 | // each client will be responsible for maintaining its own state in a
20 | // thread-safe manner.
21 | type MyABCIAppClient struct {
22 | tx []byte
23 | }
24 |
25 | // MyABCIAppClient implements loadtest.Client
26 | var _ loadtest.Client = (*MyABCIAppClient)(nil)
27 |
28 | func (f *MyABCIAppClientFactory) ValidateConfig(cfg loadtest.Config) error {
29 | // Do any checks here that you need to ensure that the load test
30 | // configuration is compatible with your client.
31 | return nil
32 | }
33 |
34 | func (f *MyABCIAppClientFactory) NewClient(cfg loadtest.Config) (loadtest.Client, error) {
35 | cmd := exec.Command("cargo", "run", "--release", "programs/fibonacci.json", "main", "--no-broadcast")
36 | var outb, errb bytes.Buffer
37 | cmd.Stdout = &outb
38 | cmd.Stderr = &errb
39 | err := cmd.Run()
40 | if err != nil {
41 | log.Fatal(err)
42 | }
43 | return &MyABCIAppClient{tx: outb.Bytes()}, nil
44 | }
45 |
46 | // GenerateTx must return the raw bytes that make up the transaction for your
47 | // ABCI app. The conversion to base64 will automatically be handled by the
48 | // loadtest package, so don't worry about that. Only return an error here if you
49 | // want to completely fail the entire load test operation.
50 | func (c *MyABCIAppClient) GenerateTx() ([]byte, error) {
51 | var newTx []byte
52 | newTx = append(newTx, c.tx[0:8]...)
53 | // Replacing the uuid with a new random one to prevent getting duplicated tx rejected
54 | newTx = append(newTx, []byte(uuid.New().String())...)
55 | newTx = append(newTx, c.tx[44:]...)
56 | return newTx, nil
57 | }
--------------------------------------------------------------------------------
/sequencer/docker-compose.yml:
--------------------------------------------------------------------------------
1 | services:
2 | prometheus:
3 | image: prom/prometheus
4 | ports:
5 | - "9090:9090"
6 | volumes:
7 | - ./prometheus.yml:/etc/prometheus/prometheus.yml
8 | grafana:
9 | image: grafana/grafana-oss
10 | ports:
11 | - "3000:3000"
12 | volumes:
13 | - ./grafana-storage:/var/lib/grafana
14 |
--------------------------------------------------------------------------------
/sequencer/playbooks/deploy.yaml:
--------------------------------------------------------------------------------
1 | - name: Deploy new version
2 | hosts: tendermint-nodes
3 | gather_facts: false
4 | any_errors_fatal: true
5 | become: true
6 | become_user: root
7 |
8 | pre_tasks:
9 |
10 | - name: Stop tendermint service
11 | ansible.builtin.systemd:
12 | state: stopped
13 | name: tendermint
14 | daemon_reload: true
15 |
16 | - name: Stop abci service
17 | ansible.builtin.systemd:
18 | state: stopped
19 | name: abci
20 | daemon_reload: true
21 |
22 | tasks:
23 |
24 | - name: Reset tendermint network
25 | ansible.builtin.shell: |
26 | /usr/bin/tendermint unsafe_reset_all --home /.tendermint
27 |
28 | - name: Delete starknet_tendermint_sequencer repo
29 | ansible.builtin.file:
30 | state: absent
31 | path: /root/starknet_tendermint_sequencer
32 |
33 | - name: Clone starknet_tendermint_sequencer repo
34 | ansible.builtin.git:
35 | repo: git@github.com:lambdaclass/starknet_tendermint_sequencer.git
36 | dest: /root/starknet_tendermint_sequencer
37 | version: main
38 | accept_hostkey: true
39 |
40 | post_tasks:
41 |
42 | - name: Start abci service
43 | ansible.builtin.systemd:
44 | state: started
45 | name: abci
46 | daemon_reload: true
47 |
48 | - name: Start tendermint service
49 | ansible.builtin.systemd:
50 | state: started
51 | name: tendermint
52 | daemon_reload: true
53 |
--------------------------------------------------------------------------------
/sequencer/programs/factorial.cairo:
--------------------------------------------------------------------------------
1 | // factorial(n) = n!
2 | func factorial(n) -> (result: felt) {
3 | if (n == 1) {
4 | return (n,);
5 | }
6 | let (a) = factorial(n - 1);
7 | return (n * a,);
8 | }
9 |
10 | func main() {
11 | // Make sure the factorial(10) == 3628800
12 | let (y) = factorial(10);
13 | y = 3628800;
14 | return ();
15 | }
--------------------------------------------------------------------------------
/sequencer/programs/fibonacci.cairo:
--------------------------------------------------------------------------------
1 | %lang starknet
2 |
3 | @external
4 | func main() {
5 | // Call fib(1, 1, 10).
6 | let result: felt = fib(1, 1, 500);
7 | ret;
8 | }
9 |
10 | func fib(first_element, second_element, n) -> (res: felt) {
11 | jmp fib_body if n != 0;
12 | tempvar result = second_element;
13 | return (second_element,);
14 |
15 | fib_body:
16 | tempvar y = first_element + second_element;
17 | return fib(second_element, y, n - 1);
18 | }
19 |
--------------------------------------------------------------------------------
/sequencer/prometheus.yml:
--------------------------------------------------------------------------------
1 | # docker run -p 9090:9090 -v /Users/aminarria/Lambda/starknet_tendermint_sequencer/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus
2 |
3 | # my global config
4 | global:
5 | scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
6 | evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
7 | # scrape_timeout is set to the global default (10s).
8 |
9 | # Alertmanager configuration
10 | alerting:
11 | alertmanagers:
12 | - static_configs:
13 | - targets:
14 | # - alertmanager:9093
15 |
16 | # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
17 | rule_files:
18 | # - "first_rules.yml"
19 | # - "second_rules.yml"
20 |
21 | # A scrape configuration containing exactly one endpoint to scrape:
22 | # Here it's Prometheus itself.
23 | scrape_configs:
24 | # The job name is added as a label `job=` to any timeseries scraped from this config.
25 | - job_name: "prometheus"
26 |
27 | # metrics_path defaults to '/metrics'
28 | # scheme defaults to 'http'.
29 |
30 | static_configs:
31 | - targets: ["localhost:9090"]
32 | - targets: ["host.docker.internal:26660"]
33 | labels:
34 | groups: 'local-tendermint'
35 | - targets: ["5.9.57.44:26660", "5.9.57.45:26660", "5.9.57.89:26660"]
36 | labels:
37 | groups: 'tendermint'
38 |
--------------------------------------------------------------------------------
/sequencer/src/abci/main.rs:
--------------------------------------------------------------------------------
1 | use application::StarknetApp;
2 | use clap::Parser;
3 | use tendermint_abci::ServerBuilder;
4 | use tracing_subscriber::{filter::LevelFilter, util::SubscriberInitExt};
5 |
6 | mod application;
7 |
8 | #[derive(Debug, Parser)]
9 | #[clap(author, version, about)]
10 | struct Cli {
11 | /// Bind the TCP server to this host.
12 | #[clap(long, default_value = "127.0.0.1")]
13 | host: String,
14 |
15 | /// Bind the TCP server to this port.
16 | #[clap(short, long, default_value = "26658")]
17 | port: u16,
18 |
19 | /// The default server read buffer size, in bytes, for each incoming client
20 | /// connection.
21 | #[clap(short, long, default_value = "1048576")]
22 | read_buf_size: usize,
23 |
24 | /// Increase output logging verbosity to DEBUG level.
25 | #[clap(short, long)]
26 | verbose: bool,
27 |
28 | /// Suppress all output logging (overrides --verbose).
29 | #[clap(short, long)]
30 | quiet: bool,
31 | }
32 |
33 | fn main() {
34 | let cli: Cli = Cli::parse();
35 | let log_level = if cli.quiet {
36 | LevelFilter::OFF
37 | } else if cli.verbose {
38 | LevelFilter::DEBUG
39 | } else {
40 | LevelFilter::INFO
41 | };
42 |
43 | let subscriber = tracing_subscriber::fmt()
44 | // Use a more compact, abbreviated log format
45 | .compact()
46 | .with_max_level(log_level)
47 | // Display the thread ID an event was recorded on
48 | .with_thread_ids(true)
49 | // Don't display the event's target (module path)
50 | .with_target(false)
51 | // Build the subscriber
52 | .finish();
53 |
54 | subscriber.init();
55 |
56 | let app = StarknetApp::new();
57 | let server = ServerBuilder::new(cli.read_buf_size)
58 | .bind(format!("{}:{}", cli.host, cli.port), app)
59 | .unwrap();
60 |
61 | server.listen().unwrap();
62 | }
63 |
--------------------------------------------------------------------------------
/sequencer/src/bench/main.rs:
--------------------------------------------------------------------------------
1 | use clap::Parser;
2 | use lib::{Transaction, TransactionType};
3 | use std::net::SocketAddr;
4 | use std::time::Instant;
5 | use tendermint_rpc::{Client, HttpClient};
6 | use tracing::{info, metadata::LevelFilter};
7 | use tracing_subscriber::util::SubscriberInitExt;
8 | use uuid::Uuid;
9 |
10 | #[derive(Parser)]
11 | #[clap()]
12 | pub struct Cli {
13 | /// Amount of concurrent threads from which transactions will be sent. Each thread will have a client connection to the network.
14 | #[clap(short, long, value_parser, value_name = "UINT", default_value_t = 4)]
15 | pub threads: i32,
16 |
17 | /// Number of transactions per second each thread sends out.
18 | #[clap(short, long, value_parser, value_name = "UINT", default_value_t = 1000)]
19 | pub transactions_per_thread: usize,
20 |
21 | /// Nodes to which transactions will be sent to (round-robin).
22 | #[clap(
23 | long,
24 | value_parser,
25 | value_name = "ADDR",
26 | use_value_delimiter = true,
27 | value_delimiter = ' '
28 | )]
29 | nodes: Vec,
30 | }
31 |
32 | #[tokio::main()]
33 | #[allow(unreachable_code)]
34 | async fn main() {
35 | todo!("New execute method needs to be implemented on bench/main.rs");
36 |
37 | let cli = Cli::parse();
38 |
39 | tracing_subscriber::fmt()
40 | // Use a more compact, abbreviated log format
41 | .compact()
42 | // Display the thread ID an event was recorded on
43 | .with_thread_ids(true)
44 | .with_max_level(LevelFilter::INFO)
45 | // Build and init the subscriber
46 | .finish()
47 | .init();
48 |
49 | // prepare transactions
50 | let program = include_str!("../../programs/fibonacci.json").to_string();
51 |
52 | let transaction_type = TransactionType::Declare { program };
53 |
54 | let transaction = Transaction::with_type(transaction_type).unwrap();
55 | info!(
56 | "Single benchmark transaction size: {} bytes",
57 | bincode::serialize(&transaction).unwrap().len()
58 | );
59 |
60 | let mut handles = vec![];
61 |
62 | let time = Instant::now();
63 |
64 | // prepare a pool of transactions for each thread in order to have them sent out as soon as possible
65 | for _i in 0..cli.threads {
66 | let mut transactions = Vec::with_capacity(cli.transactions_per_thread);
67 |
68 | for _i in 0..cli.transactions_per_thread {
69 | let t = transaction.clone();
70 | // in order to not have Tendermint see the transactions as duplicate and discard them,
71 | // clone the transactions with a different ID
72 | let t = Transaction {
73 | id: Uuid::new_v4().to_string(),
74 | transaction_hash: t.transaction_hash,
75 | transaction_type: t.transaction_type,
76 | };
77 |
78 | transactions.push(bincode::serialize(&t).unwrap());
79 | }
80 | let nodes = cli.nodes.clone();
81 |
82 | handles.push(tokio::spawn(async move {
83 | run(transactions.clone(), &nodes).await;
84 | }));
85 | }
86 |
87 | futures::future::join_all(handles).await;
88 | info!(
89 | "Time it took for all transactions to be delivered: {} ms",
90 | time.elapsed().as_millis()
91 | );
92 | }
93 |
94 | async fn run(transactions: Vec>, nodes: &Vec) {
95 | let time = Instant::now();
96 | let mut clients = vec![];
97 | for i in 0..nodes.len() {
98 | let url = format!("http://{}", &nodes.get(i).unwrap());
99 | clients.push(HttpClient::new(url.as_str()).unwrap());
100 | }
101 |
102 | let n_transactions = transactions.len();
103 | // for each transaction in this thread, send transactions in a round robin fashion to each node
104 | for (i, t) in transactions.into_iter().enumerate() {
105 | let c = clients.get(i % clients.len()); // get destination node
106 | let response = c.unwrap().broadcast_tx_async(t).await;
107 |
108 | match &response {
109 | Ok(_) => {}
110 | Err(v) => info!("failure: {}", v),
111 | }
112 |
113 | let response = response.unwrap();
114 | match response.code {
115 | tendermint::abci::Code::Ok => {}
116 | tendermint::abci::Code::Err(code) => {
117 | info!("Error executing transaction {}: {}", code, response.log);
118 | }
119 | }
120 | }
121 | info!(
122 | "transactions sent: {} in {} ms",
123 | n_transactions,
124 | time.elapsed().as_millis()
125 | );
126 | }
127 |
--------------------------------------------------------------------------------
/sequencer/src/cli/main.rs:
--------------------------------------------------------------------------------
1 | use crate::tendermint::broadcast;
2 | use anyhow::{bail, Result};
3 | use clap::{Args, Parser, Subcommand};
4 | use lib::{Transaction, TransactionType};
5 | use serde_json::json;
6 | use std::fs;
7 | use std::path::PathBuf;
8 | use std::str;
9 | use tracing_subscriber::util::SubscriberInitExt;
10 | use tracing_subscriber::EnvFilter;
11 |
12 | pub mod tendermint;
13 | const LOCAL_SEQUENCER_URL: &str = "http://127.0.0.1:26657";
14 |
15 | #[derive(Parser)]
16 | struct Cli {
17 | /// Subcommand to execute
18 | #[command(subcommand)]
19 | command: Command,
20 |
21 | /// Output log lines to stdout based on the desired log level (RUST_LOG env var).
22 | #[clap(short, long, global = false, default_value_t = false)]
23 | pub verbose: bool,
24 |
25 | /// Just run the program and return the transaction in the stdio
26 | #[clap(short, long, global = false, default_value_t = false)]
27 | pub no_broadcast: bool,
28 |
29 | /// Tendermint node url
30 | #[clap(short, long, env = "SEQUENCER_URL", default_value = LOCAL_SEQUENCER_URL)]
31 | pub url: String,
32 | }
33 |
34 | #[derive(Subcommand)]
35 | enum Command {
36 | Declare(DeclareArgs),
37 | DeployAccount(DeployArgs),
38 | Invoke(InvokeArgs),
39 | Get(GetArgs),
40 | }
41 |
42 | #[derive(Args)]
43 | pub struct GetArgs {
44 | transaction_id: String,
45 | }
46 |
47 | #[derive(Args)]
48 | pub struct DeclareArgs {
49 | #[arg(long)]
50 | contract: PathBuf,
51 | }
52 |
53 | #[derive(Args)]
54 | pub struct DeployArgs {
55 | class_hash: String,
56 | // TODO: randomize salt by default?
57 | #[arg(long, default_value = "1111")]
58 | salt: i32,
59 | #[arg(long, num_args=1.., value_delimiter = ' ')]
60 | inputs: Option>,
61 | }
62 |
63 | #[derive(Args, Debug)]
64 | pub struct InvokeArgs {
65 | /// Contract Address
66 | #[clap(short, long)]
67 | address: String,
68 |
69 | /// Function name
70 | #[clap(short, long)]
71 | function: String,
72 |
73 | /// Function input values
74 | #[clap(long, num_args=1.., value_delimiter = ' ')]
75 | inputs: Option>,
76 |
77 | /// tendermint node url
78 | #[clap(short, long, env = "SEQUENCER_URL", default_value = LOCAL_SEQUENCER_URL)]
79 | pub url: String,
80 | }
81 |
82 | #[tokio::main()]
83 | async fn main() {
84 | let cli = Cli::parse();
85 |
86 | if cli.verbose {
87 | tracing_subscriber::fmt()
88 | // Use a more compact, abbreviated log format
89 | .compact()
90 | .with_env_filter(EnvFilter::from_default_env())
91 | // Build and init the subscriber
92 | .finish()
93 | .init();
94 | }
95 |
96 | let result = match cli.command {
97 | Command::Declare(declare_args) => do_declare(declare_args, &cli.url).await,
98 | Command::DeployAccount(deploy_args) => do_deploy(deploy_args, &cli.url).await,
99 | Command::Invoke(invoke_args) => do_invoke(invoke_args, &cli.url).await,
100 | Command::Get(get_args) => {
101 | tendermint::get_transaction(&get_args.transaction_id, &cli.url).await
102 | }
103 | };
104 |
105 | let (code, output) = match result {
106 | Ok(output) => (0, json!({"id": output.id,"hash": output.transaction_hash})),
107 | Err(err) => (1, json!({"error": err.to_string()})),
108 | };
109 |
110 | println!("\n{output:#}");
111 | std::process::exit(code);
112 | }
113 |
114 | async fn do_declare(args: DeclareArgs, url: &str) -> Result {
115 | let program = fs::read_to_string(args.contract)?;
116 | let transaction_type = TransactionType::Declare { program };
117 | let transaction = Transaction::with_type(transaction_type)?;
118 | let transaction_serialized = bincode::serialize(&transaction)?;
119 |
120 | match tendermint::broadcast(transaction_serialized, url).await {
121 | Ok(_) => Ok(transaction),
122 | Err(e) => bail!("DECLARE: Error ocurred when sending out transaction: {e}"),
123 | }
124 | }
125 |
126 | async fn do_deploy(args: DeployArgs, url: &str) -> Result {
127 | let transaction_type = TransactionType::DeployAccount {
128 | class_hash: args.class_hash,
129 | salt: args.salt,
130 | inputs: args.inputs,
131 | };
132 |
133 | let transaction = Transaction::with_type(transaction_type)?;
134 | let transaction_serialized = bincode::serialize(&transaction)?;
135 |
136 | match tendermint::broadcast(transaction_serialized, url).await {
137 | Ok(_) => Ok(transaction),
138 | Err(e) => bail!("DEPLOY: Error sending out transaction: {e}"),
139 | }
140 | }
141 |
142 | async fn do_invoke(args: InvokeArgs, url: &str) -> Result {
143 | let transaction_type = TransactionType::Invoke {
144 | address: args.address,
145 | function: args.function,
146 | inputs: args.inputs,
147 | };
148 |
149 | let transaction = Transaction::with_type(transaction_type)?;
150 | let transaction_serialized = bincode::serialize(&transaction)?;
151 |
152 | match broadcast(transaction_serialized, url).await {
153 | Ok(_) => Ok(transaction),
154 | Err(e) => bail!("INVOKE: Error sending out transaction: {e}"),
155 | }
156 | }
157 |
--------------------------------------------------------------------------------
/sequencer/src/cli/tendermint.rs:
--------------------------------------------------------------------------------
1 | use anyhow::{bail, ensure, Result};
2 | use lib::Transaction;
3 | use tendermint_rpc::{query::Query, Client, HttpClient, Order};
4 | use tracing::debug;
5 |
6 | pub async fn broadcast(transaction: Vec, url: &str) -> Result<()> {
7 | let client = HttpClient::new(url).unwrap();
8 |
9 | let response = client.broadcast_tx_sync(transaction).await?;
10 |
11 | debug!("Response from CheckTx: {:?}", response);
12 | match response.code {
13 | tendermint::abci::Code::Ok => Ok(()),
14 | tendermint::abci::Code::Err(code) => {
15 | bail!("Error executing transaction {}: {}", code, response.log)
16 | }
17 | }
18 | }
19 |
20 | pub async fn get_transaction(tx_id: &str, url: &str) -> Result {
21 | let client = HttpClient::new(url)?;
22 | // todo: this index key might have to be a part of the shared lib so that both the CLI and the ABCI can be in sync
23 | let query = Query::contains("app.tx_id", tx_id);
24 |
25 | let response = client
26 | .tx_search(query, false, 1, 1, Order::Ascending)
27 | .await?;
28 |
29 | // early return with error if no transaction has been indexed for that tx id
30 | ensure!(
31 | response.total_count > 0,
32 | "Transaction ID {} is invalid or has not yet been committed to the blockchain",
33 | tx_id
34 | );
35 |
36 | let tx_bytes = response.txs.into_iter().next().unwrap().tx;
37 | let transaction: Transaction = bincode::deserialize(&tx_bytes)?;
38 |
39 | Ok(transaction)
40 | }
41 |
--------------------------------------------------------------------------------
/sequencer/src/lib/mod.rs:
--------------------------------------------------------------------------------
1 | use anyhow::{ensure, Result};
2 | use felt::Felt252;
3 | use num_traits::{Num, Zero};
4 | use serde::{Deserialize, Serialize};
5 | use starknet_rs::{
6 | hash_utils::calculate_contract_address, services::api::contract_class::ContractClass,
7 | utils::Address,
8 | };
9 | use uuid::Uuid;
10 |
11 | #[derive(Clone, Serialize, Deserialize, Debug)]
12 | pub struct Transaction {
13 | pub transaction_type: TransactionType,
14 | pub transaction_hash: String,
15 | pub id: String,
16 | }
17 |
18 | #[derive(Clone, Serialize, Deserialize, Debug)]
19 | pub enum TransactionType {
20 | /// Create new contract class.
21 | Declare { program: String },
22 |
23 | /// Create an instance of a contract which will have storage assigned. (Accounts are a contract themselves)
24 | DeployAccount {
25 | class_hash: String,
26 | salt: i32,
27 | inputs: Option>,
28 | },
29 |
30 | /// Execute a function from a deployed contract.
31 | Invoke {
32 | address: String,
33 | function: String,
34 | inputs: Option>,
35 | },
36 | }
37 |
38 | impl Transaction {
39 | pub fn with_type(transaction_type: TransactionType) -> Result {
40 | Ok(Transaction {
41 | transaction_hash: transaction_type.compute_and_hash()?,
42 | transaction_type,
43 | id: Uuid::new_v4().to_string(),
44 | })
45 | }
46 |
47 | /// Verify that the transaction id is consistent with its contents, by checking its sha256 hash.
48 | pub fn assert_integrity(&self) -> Result<()> {
49 | ensure!(
50 | self.transaction_hash == self.transaction_type.compute_and_hash()?,
51 | "Corrupted transaction: Inconsistent transaction id"
52 | );
53 |
54 | Ok(())
55 | }
56 | }
57 |
58 | impl TransactionType {
59 | // TODO: Rename this and/or structure the code differently
60 | pub fn compute_and_hash(&self) -> Result {
61 | match self {
62 | TransactionType::Declare { program } => {
63 | let contract_class = ContractClass::try_from(program.as_str())?;
64 | // This function requires cairo_programs/contracts.json to exist as it uses that cairo program to compute the hash
65 | let contract_hash = starknet_rs::core::contract_address::starknet_contract_address::compute_class_hash(&contract_class)?;
66 | Ok(format!(
67 | "{}{}",
68 | "0x",
69 | hex::encode(contract_hash.to_bytes_be())
70 | ))
71 | }
72 | TransactionType::DeployAccount {
73 | class_hash,
74 | salt,
75 | inputs,
76 | } => {
77 | let constructor_calldata = match &inputs {
78 | Some(vec) => vec.iter().map(|&n| n.into()).collect(),
79 | None => Vec::new(),
80 | };
81 |
82 | let contract_address = calculate_contract_address(
83 | &Address((*salt).into()),
84 | &felt::Felt252::from_str_radix(&class_hash[2..], 16).unwrap(), // TODO: Handle these errors better
85 | &constructor_calldata,
86 | Address(Felt252::zero()), // TODO: Deployer address is hardcoded to 0 in starknet-in-rust, ask why
87 | )?;
88 |
89 | Ok(format!(
90 | "{}{}",
91 | "0x",
92 | hex::encode(contract_address.to_bytes_be())
93 | ))
94 | }
95 | TransactionType::Invoke {
96 | address,
97 | function,
98 | inputs,
99 | } => Ok(format!(
100 | "Invoked {function} with inputs {inputs:?} for contract in address {address}"
101 | )),
102 | }
103 | }
104 | }
105 |
--------------------------------------------------------------------------------
/sequencer/tests/client.rs:
--------------------------------------------------------------------------------
1 | use assert_cmd::{assert::Assert, Command};
2 | use serde::de::DeserializeOwned;
3 |
4 | #[test]
5 | fn deploy_fibonacci() {
6 | let output = client_command(&["declare", "cairo_programs/fibonacci.json"]);
7 | output.unwrap(); // if exit code was not successful, this will panic
8 |
9 | let output = client_command(&["deploy-account", "class_hash"]);
10 | output.unwrap();
11 |
12 | // todo: attempt invoke
13 | // todo: decide whether:
14 | // - CLI returns class_hash on check_tx,
15 | // - deliver_tx indexes class hash for a specific tx and creates an event for it,
16 | // - deliver_tx indexes class hash for a specific tx and we are able to use the `query` hook,
17 | //
18 | }
19 |
20 | fn client_command(args: &[&str]) -> Result {
21 | let command = &mut Command::cargo_bin("cli").unwrap();
22 |
23 | command
24 | .args(args)
25 | .assert()
26 | .try_success()
27 | .map(parse_output)
28 | .map_err(|e| e.to_string())
29 | }
30 |
31 | /// Extract the command assert output and deserialize it as json
32 | fn parse_output(result: Assert) -> T {
33 | let output = String::from_utf8(result.get_output().stdout.to_vec()).unwrap();
34 | serde_json::from_str(&output).unwrap()
35 | }
36 |
--------------------------------------------------------------------------------