├── .cargo ├── audit.toml └── config.toml ├── .chglog ├── CHANGELOG.tpl.md └── config.yml ├── .config ├── hakari.toml └── nextest.toml ├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── BASIC_ISSUE_FORM.yml │ └── EXTERNAL_ISSUE_FORM.yml ├── dependabot.yml └── workflows │ ├── assign-reviewers.yml │ ├── audit.yml │ ├── build-and-test.yml │ ├── build-without-lockfile.yml │ ├── build_nix.yml │ ├── careful.yml │ ├── coverage.yml │ ├── doc.yml │ ├── lint.yml │ ├── semver-check.yml │ ├── test-sequencer.yml │ ├── typos.yml │ └── update_nix.yml ├── .gitignore ├── .typos.toml ├── .vscode ├── launch.json └── tasks.json ├── CHANGELOG.md ├── CODEOWNERS ├── CODESTYLE.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── WORKFLOW.md ├── audits ├── README.md └── internal-reviews │ └── EspressoHotshot-2024internal.pdf ├── clippy.toml ├── config ├── ValidatorConfigExample └── ValidatorConfigFile.toml ├── crates ├── builder-api │ ├── Cargo.toml │ ├── README.md │ ├── api │ │ ├── v0_1 │ │ │ ├── builder.toml │ │ │ └── submit.toml │ │ └── v0_3 │ │ │ ├── builder.toml │ │ │ └── submit.toml │ └── src │ │ ├── api.rs │ │ ├── lib.rs │ │ ├── v0_1 │ │ ├── block_info.rs │ │ ├── builder.rs │ │ ├── data_source.rs │ │ ├── mod.rs │ │ └── query_data.rs │ │ └── v0_99 │ │ ├── builder.rs │ │ ├── data_source.rs │ │ └── mod.rs ├── example-types │ ├── Cargo.toml │ └── src │ │ ├── auction_results_provider_types.rs │ │ ├── block_types.rs │ │ ├── lib.rs │ │ ├── node_types.rs │ │ ├── state_types.rs │ │ ├── storage_types.rs │ │ └── testable_delay.rs ├── examples │ ├── Cargo.toml │ ├── combined │ │ ├── all.rs │ │ ├── multi-validator.rs │ │ ├── orchestrator.rs │ │ ├── types.rs │ │ └── validator.rs │ ├── infra │ │ └── mod.rs │ ├── libp2p │ │ ├── all.rs │ │ ├── multi-validator.rs │ │ ├── types.rs │ │ └── validator.rs │ ├── orchestrator.rs │ └── push-cdn │ │ ├── README.md │ │ ├── all.rs │ │ ├── broker.rs │ │ ├── marshal.rs │ │ ├── multi-validator.rs │ │ ├── types.rs │ │ ├── validator.rs │ │ └── whitelist-adapter.rs ├── fakeapi │ ├── Cargo.toml │ ├── apis │ │ └── solver.toml │ └── src │ │ ├── fake_solver.rs │ │ └── lib.rs ├── hotshot-stake-table │ ├── Cargo.toml │ └── src │ │ ├── config.rs │ │ ├── lib.rs │ │ ├── mt_based.rs │ │ ├── mt_based │ │ ├── config.rs │ │ └── internal.rs │ │ ├── utils.rs │ │ ├── vec_based.rs │ │ └── vec_based │ │ └── config.rs ├── hotshot │ ├── Cargo.toml │ └── src │ │ ├── documentation.rs │ │ ├── helpers.rs │ │ ├── lib.rs │ │ ├── tasks │ │ ├── mod.rs │ │ └── task_state.rs │ │ ├── traits.rs │ │ ├── traits │ │ ├── election │ │ │ ├── helpers.rs │ │ │ ├── mod.rs │ │ │ ├── randomized_committee.rs │ │ │ ├── randomized_committee_members.rs │ │ │ ├── static_committee.rs │ │ │ ├── static_committee_leader_two_views.rs │ │ │ └── two_static_committees.rs │ │ ├── networking.rs │ │ ├── networking │ │ │ ├── combined_network.rs │ │ │ ├── libp2p_network.rs │ │ │ ├── memory_network.rs │ │ │ └── push_cdn_network.rs │ │ └── node_implementation.rs │ │ ├── types.rs │ │ └── types │ │ ├── event.rs │ │ └── handle.rs ├── libp2p-networking │ ├── .cargo │ │ └── config │ ├── .gitignore │ ├── Cargo.toml │ ├── flamegraph.sh │ ├── src │ │ ├── lib.rs │ │ └── network │ │ │ ├── behaviours │ │ │ ├── dht │ │ │ │ ├── bootstrap.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── record.rs │ │ │ │ └── store │ │ │ │ │ ├── mod.rs │ │ │ │ │ ├── persistent.rs │ │ │ │ │ └── validated.rs │ │ │ ├── direct_message.rs │ │ │ ├── exponential_backoff.rs │ │ │ └── mod.rs │ │ │ ├── cbor.rs │ │ │ ├── def.rs │ │ │ ├── mod.rs │ │ │ ├── node.rs │ │ │ ├── node │ │ │ ├── config.rs │ │ │ └── handle.rs │ │ │ └── transport.rs │ └── web │ │ └── index.html ├── macros │ ├── Cargo.toml │ └── src │ │ └── lib.rs ├── orchestrator │ ├── Cargo.toml │ ├── README.md │ ├── api.toml │ ├── run-config.toml │ ├── src │ │ ├── client.rs │ │ └── lib.rs │ └── staging-config.toml ├── request-response │ ├── Cargo.toml │ └── src │ │ ├── data_source.rs │ │ ├── lib.rs │ │ ├── message.rs │ │ ├── network.rs │ │ ├── recipient_source.rs │ │ ├── request.rs │ │ └── util.rs ├── task-impls │ ├── Cargo.toml │ ├── HotShot_event_architecture.drawio │ ├── HotShot_event_architecture.png │ ├── README.md │ └── src │ │ ├── builder.rs │ │ ├── consensus │ │ ├── handlers.rs │ │ └── mod.rs │ │ ├── da.rs │ │ ├── events.rs │ │ ├── harness.rs │ │ ├── helpers.rs │ │ ├── lib.rs │ │ ├── network.rs │ │ ├── quorum_proposal │ │ ├── handlers.rs │ │ └── mod.rs │ │ ├── quorum_proposal_recv │ │ ├── handlers.rs │ │ └── mod.rs │ │ ├── quorum_vote │ │ ├── handlers.rs │ │ └── mod.rs │ │ ├── request.rs │ │ ├── response.rs │ │ ├── rewind.rs │ │ ├── transactions.rs │ │ ├── upgrade.rs │ │ ├── vid.rs │ │ ├── view_sync.rs │ │ └── vote_collection.rs ├── task │ ├── Cargo.toml │ └── src │ │ ├── dependency.rs │ │ ├── dependency_task.rs │ │ ├── lib.rs │ │ └── task.rs ├── testing │ ├── .gitignore │ ├── Cargo.toml │ ├── README.md │ ├── src │ │ ├── block_builder │ │ │ ├── mod.rs │ │ │ ├── random.rs │ │ │ └── simple.rs │ │ ├── byzantine │ │ │ ├── byzantine_behaviour.rs │ │ │ └── mod.rs │ │ ├── completion_task.rs │ │ ├── consistency_task.rs │ │ ├── helpers.rs │ │ ├── lib.rs │ │ ├── node_ctx.rs │ │ ├── overall_safety_task.rs │ │ ├── predicates │ │ │ ├── event.rs │ │ │ ├── mod.rs │ │ │ ├── upgrade_with_proposal.rs │ │ │ └── upgrade_with_vote.rs │ │ ├── script.rs │ │ ├── spinning_task.rs │ │ ├── test_builder.rs │ │ ├── test_helpers.rs │ │ ├── test_launcher.rs │ │ ├── test_runner.rs │ │ ├── test_task.rs │ │ ├── txn_task.rs │ │ ├── view_generator.rs │ │ └── view_sync_task.rs │ └── tests │ │ ├── tests_1.rs │ │ ├── tests_1 │ │ ├── block_builder.rs │ │ ├── da_task.rs │ │ ├── gen_key_pair.rs │ │ ├── libp2p.rs │ │ ├── message.rs │ │ ├── network_task.rs │ │ ├── quorum_proposal_recv_task.rs │ │ ├── quorum_proposal_task.rs │ │ ├── quorum_vote_task.rs │ │ ├── test_success.rs │ │ ├── test_with_failures_2.rs │ │ ├── transaction_task.rs │ │ ├── upgrade_task_with_proposal.rs │ │ ├── upgrade_task_with_vote.rs │ │ ├── vid_task.rs │ │ ├── view_sync_task.rs │ │ └── vote_dependency_handle.rs │ │ ├── tests_2.rs │ │ ├── tests_2 │ │ └── catchup.rs │ │ ├── tests_3.rs │ │ ├── tests_3 │ │ ├── byzantine_tests.rs │ │ ├── memory_network.rs │ │ └── test_with_failures_half_f.rs │ │ ├── tests_4.rs │ │ ├── tests_4 │ │ ├── byzantine_tests.rs │ │ ├── test_marketplace.rs │ │ ├── test_with_builder_failures.rs │ │ └── test_with_failures_f.rs │ │ ├── tests_5.rs │ │ ├── tests_5 │ │ ├── broken_3_chain.rs │ │ ├── combined_network.rs │ │ ├── fake_solver.rs │ │ ├── push_cdn.rs │ │ ├── test_with_failures.rs │ │ ├── timeout.rs │ │ └── unreliable_network.rs │ │ ├── tests_6.rs │ │ └── tests_6 │ │ └── test_epochs.rs ├── types │ ├── Cargo.toml │ └── src │ │ ├── bundle.rs │ │ ├── consensus.rs │ │ ├── constants.rs │ │ ├── data.rs │ │ ├── data │ │ └── vid_disperse.rs │ │ ├── drb.rs │ │ ├── error.rs │ │ ├── event.rs │ │ ├── hotshot_config_file.rs │ │ ├── lib.rs │ │ ├── light_client.rs │ │ ├── message.rs │ │ ├── network.rs │ │ ├── qc.rs │ │ ├── request_response.rs │ │ ├── signature_key.rs │ │ ├── simple_certificate.rs │ │ ├── simple_vote.rs │ │ ├── stake_table.rs │ │ ├── traits.rs │ │ ├── traits │ │ ├── auction_results_provider.rs │ │ ├── block_contents.rs │ │ ├── consensus_api.rs │ │ ├── election.rs │ │ ├── metrics.rs │ │ ├── network.rs │ │ ├── node_implementation.rs │ │ ├── qc.rs │ │ ├── signature_key.rs │ │ ├── stake_table.rs │ │ ├── states.rs │ │ └── storage.rs │ │ ├── upgrade_config.rs │ │ ├── utils.rs │ │ ├── validator_config.rs │ │ ├── vid.rs │ │ ├── vid │ │ ├── advz.rs │ │ └── avidm.rs │ │ └── vote.rs ├── utils │ ├── Cargo.toml │ └── src │ │ ├── anytrace.rs │ │ ├── anytrace │ │ └── macros.rs │ │ └── lib.rs └── workspace-hack │ ├── .gitattributes │ ├── Cargo.toml │ ├── build.rs │ └── src │ └── lib.rs ├── default.nix ├── docker ├── cdn-broker.Dockerfile ├── cdn-marshal.Dockerfile ├── orchestrator.Dockerfile ├── validator-cdn-local.Dockerfile ├── validator-cdn.Dockerfile ├── validator-combined.Dockerfile └── validator-libp2p.Dockerfile ├── docs ├── README.md ├── diagrams │ ├── HotShotFlow.drawio │ ├── README.md │ ├── event_discriptions │ │ ├── BlockFromBuilderRecv.md │ │ ├── General.md │ │ ├── OptimisticDACertificateRecv.md │ │ ├── OptimisticDAProposalRecv.md │ │ ├── QuorumProposalRecv.md │ │ ├── Timeout.md │ │ ├── VIDDataFromBuilderRecv.md │ │ ├── VIDShareRecv.md │ │ ├── ViewChange.md │ │ ├── ViewSyncCommitCertificateRecv.md │ │ ├── ViewSyncFinalizeCertificateRecv.md │ │ ├── ViewSyncPreCommitCertificateRecv.md │ │ ├── ViewSyncTimeout.md │ │ ├── ViewSyncTrigger.md │ │ └── VoteOnQuorumProposal.md │ └── images │ │ ├── HotShotFlow-BlockPayloadFromBuilderRecv.drawio.png │ │ ├── HotShotFlow-OptimisticDACertificateRecv.drawio.png │ │ ├── HotShotFlow-OptimisticDAProposalRecv.drawio.png │ │ ├── HotShotFlow-QuorumProposalRecv.drawio.png │ │ ├── HotShotFlow-QuorumProposalSend.drawio.png │ │ ├── HotShotFlow-Timeout.drawio.png │ │ ├── HotShotFlow-VIDDataFromBuilderRecv.drawio.png │ │ ├── HotShotFlow-VIDShareRecv.drawio.png │ │ ├── HotShotFlow-ViewChange.drawio.png │ │ ├── HotShotFlow-ViewSyncCommitCertificateRecv.drawio.png │ │ ├── HotShotFlow-ViewSyncFinalizeCertificateRecv.drawio.png │ │ ├── HotShotFlow-ViewSyncPreCommitCertificateRecv.drawio.png │ │ ├── HotShotFlow-ViewSyncTimeout.drawio.png │ │ ├── HotShotFlow-ViewSyncTrigger.drawio.png │ │ ├── HotShotFlow-VoteOnQuorumProposal.drawio.png │ │ └── HotShotFlow-VoteRecv.drawio.png └── espresso-sequencer-paper.pdf ├── flake.lock ├── flake.nix ├── justfile ├── pull_request_template.md ├── repl.nix ├── rust-toolchain.toml ├── rustfmt.toml ├── scripts ├── .gitignore ├── auto-integration │ ├── .gitignore │ ├── README.md │ ├── requirements.txt │ └── run-integration.py ├── benchmark_scripts │ ├── aws_ecs_benchmarks_cdn.sh │ ├── aws_ecs_benchmarks_cdn_gpu.sh │ ├── benchmarks_start_cdn_broker.sh │ └── benchmarks_start_leader_gpu.sh ├── benchmarks_results │ ├── README.md │ └── results_upload.csv ├── count_fds.sh ├── flakiness.sh ├── nix_bump_pr_changes.py └── runfail.sh └── shell.nix /.cargo/audit.toml: -------------------------------------------------------------------------------- 1 | [advisories] 2 | ignore = [ 3 | # Tungstenite allows remote attackers to cause a denial of service 4 | # Dependency of async-tungstenite -> tide-websockets / surf-disco 5 | "RUSTSEC-2023-0065", 6 | # DoS in WebPKI that comes with tide_disco 7 | "RUSTSEC-2023-0052", 8 | # DoS in WebPKI that comes with tide_disco 9 | "RUSTSEC-2023-0052", 10 | # rustls vulnerability that could cause an infinite loop based on network input, coming from tide_disco 11 | "RUSTSEC-2024-0336", 12 | # Unfixed "Marvin" vulnerability in `RSA`, unused in sqlite dependency 13 | "RUSTSEC-2023-0071", 14 | ] 15 | -------------------------------------------------------------------------------- /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [registries] 2 | translucence = { index = "https://dl.cloudsmith.io/basic/translucence/tl/cargo/index.git" } 3 | 4 | [net] 5 | git-fetch-with-cli = true 6 | -------------------------------------------------------------------------------- /.chglog/CHANGELOG.tpl.md: -------------------------------------------------------------------------------- 1 | {{ if .Versions -}} 2 | 3 | ## [Unreleased] 4 | 5 | {{ if .Unreleased.CommitGroups -}} 6 | {{ range .Unreleased.CommitGroups -}} 7 | ### {{ .Title }} 8 | {{ range .Commits -}} 9 | - {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} 10 | {{ end }} 11 | {{ end -}} 12 | {{ end -}} 13 | {{ end -}} 14 | 15 | {{ range .Versions }} 16 | 17 | ## {{ if .Tag.Previous }}[{{ .Tag.Name }}]{{ else }}{{ .Tag.Name }}{{ end }} - {{ datetime "2006-01-02" .Tag.Date }} 18 | {{ range .CommitGroups -}} 19 | ### {{ .Title }} 20 | {{ range .Commits -}} 21 | - {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} 22 | {{ end }} 23 | {{ end -}} 24 | 25 | {{- if .RevertCommits -}} 26 | ### Reverts 27 | {{ range .RevertCommits -}} 28 | - {{ .Revert.Header }} 29 | {{ end }} 30 | {{ end -}} 31 | 32 | {{- if .NoteGroups -}} 33 | {{ range .NoteGroups -}} 34 | ### {{ .Title }} 35 | {{ range .Notes }} 36 | {{ .Body }} 37 | {{ end }} 38 | {{ end -}} 39 | {{ end -}} 40 | {{ end -}} 41 | 42 | {{- if .Versions }} 43 | [Unreleased]: {{ .Info.RepositoryURL }}/compare/{{ $latest := index .Versions 0 }}{{ $latest.Tag.Name }}...HEAD 44 | {{ range .Versions -}} 45 | {{ if .Tag.Previous -}} 46 | [{{ .Tag.Name }}]: {{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }} 47 | {{ end -}} 48 | {{ end -}} 49 | {{ end -}} -------------------------------------------------------------------------------- /.chglog/config.yml: -------------------------------------------------------------------------------- 1 | style: gitlab 2 | template: CHANGELOG.tpl.md 3 | info: 4 | title: CHANGELOG 5 | repository_url: https://github.com/espressosystems/hotshot 6 | options: 7 | commits: 8 | filters: 9 | Type: 10 | - feat 11 | - fix 12 | - perf 13 | - refactor 14 | commit_groups: 15 | title_maps: 16 | feat: Features 17 | fix: Bug Fixes 18 | perf: Performance Improvements 19 | refactor: Code Refactoring 20 | header: 21 | pattern: "^(\\w*)!?\\:\\s(.*)$" 22 | pattern_maps: 23 | - Type 24 | - Subject 25 | notes: 26 | keywords: 27 | - BREAKING CHANGE 28 | -------------------------------------------------------------------------------- /.config/hakari.toml: -------------------------------------------------------------------------------- 1 | # This file contains settings for `cargo hakari`. 2 | # See https://docs.rs/cargo-hakari/latest/cargo_hakari/config for a full list of options. 3 | 4 | hakari-package = "workspace-hack" 5 | 6 | # Format version for hakari's output. Version 4 requires cargo-hakari 0.9.22 or above. 7 | dep-format-version = "4" 8 | 9 | # Setting workspace.resolver = "2" or higher in the root Cargo.toml is HIGHLY recommended. 10 | # Hakari works much better with the v2 resolver. (The v2 and v3 resolvers are identical from 11 | # hakari's perspective, so you're welcome to set either.) 12 | # 13 | # For more about the new feature resolver, see: 14 | # https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html#cargos-new-feature-resolver 15 | resolver = "2" 16 | 17 | # Add triples corresponding to platforms commonly used by developers here. 18 | # https://doc.rust-lang.org/rustc/platform-support.html 19 | platforms = [ 20 | # "x86_64-unknown-linux-gnu", 21 | # "x86_64-apple-darwin", 22 | # "aarch64-apple-darwin", 23 | # "x86_64-pc-windows-msvc", 24 | ] 25 | 26 | # Write out exact versions rather than a semver range. (Defaults to false.) 27 | # exact-versions = true 28 | 29 | [traversal-excludes] 30 | third-party = [ 31 | # `jf-vid` has features that need to be explicitly enabled via HotShot features -- 32 | # namely "gpu-vid" and "test-srs". Hence, we exclude it from `workspace-hack` so that 33 | # these features are not enabled for all crates by default. 34 | { name = "jf-vid", git = "https://github.com/EspressoSystems/jellyfish", tag = "0.4.5" }, 35 | # `sqlx` has internal packages that can have different features depending on the host/target platform. 36 | # This only gets pulled in indirectly via the CDN, so it's safe to just exclude. 37 | { name = "sqlx" } 38 | ] 39 | -------------------------------------------------------------------------------- /.config/nextest.toml: -------------------------------------------------------------------------------- 1 | [profile.ci] 2 | retries = 2 3 | slow-timeout = { period = "1m", terminate-after = 3 } 4 | final-status-level = "flaky" 5 | threads-required = "num-test-threads" 6 | 7 | [profile.local] 8 | retries = 0 9 | slow-timeout = { period = "1m", terminate-after = 3 } 10 | final-status-level = "slow" 11 | threads-required = "num-test-threads" 12 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | /target/tmp 3 | /target/release* 4 | /target/debug 5 | /target/x86_64-unknown-linux-musl/release*/build 6 | /target/x86_64-unknown-linux-musl/release*/deps 7 | /target/x86_64-unknown-linux-musl/release*/incremental 8 | !/target/x86_64-unknown-linux-musl/release*/examples/ 9 | !/target/release*/examples/ 10 | !/target/release-lto/examples/ 11 | !/target/release*/benchmark_client 12 | Dockerfile* 13 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/EXTERNAL_ISSUE_FORM.yml: -------------------------------------------------------------------------------- 1 | name: External - Basic Issue 2 | description: Create an external HotShot issue 3 | title: "[EXTERNAL] - " 4 | labels: ["external"] 5 | assignees: ["elliedavidson"] 6 | body: 7 | - type: markdown 8 | attributes: 9 | value: | 10 | *If you are an internal collaborator, use our internal issue form instead* 11 | - type: textarea 12 | id: task_reason 13 | attributes: 14 | label: What is this task and why do we need to work on it? 15 | placeholder: | 16 | If this is a feature request, describe what the feature is and why it is important to add to HotShot. 17 | If this is a bug report, describe the bug and its severity level. 18 | validations: 19 | required: true 20 | - type: textarea 21 | id: other_details 22 | attributes: 23 | label: Are there any other details to include? 24 | placeholder: | 25 | Include other details here such as: open questions, directions to reproduce a bug, relevant error logs, etc. 26 | 27 | E.g. To reproduce this bug run `just test_basic`. You should see logs similar to the ones below: 28 | `ERROR: This is an important error!` 29 | validations: 30 | required: false 31 | 32 | - type: markdown 33 | attributes: 34 | value: | 35 | ### Complete the following items before submitting this issue: 36 | * Label this issue appropriately. This form will automatically label issues with the `external` label. If you are submitting a bug report, please also add the `bug` label. 37 | * Ensure this issue is titled correctly. The title should be in the form of [EXTERNAL] - Short task description. 38 | 39 | Thank you for submitting an issue to HotShot! 40 | 41 | 42 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | 8 | - package-ecosystem: cargo 9 | directory: "/" 10 | schedule: 11 | interval: daily 12 | groups: 13 | all: 14 | patterns: 15 | - "*" 16 | exclude-patterns: 17 | - "cdn-*" 18 | - "ark-*" 19 | cdn: 20 | patterns: 21 | - "cdn-*" 22 | ark: 23 | patterns: 24 | - "ark-*" 25 | -------------------------------------------------------------------------------- /.github/workflows/assign-reviewers.yml: -------------------------------------------------------------------------------- 1 | name: Choose random assignees 2 | on: 3 | pull_request: 4 | types: [opened, ready_for_review, reopened] 5 | 6 | jobs: 7 | test: 8 | runs-on: ubuntu-latest 9 | if: github.event.pull_request.draft == false 10 | steps: 11 | - name: Choose random assignees 12 | run: | 13 | # Skip PRs that have assignees already 14 | gh pr view ${{ github.event.number }} --repo ${{ github.repository }} --json assignees \ 15 | | jq '.assignees | length > 0' -e && \ 16 | exit 0 17 | 18 | chosen=$(perl \ 19 | -e 'use List::Util qw(shuffle head);' \ 20 | -e 'my @assignees = grep { "${{ github.event.sender.login }}" ne $_ } shuffle split /\s+/, $ENV{ASSIGNEES};' \ 21 | -e 'print join ",", head $ENV{NUM_ASSIGNEES}, @assignees' \ 22 | ) 23 | 24 | gh pr edit ${{ github.event.number }} --add-assignee $chosen --repo ${{ github.repository }} 25 | env: 26 | GITHUB_TOKEN: ${{ github.token }} 27 | NUM_ASSIGNEES: 2 28 | ASSIGNEES: jparr721 rob-maron ss-es bfish713 QuentinI shenkeyao lukaszrzasik lukeiannucci 29 | -------------------------------------------------------------------------------- /.github/workflows/audit.yml: -------------------------------------------------------------------------------- 1 | name: Run Cargo Audit 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'develop' 7 | - 'main' 8 | pull_request: 9 | schedule: 10 | - cron: '0 0 * * 1' # Run on Mondays 11 | workflow_dispatch: 12 | 13 | concurrency: 14 | group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.run_number) || github.ref }} 15 | cancel-in-progress: true 16 | 17 | jobs: 18 | audit: 19 | runs-on: ubuntu-latest 20 | timeout-minutes: 120 21 | steps: 22 | - name: Checkout Repository 23 | uses: actions/checkout@v4 24 | 25 | - name: Install cargo audit 26 | run: cargo install cargo-audit 27 | 28 | - name: Audit Dependencies 29 | run: cargo audit 30 | -------------------------------------------------------------------------------- /.github/workflows/build-without-lockfile.yml: -------------------------------------------------------------------------------- 1 | name: Build without committed Cargo.lock 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - release-* 8 | tags: 9 | # YYYYMMDD 10 | - "20[0-9][0-9][0-1][0-9][0-3][0-9]*" 11 | schedule: 12 | - cron: "0 0 * * 1" 13 | pull_request: 14 | workflow_dispatch: 15 | 16 | concurrency: 17 | group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.run_number) || github.ref }} 18 | cancel-in-progress: true 19 | 20 | jobs: 21 | build-ignore-lockfile: 22 | runs-on: ubuntu-latest 23 | steps: 24 | - name: Checkout Repository 25 | uses: actions/checkout@v4 26 | with: 27 | submodules: recursive 28 | 29 | - name: Enable Rust Caching 30 | uses: Swatinem/rust-cache@v2 31 | with: 32 | prefix-key: v1-rust 33 | 34 | - name: Build without committed Cargo.lock 35 | run: | 36 | cargo generate-lockfile 37 | cargo check --all-targets 38 | -------------------------------------------------------------------------------- /.github/workflows/build_nix.yml: -------------------------------------------------------------------------------- 1 | name: Build with Nix Workflow 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'develop' 7 | - 'main' 8 | - 'nix*' 9 | schedule: 10 | - cron: '0 0 * * 1' 11 | workflow_dispatch: 12 | 13 | concurrency: 14 | group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.run_number) || github.ref }} 15 | cancel-in-progress: true 16 | 17 | jobs: 18 | nix-build: 19 | runs-on: ubuntu-latest 20 | timeout-minutes: 120 21 | steps: 22 | - name: Checkout Repository 23 | uses: actions/checkout@v4 24 | 25 | - name: Install Nix 26 | uses: cachix/install-nix-action@v30 27 | 28 | - name: Nix Caching 29 | uses: cachix/cachix-action@v15 30 | with: 31 | name: espresso-systems-private 32 | authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' 33 | skipPush: ${{ github.actor == 'dependabot[bot]' }} 34 | 35 | - name: Enable Rust Cache 36 | uses: Swatinem/rust-cache@v2 37 | with: 38 | shared-key: "nix-build" 39 | 40 | # sanity check that repository builds with nix 41 | - name: Initialize Nix Environment 42 | run: | 43 | nix develop -c echo Nix Setup Complete 44 | 45 | # sanity check that repository builds with nix 46 | - name: Build 47 | run: | 48 | nix develop -c just build 49 | -------------------------------------------------------------------------------- /.github/workflows/careful.yml: -------------------------------------------------------------------------------- 1 | name: Careful Workflow 2 | 3 | on: 4 | schedule: 5 | # run at midnight on monday 6 | - cron: '0 0 * * 1' 7 | workflow_dispatch: 8 | 9 | concurrency: 10 | group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.run_number) || github.ref }} 11 | cancel-in-progress: true 12 | 13 | jobs: 14 | careful: 15 | runs-on: ubuntu-latest 16 | timeout-minutes: 120 17 | steps: 18 | - name: Install Nix 19 | uses: cachix/install-nix-action@v30 20 | 21 | - name: Nix Caching 22 | uses: cachix/cachix-action@v15 23 | with: 24 | name: espresso-systems-private 25 | authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' 26 | skipPush: ${{ github.actor == 'dependabot[bot]' }} 27 | 28 | - name: Checkout Repository 29 | uses: actions/checkout@v4 30 | 31 | - name: Run careful tests 32 | run: | 33 | nix develop .#correctnessShell -c just tokio careful 34 | timeout-minutes: 90 35 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Code Coverage Workflow 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * 1' 6 | workflow_dispatch: 7 | 8 | concurrency: 9 | group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.run_number) || github.ref }} 10 | cancel-in-progress: true 11 | 12 | jobs: 13 | code-coverage: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Install Nix 17 | uses: cachix/install-nix-action@v30 18 | 19 | - name: Nix Caching 20 | uses: cachix/cachix-action@v15 21 | with: 22 | name: espresso-systems-private 23 | authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' 24 | skipPush: ${{ github.actor == 'dependabot[bot]' }} 25 | 26 | - name: Checkout Repository 27 | uses: actions/checkout@v4 28 | 29 | - name: Generate coverage reports 30 | # Use the `release` profile rather than `release-lto` as other workflows do, since `-- 31 | # profile=release-lto` will cause the `failed to generate report` error. 32 | run: | 33 | nix develop .#perfShell -c just tokio code_coverage 34 | timeout-minutes: 90 35 | 36 | - name: Coveralls upload 37 | uses: coverallsapp/github-action@master 38 | with: 39 | github-token: ${{ secrets.GITHUB_TOKEN }} 40 | path-to-lcov: lcov.info 41 | -------------------------------------------------------------------------------- /.github/workflows/doc.yml: -------------------------------------------------------------------------------- 1 | name: Documentation 2 | on: 3 | push: 4 | branches: 5 | - 'main' 6 | - 'develop' 7 | pull_request: 8 | schedule: 9 | - cron: '0 0 * * 1' 10 | workflow_dispatch: 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.run_number) || github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | build: 18 | runs-on: ubuntu-latest 19 | timeout-minutes: 35 20 | steps: 21 | - name: Checkout Repository 22 | uses: actions/checkout@v4 23 | 24 | - uses: taiki-e/install-action@just 25 | 26 | - uses: Swatinem/rust-cache@v2 27 | name: Enable Rust Caching 28 | with: 29 | shared-key: "build-and-test" 30 | save-if: false 31 | 32 | - name: Test Docs 33 | run: | 34 | just doc_test 35 | 36 | - name: Build Docs 37 | run: | 38 | just doc 39 | 40 | - name: Create documentation 41 | if: ${{ github.ref == 'refs/heads/main' }} 42 | run: | 43 | cp -R target/doc public 44 | echo '' > public/index.html 45 | 46 | - name: Deploy 47 | uses: peaceiris/actions-gh-pages@v4 48 | if: ${{ github.ref == 'refs/heads/main' }} 49 | with: 50 | github_token: ${{ secrets.GITHUB_TOKEN }} 51 | publish_dir: ./public 52 | cname: hotshot.docs.espressosys.com 53 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'develop' 7 | - 'main' 8 | pull_request: 9 | schedule: 10 | - cron: '0 0 * * 1' 11 | workflow_dispatch: 12 | 13 | concurrency: 14 | group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.run_number) || github.ref }} 15 | cancel-in-progress: true 16 | 17 | jobs: 18 | verify-workspace-hack: 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v4 22 | name: Checkout Repository 23 | 24 | - name: Install cargo-hakari from crates.io 25 | uses: baptiste0928/cargo-install@v3 26 | with: 27 | crate: cargo-hakari 28 | 29 | - name: Run cargo-hakari 30 | run: | 31 | cargo hakari generate --diff 32 | 33 | clippy: 34 | strategy: 35 | fail-fast: false 36 | runs-on: ubuntu-latest 37 | steps: 38 | - uses: actions/checkout@v4 39 | name: Checkout Repository 40 | 41 | - name: Install Rust 42 | uses: mkroening/rust-toolchain-toml@main 43 | 44 | - uses: Swatinem/rust-cache@v2 45 | name: Enable Rust Caching 46 | with: 47 | shared-key: "lint" 48 | save-if: ${{ github.ref == 'refs/heads/main' }} 49 | 50 | - uses: taiki-e/install-action@just 51 | 52 | - name: Run clippy 53 | run: | 54 | just clippy 55 | 56 | fmt: 57 | runs-on: ubuntu-latest 58 | steps: 59 | - uses: actions/checkout@v4 60 | name: Checkout Repository 61 | 62 | - name: Install Rust 63 | uses: mkroening/rust-toolchain-toml@main 64 | 65 | - uses: taiki-e/install-action@just 66 | 67 | - name: Check rustfmt 68 | run: | 69 | just fmt_check 70 | -------------------------------------------------------------------------------- /.github/workflows/semver-check.yml: -------------------------------------------------------------------------------- 1 | name: Run semver check 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | baseline: 7 | description: "Baseline git revision to check against" 8 | required: true 9 | type: string 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.run_number) || github.ref }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | test-sequencer: 17 | runs-on: ubuntu-latest 18 | name: semver 19 | 20 | steps: 21 | - uses: actions/checkout@v4 22 | name: Checkout Repository 23 | with: 24 | path: current 25 | 26 | - uses: actions/checkout@v4 27 | name: Checkout Baseline 28 | with: 29 | path: baseline 30 | ref: ${{ inputs.baseline }} 31 | 32 | - uses: taiki-e/install-action@just 33 | 34 | - name: Install Rust 35 | uses: mkroening/rust-toolchain-toml@main 36 | 37 | - uses: Swatinem/rust-cache@v2 38 | name: Enable Rust Caching 39 | with: 40 | shared-key: "build-and-test" 41 | save-if: false 42 | 43 | - name: Install cargo-semver-checks and cargo-workspaces 44 | run: | 45 | cargo install cargo-semver-checks --locked 46 | cargo install cargo-workspaces 47 | 48 | - name: Run cargo-semver-checks 49 | run: | 50 | cd current 51 | just semver --baseline-root ../baseline 52 | 53 | -------------------------------------------------------------------------------- /.github/workflows/test-sequencer.yml: -------------------------------------------------------------------------------- 1 | name: Test sequencer 2 | 3 | on: 4 | schedule: 5 | # Monday midnight 6 | - cron: '0 0 * * 1' 7 | workflow_dispatch: 8 | 9 | concurrency: 10 | group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.run_number) || github.ref }} 11 | cancel-in-progress: true 12 | 13 | jobs: 14 | test-sequencer: 15 | runs-on: ubuntu-latest 16 | name: Test sequencer 17 | steps: 18 | - uses: actions/checkout@v4 19 | name: Checkout Repository 20 | with: 21 | path: hotshot 22 | 23 | - uses: actions/checkout@v4 24 | name: Checkout Sequencer Repository 25 | with: 26 | repository: EspressoSystems/espresso-sequencer 27 | path: sequencer 28 | submodules: true 29 | 30 | - name: Install Rust 31 | uses: mkroening/rust-toolchain-toml@main 32 | 33 | - uses: Swatinem/rust-cache@v2 34 | name: Enable Rust Caching 35 | with: 36 | shared-key: "" 37 | prefix-key: sequencer 38 | 39 | - name: Install Foundry 40 | uses: foundry-rs/foundry-toolchain@v1 41 | # TODO: remove version pinning once sequencer repository does that 42 | with: 43 | version: "nightly-60ec00296f00754bc21ed68fd05ab6b54b50e024" 44 | 45 | - name: Patch sequencer dependencies 46 | run: | 47 | mkdir -p .cargo 48 | cat << EOF > .cargo/config.toml 49 | [patch.'https://github.com/EspressoSystems/hotshot'] 50 | hotshot = { path = "${GITHUB_WORKSPACE}/hotshot/crates/hotshot" } 51 | hotshot-constants = { path = "${GITHUB_WORKSPACE}/hotshot/crates/constants" } 52 | hotshot-qc = { path = "${GITHUB_WORKSPACE}/hotshot/crates/hotshot-qc" } 53 | hotshot-signature-key = { path = "${GITHUB_WORKSPACE}/hotshot/crates/hotshot-signature-key" } 54 | hotshot-stake-table = { path = "${GITHUB_WORKSPACE}/hotshot/crates/hotshot-stake-table" } 55 | hotshot-state-prover = { path = "${GITHUB_WORKSPACE}/hotshot/crates/hotshot-state-prover" } 56 | hotshot-orchestrator = { path = "${GITHUB_WORKSPACE}/hotshot/crates/orchestrator" } 57 | hotshot-web-server = { path = "${GITHUB_WORKSPACE}/hotshot/crates/web_server" } 58 | hotshot-task-impls = { path = "${GITHUB_WORKSPACE}/hotshot/crates/task-impls" } 59 | hotshot-testing = { path = "${GITHUB_WORKSPACE}/hotshot/crates/testing" } 60 | libp2p-networking = { path = "${GITHUB_WORKSPACE}/hotshot/crates/libp2p-networking" } 61 | EOF 62 | 63 | - name: Build sequencer tests 64 | working-directory: sequencer 65 | run: | 66 | cargo test --release --workspace --all-features --no-run 67 | 68 | - name: Run sequencer tests 69 | working-directory: sequencer 70 | run: | 71 | cargo test --release --workspace --all-features --verbose -- --test-threads 1 --nocapture 72 | -------------------------------------------------------------------------------- /.github/workflows/typos.yml: -------------------------------------------------------------------------------- 1 | name: Typos 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - release-* 8 | pull_request: 9 | workflow_dispatch: 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.ref }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | typos: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v4 20 | name: Checkout Repository 21 | 22 | - name: typos-action 23 | uses: crate-ci/typos@v1.28.4 24 | -------------------------------------------------------------------------------- /.github/workflows/update_nix.yml: -------------------------------------------------------------------------------- 1 | name: update-flake-lock 2 | on: 3 | workflow_dispatch: # allows manual triggering 4 | schedule: 5 | - cron: '0 0 * * 0' # runs weekly on Sunday at 00:00 6 | 7 | concurrency: 8 | group: ${{ github.workflow }}-${{ (github.ref == 'refs/heads/main' && github.run_number) || github.ref }} 9 | cancel-in-progress: true 10 | 11 | jobs: 12 | lockfile: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout repository 16 | uses: actions/checkout@v4 17 | 18 | - name: Install Nix 19 | uses: cachix/install-nix-action@v30 20 | with: 21 | extra_nix_config: | 22 | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} 23 | 24 | - name: Nix Caching 25 | uses: cachix/cachix-action@v15 26 | with: 27 | name: espresso-systems-private 28 | authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' 29 | skipPush: ${{ github.actor == 'dependabot[bot]' }} 30 | 31 | - name: Update flake.lock 32 | uses: DeterminateSystems/update-flake-lock@v24 33 | with: 34 | pr-title: "Weekly PR to bump flake.nix" # Title of PR to be created 35 | pr-labels: | # Labels to be set on the PR 36 | dependencies 37 | automated 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .envrc 2 | **/target 3 | **/result 4 | **/out*.txt 5 | **/out*.json 6 | **/.idea 7 | /*.pdf 8 | **/target_dirs 9 | /target_dirs 10 | /.vscode/settings.json 11 | **/.DS_Store 12 | *.cache 13 | **/.direnv 14 | **/*.bkp 15 | *.sqlite 16 | **/results.csv 17 | .github/workflows/preserve-build-and-test-self-hosted.yml 18 | .github/workflows/preserve-build-and-test.yml 19 | config/ValidatorConfigOutput 20 | scripts/preserve_ci_ecs_benchmarks.sh 21 | **/*.bin 22 | **/shutdown.sh 23 | *.log 24 | -------------------------------------------------------------------------------- /.typos.toml: -------------------------------------------------------------------------------- 1 | [files] 2 | extend-exclude = [ 3 | "*.drawio", 4 | "crates/orchestrator/run-config.toml" 5 | ] 6 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "type": "cargo", 6 | "command": "build", 7 | "args": [ 8 | "--features=full-ci" 9 | ], 10 | "problemMatcher": [ 11 | "$rustc" 12 | ], 13 | "group": { 14 | "kind": "build", 15 | "isDefault": true 16 | }, 17 | "label": "rust: cargo build" 18 | }, 19 | ] 20 | } -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | ## [0.0.4] - 2021-11-01 3 | ### Features 4 | - Downgrade possibly-temporary network faults to warnings 5 | - Improve logging when an invalid transaction is submitted 6 | 7 | 8 | 9 | ## [0.0.3] - 2021-10-27 10 | ### Features 11 | - Implement janky catchup 12 | 13 | ### BREAKING CHANGE 14 | 15 | Adds new type parameter, corresponding to the state type, to Message 16 | 17 | 18 | ## [0.0.2] - 2021-10-19 19 | ### Bug Fixes 20 | - Fix leaders not sending themselves commit votes 21 | - Fix state not getting stored properly 22 | 23 | ### Features 24 | - StatefulHandler trait 25 | - Reexport traits from traits module 26 | - State Machine + Node Implementation 27 | - state machine mvp megasquash 28 | - Replace tokio broadcast queue with unbounded equivalent 29 | 30 | ### BREAKING CHANGE 31 | 32 | Changes queue type in hotshot methods 33 | 34 | 35 | 36 | ## [0.0.1] - 2021-08-20 37 | 38 | 39 | ## 0.0.0 - 2021-07-07 40 | 41 | [Unreleased]: https://github.com/EspressoSystems/hotshot/compare/0.0.4...HEAD 42 | [0.0.4]: https://github.com/EspressoSystems/hotshot/compare/0.0.3...0.0.4 43 | [0.0.3]: https://github.com/EspressoSystems/hotshot/compare/0.0.2...0.0.3 44 | [0.0.2]: https://github.com/EspressoSystems/hotshot/compare/0.0.1...0.0.2 45 | [0.0.1]: https://github.com/EspressoSystems/hotshot/compare/0.0.0...0.0.1 46 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Default owners and repository maintainers: 2 | * @bfish713 3 | 4 | # Owner of `constants` crate 5 | /constants/ @shenkeyao 6 | 7 | # Owner of `hotshot` crate 8 | /hotshot/ @shenkeyao 9 | 10 | # Owner of `hotshot-qc` crate 11 | /hotshot-qc/ @dailinsubjam 12 | 13 | # Owner of `hotshot-stake-table` crate 14 | /hotshot-stake-table/ @dailinsubjam 15 | 16 | # Owner of `hotshot-state-prover` crate 17 | /hotshot-state-prover/ @dailinsubjam 18 | 19 | # Owner of `libp2p-networking` crate 20 | /libp2p-networking/ @rob-maron 21 | 22 | # Owner of `orchestrator` crate 23 | /orchestrator/ @rob-maron @elliedavidson 24 | 25 | # Owner of `task` crate 26 | /task/ @rob-maron 27 | 28 | # Owner of `task-impls` crate 29 | /task-impls/ @bfish713 @jparr721 @shenkeyao 30 | 31 | # Owner of `testing` crate 32 | /testing/ @bfish713 33 | 34 | # Owner of `types` crate 35 | /types/ @shenkeyao 36 | 37 | # Owner of `utils` crate 38 | /utils/ @shenkeyao 39 | 40 | # Owner of `web_server` crate 41 | /web_server/ @elliedavidson 42 | 43 | # Owner of updating dependencies 44 | # This owner is updated on a monthly basis 45 | *.lock @ss-es 46 | **/Cargo.toml @ss-es 47 | Cargo.toml @ss-es 48 | flake.nix @ss-es 49 | -------------------------------------------------------------------------------- /CODESTYLE.md: -------------------------------------------------------------------------------- 1 | # HotShot Code Style Guide 2 | 3 | ## Logging Guidelines 4 | 5 | ### Debug 6 | Use `debug!` for routine events that occur frequently within the system. 7 | 8 | Example: 9 | ```rust 10 | debug!("View {} decided", view_number); 11 | ``` 12 | 13 | ### Info 14 | Use `info!` for events that occur under specific conditions, which are not issues but might aid in debugging. 15 | 16 | Example: 17 | ```rust 18 | if missing_data { 19 | info!("Fetching missing data for query {}", query_id); 20 | } 21 | ``` 22 | 23 | ### Warn 24 | Use `warn!` for events that indicate a potential issue, which the system can handle, but might require human attention. 25 | 26 | Example: 27 | ```rust 28 | if message_loss_rate > threshold { 29 | warn!("Increased message loss detected: {}", message_loss_rate); 30 | } 31 | ``` 32 | 33 | ### Error 34 | Use `error!` for critical issues that could lead to a permanent degradation of the system without manual intervention. 35 | 36 | Example, we log an error when safety and liveness are violated: 37 | ```rust 38 | if !safety_check && !liveness_check { 39 | error!("Failed safety and liveness check \n High QC is {:?} Proposal QC is {:?} Locked view is {:?}", consensus.high_qc(), proposal.data.clone(), consensus.locked_view()); 40 | } 41 | ``` 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021-2025 Espresso Systems 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ARCHIVED 2 | 3 | This repository has been archived and all HotShot code has been moved to the [Espresso Sequencer monorepo](https://github.com/EspressoSystems/espresso-sequencer). 4 | 5 | The latest version of this code exists in the root directory of the above repository and is prefixed with `hotshot-`. 6 | -------------------------------------------------------------------------------- /WORKFLOW.md: -------------------------------------------------------------------------------- 1 | # Branch Naming Convention 2 | 3 | Branches to be merged must be named in the following fashion in order to trigger CI: 4 | 5 | ``` 6 | $INITIALS/$DESCRIPTION 7 | ``` 8 | 9 | Example: if the initials are `jr` and the description is `add_a_feature`, then the branch would be `jr/add_a_feature` 10 | 11 | # Best Practices 12 | 13 | - No pushing to main at all, only through pull requests 14 | - Releases are a pull request with just the release version bump, release after this is merged into main: https://github.com/bincode-org/bincode/pull/510 15 | - Only append commits to a pull request, don't rebase/squash/force push anything because this makes reviewing more annoying 16 | - Force pushing when rebasing onto main is fine, but merge commits are also fine 17 | - Reviewers should only review/approve, let the creator of the PR complete the PR 18 | - Always squash & merge when completing your pull request 19 | - The full history will be in the pull request 20 | - The main branch will have a single commit with the summary of the PR with a link if you want more details 21 | 22 | -------------------------------------------------------------------------------- /audits/README.md: -------------------------------------------------------------------------------- 1 | # Espresso Systems security audits 2 | 3 | 4 | Internal audits 5 | 6 | | Scope & Delivery date | Report | 7 | |-----------------------------|---------------------------------------------------------------| 8 | | HotShot - July 29, 2024 | [Report](./internal-reviews/EspressoHotshot-2024internal.pdf) | 9 | -------------------------------------------------------------------------------- /audits/internal-reviews/EspressoHotshot-2024internal.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/audits/internal-reviews/EspressoHotshot-2024internal.pdf -------------------------------------------------------------------------------- /clippy.toml: -------------------------------------------------------------------------------- 1 | allowed-wildcard-imports = [ "utils", "hotshot_task_impls", "hotshot_types" ] 2 | -------------------------------------------------------------------------------- /config/ValidatorConfigExample: -------------------------------------------------------------------------------- 1 | ValidatorConfig { 2 | public_key: VerKey( 3 | ( 4 | QuadExtField(2264797523581107490935262917175769123227923636811928330606075281145117212394 + 15807017392833049888165434456991157794698032464874424842715555348468160607934 * u), 5 | QuadExtField(7996517616082121122160563552650547601395271017260499735456299700133762512689 + 7504045709281061282278228438613345070383424761478787301859187055302953740948 * u), 6 | QuadExtField(1515973040548822760825076242090160370742046237881440422068330135941139244581 + 20251846261653098602911417004145145971080304248810966341160788194007704966108 * u) 7 | ) 8 | ), 9 | private_key: SignKey( 10 | BigInt( 11 | [3505488234151006356, 6655477166151225138, 3291219027844407676, 2153641080015542578] 12 | ) 13 | ), 14 | stake_value: 1, 15 | state_key_pair: StateKeyPair( 16 | KeyPair { 17 | sk: SignKey( 18 | BigInt( 19 | [2822822805887490846, 6664316196088353173, 4926510007447087464, 116097479308258694] 20 | ) 21 | ), 22 | vk: VerKey( 23 | Projective { 24 | x: BigInt([11315198235793138814, 4744451806709910489, 6921831025042192557, 1125393823825936625]), 25 | y: BigInt([13035879815613524256, 18225673961538637854, 12006860967936477969, 1516668567229692859]), 26 | t: BigInt([13450777528397789701, 12242009376162249168, 12596256366242272750, 3368076418495976469]), 27 | z: BigInt([10465708325245823445, 13967918689717629445, 14943426723808572731, 621075342718756551]) 28 | } 29 | ) 30 | } 31 | ) 32 | } -------------------------------------------------------------------------------- /config/ValidatorConfigFile.toml: -------------------------------------------------------------------------------- 1 | is_da = true 2 | seed = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] 3 | node_id = 0 4 | -------------------------------------------------------------------------------- /crates/builder-api/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "hotshot-builder-api" 3 | version = "0.1.7" 4 | edition = "2021" 5 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 6 | 7 | [dependencies] 8 | async-trait = { workspace = true } 9 | clap = { workspace = true } 10 | committable = { workspace = true } 11 | derive_more = { workspace = true, features = ["from"] } 12 | futures = { workspace = true } 13 | hotshot-types = { path = "../types" } 14 | serde = { workspace = true } 15 | tagged-base64 = { workspace = true } 16 | thiserror = { workspace = true } 17 | tide-disco = { workspace = true } 18 | toml = { workspace = true } 19 | vbs = { workspace = true } 20 | workspace-hack = { version = "0.1", path = "../workspace-hack" } 21 | -------------------------------------------------------------------------------- /crates/builder-api/README.md: -------------------------------------------------------------------------------- 1 | # hotshot-builder-api 2 | Minimal dependencies shared API definitions for HotShot Builder protocol 3 | 4 | # HotShot Consensus Module 5 | -------------------------------------------------------------------------------- /crates/builder-api/api/v0_1/submit.toml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Espresso Systems (espressosys.com) 2 | # This file is part of the HotShot Builder Protocol. 3 | # 4 | # MIT License 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | [meta] 26 | NAME = "hs-builder-submit" 27 | DESCRIPTION = "" 28 | FORMAT_VERSION = "0.1.0" 29 | 30 | [route.submit_txn] 31 | PATH = ["/submit"] 32 | METHOD = "POST" 33 | DOC = """ 34 | Submit a transaction to builder's private mempool." 35 | 36 | Returns transaction hash 37 | """ 38 | 39 | [route.submit_batch] 40 | PATH = ["/batch"] 41 | METHOD = "POST" 42 | DOC = """ 43 | Submit a list of transactions to builder's private mempool." 44 | 45 | Returns the corresponding list of transaction hashes 46 | """ 47 | 48 | [route.get_status] 49 | PATH = ["status/:transaction_hash"] 50 | METHOD = "GET" 51 | ":transaction_hash" = "TaggedBase64" 52 | DOC = """ 53 | Get the transaction's status. 54 | 55 | Returns "pending", "sequenced" or "rejected" with error. 56 | """ -------------------------------------------------------------------------------- /crates/builder-api/api/v0_3/builder.toml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Espresso Systems (espressosys.com) 2 | # This file is part of the HotShot Builder Protocol. 3 | # 4 | # MIT License 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | [meta] 25 | NAME = "hs-builder-get" 26 | DESCRIPTION = "" 27 | FORMAT_VERSION = "0.1.0" 28 | 29 | [route.bundle] 30 | PATH = ["bundle/:parent_view/:parent_hash/:view_number"] 31 | ":parent_view" = "Integer" 32 | ":parent_hash" = "TaggedBase64" 33 | ":view_number" = "Integer" 34 | DOC = """ 35 | Fetch the bundle from the builder for the specified view. 36 | """ 37 | 38 | [route.builder_address] 39 | PATH = ["builderaddress"] 40 | DOC = """ 41 | Get the builder's address. 42 | 43 | Returns the builder's public key 44 | """ 45 | -------------------------------------------------------------------------------- /crates/builder-api/api/v0_3/submit.toml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Espresso Systems (espressosys.com) 2 | # This file is part of the HotShot Builder Protocol. 3 | # 4 | # MIT License 5 | # 6 | # Permission is hereby granted, free of charge, to any person obtaining a copy 7 | # of this software and associated documentation files (the "Software"), to deal 8 | # in the Software without restriction, including without limitation the rights 9 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | # copies of the Software, and to permit persons to whom the Software is 11 | # furnished to do so, subject to the following conditions: 12 | 13 | # The above copyright notice and this permission notice shall be included in all 14 | # copies or substantial portions of the Software. 15 | 16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | # SOFTWARE. 23 | 24 | 25 | [meta] 26 | NAME = "hs-builder-submit" 27 | DESCRIPTION = "" 28 | FORMAT_VERSION = "0.1.0" 29 | 30 | [route.submit_txn] 31 | PATH = ["/submit"] 32 | METHOD = "POST" 33 | DOC = """ 34 | Submit a transaction to builder's private mempool." 35 | 36 | Returns transaction hash 37 | """ 38 | 39 | [route.submit_batch] 40 | PATH = ["/batch"] 41 | METHOD = "POST" 42 | DOC = """ 43 | Submit a list of transactions to builder's private mempool." 44 | 45 | Returns the corresponding list of transaction hashes 46 | """ 47 | -------------------------------------------------------------------------------- /crates/builder-api/src/api.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use std::{fs, path::Path}; 8 | 9 | use tide_disco::api::{Api, ApiError}; 10 | use toml::{map::Entry, Value}; 11 | use vbs::version::StaticVersionType; 12 | 13 | pub(crate) fn load_api( 14 | path: Option>, 15 | default: &str, 16 | extensions: impl IntoIterator, 17 | ) -> Result, ApiError> { 18 | let mut toml = match path { 19 | Some(path) => load_toml(path.as_ref())?, 20 | None => toml::from_str(default).map_err(|err| ApiError::CannotReadToml { 21 | reason: err.to_string(), 22 | })?, 23 | }; 24 | for extension in extensions { 25 | merge_toml(&mut toml, extension); 26 | } 27 | Api::new(toml) 28 | } 29 | 30 | fn merge_toml(into: &mut Value, from: Value) { 31 | if let (Value::Table(into), Value::Table(from)) = (into, from) { 32 | for (key, value) in from { 33 | match into.entry(key) { 34 | Entry::Occupied(mut entry) => merge_toml(entry.get_mut(), value), 35 | Entry::Vacant(entry) => { 36 | entry.insert(value); 37 | } 38 | } 39 | } 40 | } 41 | } 42 | 43 | fn load_toml(path: &Path) -> Result { 44 | let bytes = fs::read(path).map_err(|err| ApiError::CannotReadToml { 45 | reason: err.to_string(), 46 | })?; 47 | let string = std::str::from_utf8(&bytes).map_err(|err| ApiError::CannotReadToml { 48 | reason: err.to_string(), 49 | })?; 50 | toml::from_str(string).map_err(|err| ApiError::CannotReadToml { 51 | reason: err.to_string(), 52 | }) 53 | } 54 | -------------------------------------------------------------------------------- /crates/builder-api/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | mod api; 8 | pub mod v0_1; 9 | pub mod v0_2 { 10 | pub use super::v0_1::*; 11 | pub type Version = vbs::version::StaticVersion<0, 2>; 12 | } 13 | pub mod v0_99; 14 | -------------------------------------------------------------------------------- /crates/builder-api/src/v0_1/block_info.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use std::marker::PhantomData; 8 | 9 | use hotshot_types::{ 10 | traits::{node_implementation::NodeType, signature_key::BuilderSignatureKey, BlockPayload}, 11 | utils::BuilderCommitment, 12 | vid::VidCommitment, 13 | }; 14 | use serde::{Deserialize, Serialize}; 15 | 16 | #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Hash)] 17 | #[serde(bound = "")] 18 | pub struct AvailableBlockInfo { 19 | pub block_hash: BuilderCommitment, 20 | pub block_size: u64, 21 | pub offered_fee: u64, 22 | pub signature: 23 | <::BuilderSignatureKey as BuilderSignatureKey>::BuilderSignature, 24 | pub sender: ::BuilderSignatureKey, 25 | pub _phantom: PhantomData, 26 | } 27 | 28 | #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Hash)] 29 | #[serde(bound = "")] 30 | pub struct AvailableBlockData { 31 | pub block_payload: TYPES::BlockPayload, 32 | pub metadata: >::Metadata, 33 | pub signature: 34 | <::BuilderSignatureKey as BuilderSignatureKey>::BuilderSignature, 35 | pub sender: ::BuilderSignatureKey, 36 | } 37 | 38 | impl AvailableBlockData { 39 | pub fn validate_signature(&self) -> bool { 40 | // verify the signature over the message, construct the builder commitment 41 | let builder_commitment = self.block_payload.builder_commitment(&self.metadata); 42 | self.sender 43 | .validate_builder_signature(&self.signature, builder_commitment.as_ref()) 44 | } 45 | } 46 | 47 | #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Hash)] 48 | #[serde(bound = "")] 49 | pub struct AvailableBlockHeaderInput { 50 | pub vid_commitment: VidCommitment, 51 | // signature over vid_commitment, BlockPayload::Metadata, and offered_fee 52 | pub fee_signature: 53 | <::BuilderSignatureKey as BuilderSignatureKey>::BuilderSignature, 54 | // signature over the current response 55 | pub message_signature: 56 | <::BuilderSignatureKey as BuilderSignatureKey>::BuilderSignature, 57 | pub sender: ::BuilderSignatureKey, 58 | } 59 | 60 | impl AvailableBlockHeaderInput { 61 | pub fn validate_signature( 62 | &self, 63 | offered_fee: u64, 64 | metadata: &>::Metadata, 65 | ) -> bool { 66 | self.sender 67 | .validate_builder_signature(&self.message_signature, self.vid_commitment.as_ref()) 68 | && self.sender.validate_fee_signature( 69 | &self.fee_signature, 70 | offered_fee, 71 | metadata, 72 | &self.vid_commitment, 73 | ) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /crates/builder-api/src/v0_1/data_source.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use async_trait::async_trait; 8 | use committable::Commitment; 9 | use hotshot_types::{ 10 | traits::{node_implementation::NodeType, signature_key::SignatureKey}, 11 | utils::BuilderCommitment, 12 | vid::VidCommitment, 13 | }; 14 | 15 | use super::{ 16 | block_info::{AvailableBlockData, AvailableBlockHeaderInput, AvailableBlockInfo}, 17 | builder::{BuildError, TransactionStatus}, 18 | }; 19 | 20 | #[async_trait] 21 | pub trait BuilderDataSource { 22 | /// To get the list of available blocks 23 | async fn available_blocks( 24 | &self, 25 | for_parent: &VidCommitment, 26 | view_number: u64, 27 | sender: TYPES::SignatureKey, 28 | signature: &::PureAssembledSignatureType, 29 | ) -> Result>, BuildError>; 30 | 31 | /// To claim a block from the list of provided available blocks 32 | async fn claim_block( 33 | &self, 34 | block_hash: &BuilderCommitment, 35 | view_number: u64, 36 | sender: TYPES::SignatureKey, 37 | signature: &::PureAssembledSignatureType, 38 | ) -> Result, BuildError>; 39 | 40 | /// To claim a block from the list of provided available blocks and provide the number of nodes 41 | /// information to the builder for VID computation. 42 | async fn claim_block_with_num_nodes( 43 | &self, 44 | block_hash: &BuilderCommitment, 45 | view_number: u64, 46 | sender: TYPES::SignatureKey, 47 | signature: &::PureAssembledSignatureType, 48 | num_nodes: usize, 49 | ) -> Result, BuildError>; 50 | 51 | /// To claim a block header input 52 | async fn claim_block_header_input( 53 | &self, 54 | block_hash: &BuilderCommitment, 55 | view_number: u64, 56 | sender: TYPES::SignatureKey, 57 | signature: &::PureAssembledSignatureType, 58 | ) -> Result, BuildError>; 59 | 60 | /// To get the builder's address 61 | async fn builder_address(&self) -> Result; 62 | } 63 | 64 | #[async_trait] 65 | pub trait AcceptsTxnSubmits 66 | where 67 | I: NodeType, 68 | { 69 | async fn submit_txns( 70 | &self, 71 | txns: Vec<::Transaction>, 72 | ) -> Result::Transaction>>, BuildError>; 73 | 74 | async fn txn_status( 75 | &self, 76 | txn_hash: Commitment<::Transaction>, 77 | ) -> Result; 78 | } 79 | -------------------------------------------------------------------------------- /crates/builder-api/src/v0_1/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod block_info; 2 | pub mod builder; 3 | pub mod data_source; 4 | pub mod query_data; 5 | 6 | pub type Version = vbs::version::StaticVersion<0, 1>; 7 | -------------------------------------------------------------------------------- /crates/builder-api/src/v0_1/query_data.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use hotshot_types::traits::node_implementation::NodeType; 8 | use serde::{Deserialize, Serialize}; 9 | 10 | use super::block_info::AvailableBlockInfo; 11 | 12 | #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq, Hash)] 13 | #[serde(bound = "")] 14 | pub struct AvailableBlocksQueryData { 15 | pub blocks: Vec>, 16 | } 17 | -------------------------------------------------------------------------------- /crates/builder-api/src/v0_99/builder.rs: -------------------------------------------------------------------------------- 1 | use futures::FutureExt; 2 | use hotshot_types::traits::node_implementation::NodeType; 3 | use tide_disco::{api::ApiError, method::ReadState, Api}; 4 | 5 | use super::{data_source::BuilderDataSource, Version}; 6 | use crate::api::load_api; 7 | /// No changes to these types 8 | pub use crate::v0_1::builder::{submit_api, BuildError, Error, Options}; 9 | 10 | pub fn define_api( 11 | options: &Options, 12 | ) -> Result, ApiError> 13 | where 14 | State: 'static + Send + Sync + ReadState, 15 | ::State: Send + Sync + BuilderDataSource, 16 | { 17 | let mut api = load_api::( 18 | options.api_path.as_ref(), 19 | include_str!("../../api/v0_3/builder.toml"), 20 | options.extensions.clone(), 21 | )?; 22 | api.with_version("0.0.3".parse().unwrap()) 23 | .get("bundle", |req, state| { 24 | async move { 25 | let parent_view = req.integer_param("parent_view")?; 26 | let parent_hash = req.blob_param("parent_hash")?; 27 | let view_number = req.integer_param("view_number")?; 28 | state 29 | .bundle(parent_view, &parent_hash, view_number) 30 | .await 31 | .map_err(|source| Error::BlockClaim { 32 | source, 33 | resource: format!( 34 | "Block for parent {parent_hash}@{parent_view} and view {view_number}" 35 | ), 36 | }) 37 | } 38 | .boxed() 39 | })? 40 | .get("builder_address", |_req, state| { 41 | async move { state.builder_address().await.map_err(Error::BuilderAddress) }.boxed() 42 | })?; 43 | Ok(api) 44 | } 45 | -------------------------------------------------------------------------------- /crates/builder-api/src/v0_99/data_source.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use hotshot_types::{bundle::Bundle, traits::node_implementation::NodeType, vid::VidCommitment}; 3 | 4 | use super::builder::BuildError; 5 | /// No changes to these types 6 | pub use crate::v0_1::data_source::AcceptsTxnSubmits; 7 | 8 | #[async_trait] 9 | pub trait BuilderDataSource { 10 | /// To get the list of available blocks 11 | async fn bundle( 12 | &self, 13 | parent_view: u64, 14 | parent_hash: &VidCommitment, 15 | view_number: u64, 16 | ) -> Result, BuildError>; 17 | 18 | /// To get the builder's address 19 | async fn builder_address(&self) -> Result; 20 | } 21 | -------------------------------------------------------------------------------- /crates/builder-api/src/v0_99/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod builder; 2 | pub mod data_source; 3 | /// No changes to this module 4 | pub use super::v0_1::query_data; 5 | 6 | pub type Version = vbs::version::StaticVersion<0, 99>; 7 | -------------------------------------------------------------------------------- /crates/example-types/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "hotshot-example-types" 3 | version = { workspace = true } 4 | edition = { workspace = true } 5 | description = "Types and traits for the HotShot consesus module" 6 | authors = { workspace = true } 7 | 8 | [features] 9 | default = [] 10 | # NOTE this is used to activate the slow tests we don't wish to run in CI 11 | slow-tests = [] 12 | 13 | [dependencies] 14 | anyhow = { workspace = true } 15 | async-lock = { workspace = true } 16 | async-trait = { workspace = true } 17 | committable = { workspace = true } 18 | hotshot = { path = "../hotshot" } 19 | hotshot-task-impls = { path = "../task-impls", version = "0.5.36", default-features = false } 20 | hotshot-types = { path = "../types" } 21 | jf-vid = { workspace = true } 22 | rand = { workspace = true } 23 | reqwest = { workspace = true } 24 | serde = { workspace = true } 25 | sha2 = { workspace = true } 26 | sha3 = "^0.10" 27 | thiserror = { workspace = true } 28 | time = { workspace = true } 29 | tokio = { workspace = true } 30 | url = { workspace = true } 31 | vbs = { workspace = true } 32 | workspace-hack = { version = "0.1", path = "../workspace-hack" } 33 | -------------------------------------------------------------------------------- /crates/example-types/src/auction_results_provider_types.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use anyhow::{bail, Result}; 8 | use async_trait::async_trait; 9 | use hotshot_types::traits::{ 10 | auction_results_provider::AuctionResultsProvider, 11 | node_implementation::{HasUrls, NodeType}, 12 | }; 13 | use serde::{Deserialize, Serialize}; 14 | use url::Url; 15 | 16 | /// A mock result for the auction solver. This type is just a pointer to a URL. 17 | #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] 18 | pub struct TestAuctionResult { 19 | /// The URL of the builder to reach out to. 20 | pub urls: Vec, 21 | } 22 | 23 | impl HasUrls for TestAuctionResult { 24 | fn urls(&self) -> Vec { 25 | self.urls.clone() 26 | } 27 | } 28 | 29 | /// The test auction results type is used to mimic the results from the Solver. 30 | #[derive(Clone, Debug, Default)] 31 | pub struct TestAuctionResultsProvider { 32 | /// We intentionally allow for the results to be pre-cooked for the unit test to guarantee a 33 | /// particular outcome is met. 34 | pub solver_results: TYPES::AuctionResult, 35 | 36 | /// A canned type to ensure that an error is thrown in absence of a true fault-injectible 37 | /// system for logical tests. This will guarantee that `fetch_auction_result` always throws an 38 | /// error. 39 | pub should_return_err: bool, 40 | 41 | /// The broadcast URL that the solver is running on. This type allows for the url to be 42 | /// optional, where `None` means to just return whatever `solver_results` contains, and `Some` 43 | /// means that we have a `FakeSolver` instance available to query. 44 | pub broadcast_url: Option, 45 | } 46 | 47 | #[async_trait] 48 | impl AuctionResultsProvider for TestAuctionResultsProvider { 49 | /// Mock fetching the auction results, with optional error injection to simulate failure cases 50 | /// in the solver. 51 | async fn fetch_auction_result(&self, view_number: TYPES::View) -> Result { 52 | if let Some(url) = &self.broadcast_url { 53 | let resp = 54 | reqwest::get(url.join(&format!("/v0/api/auction_results/{}", *view_number))?) 55 | .await? 56 | .json::() 57 | .await?; 58 | 59 | Ok(resp) 60 | } else { 61 | if self.should_return_err { 62 | bail!("Something went wrong") 63 | } 64 | 65 | // Otherwise, return our pre-made results 66 | Ok(self.solver_results.clone()) 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /crates/example-types/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | /// block types 8 | pub mod block_types; 9 | 10 | /// Implementations for testing/examples 11 | pub mod state_types; 12 | 13 | /// node types 14 | pub mod node_types; 15 | 16 | /// storage types for hotshot storage 17 | pub mod storage_types; 18 | 19 | /// auction types for solver-to-hotshot interactions 20 | pub mod auction_results_provider_types; 21 | 22 | /// add a delay to async functions 23 | pub mod testable_delay; 24 | -------------------------------------------------------------------------------- /crates/examples/combined/multi-validator.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! A multi-validator using both the web server libp2p 8 | use clap::Parser; 9 | use hotshot::helpers::initialize_logging; 10 | use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; 11 | use hotshot_orchestrator::client::{MultiValidatorArgs, ValidatorArgs}; 12 | use tokio::spawn; 13 | use tracing::instrument; 14 | 15 | use crate::types::{Network, NodeImpl, ThisRun}; 16 | 17 | /// types used for this example 18 | pub mod types; 19 | 20 | /// general infra used for this example 21 | #[path = "../infra/mod.rs"] 22 | pub mod infra; 23 | 24 | #[tokio::main] 25 | #[instrument] 26 | async fn main() { 27 | // Initialize logging 28 | initialize_logging(); 29 | 30 | let args = MultiValidatorArgs::parse(); 31 | tracing::debug!("connecting to orchestrator at {:?}", args.url); 32 | let mut nodes = Vec::new(); 33 | for node_index in 0..args.num_nodes { 34 | let args = args.clone(); 35 | 36 | let node = spawn(async move { 37 | infra::main_entry_point::( 38 | ValidatorArgs::from_multi_args(args, node_index), 39 | ) 40 | .await; 41 | }); 42 | nodes.push(node); 43 | } 44 | let _result = futures::future::join_all(nodes).await; 45 | } 46 | -------------------------------------------------------------------------------- /crates/examples/combined/orchestrator.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Orchestrator using the web server 8 | /// types used for this example 9 | pub mod types; 10 | 11 | use hotshot::helpers::initialize_logging; 12 | use hotshot_example_types::state_types::TestTypes; 13 | use tracing::instrument; 14 | 15 | use crate::infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs}; 16 | /// general infra used for this example 17 | #[path = "../infra/mod.rs"] 18 | pub mod infra; 19 | 20 | #[tokio::main] 21 | #[instrument] 22 | async fn main() { 23 | // Initialize logging 24 | initialize_logging(); 25 | 26 | let (config, orchestrator_url) = read_orchestrator_init_config::(); 27 | run_orchestrator::(OrchestratorArgs:: { 28 | url: orchestrator_url.clone(), 29 | config: config.clone(), 30 | }) 31 | .await; 32 | } 33 | -------------------------------------------------------------------------------- /crates/examples/combined/types.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use std::fmt::Debug; 8 | 9 | use hotshot::traits::implementations::CombinedNetworks; 10 | use hotshot_example_types::{ 11 | auction_results_provider_types::TestAuctionResultsProvider, state_types::TestTypes, 12 | storage_types::TestStorage, 13 | }; 14 | use hotshot_types::traits::node_implementation::NodeImplementation; 15 | use serde::{Deserialize, Serialize}; 16 | 17 | use crate::infra::CombinedDaRun; 18 | 19 | /// dummy struct so we can choose types 20 | #[derive(Clone, Debug, Deserialize, Serialize, Hash, PartialEq, Eq)] 21 | pub struct NodeImpl {} 22 | 23 | /// Convenience type alias 24 | pub type Network = CombinedNetworks; 25 | 26 | impl NodeImplementation for NodeImpl { 27 | type Network = Network; 28 | type Storage = TestStorage; 29 | type AuctionResultsProvider = TestAuctionResultsProvider; 30 | } 31 | /// convenience type alias 32 | pub type ThisRun = CombinedDaRun; 33 | -------------------------------------------------------------------------------- /crates/examples/combined/validator.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! A validator using both the web server and libp2p 8 | 9 | use clap::Parser; 10 | use hotshot::helpers::initialize_logging; 11 | use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; 12 | use hotshot_orchestrator::client::ValidatorArgs; 13 | use local_ip_address::local_ip; 14 | use tracing::{debug, instrument}; 15 | 16 | use crate::types::{Network, NodeImpl, ThisRun}; 17 | 18 | /// types used for this example 19 | pub mod types; 20 | 21 | /// general infra used for this example 22 | #[path = "../infra/mod.rs"] 23 | pub mod infra; 24 | 25 | #[tokio::main] 26 | #[instrument] 27 | async fn main() { 28 | // Initialize logging 29 | initialize_logging(); 30 | 31 | let mut args = ValidatorArgs::parse(); 32 | 33 | // If we did not set the advertise address, use our local IP and port 8000 34 | let local_ip = local_ip().expect("failed to get local IP"); 35 | args.advertise_address = Some(args.advertise_address.unwrap_or(format!("{local_ip}:8000"))); 36 | 37 | debug!("connecting to orchestrator at {:?}", args.url); 38 | infra::main_entry_point::(args).await; 39 | } 40 | -------------------------------------------------------------------------------- /crates/examples/libp2p/all.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! An example program using libp2p 8 | /// types used for this example 9 | pub mod types; 10 | 11 | use hotshot::helpers::initialize_logging; 12 | use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; 13 | use hotshot_orchestrator::client::ValidatorArgs; 14 | use infra::{gen_local_address, BUILDER_BASE_PORT, VALIDATOR_BASE_PORT}; 15 | use tokio::spawn; 16 | use tracing::instrument; 17 | 18 | use crate::{ 19 | infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs}, 20 | types::{Network, NodeImpl, ThisRun}, 21 | }; 22 | 23 | /// general infra used for this example 24 | #[path = "../infra/mod.rs"] 25 | pub mod infra; 26 | 27 | #[tokio::main] 28 | #[instrument] 29 | async fn main() { 30 | // Initialize logging 31 | initialize_logging(); 32 | 33 | // use configfile args 34 | let (config, orchestrator_url) = read_orchestrator_init_config::(); 35 | 36 | // orchestrator 37 | spawn(run_orchestrator::(OrchestratorArgs { 38 | url: orchestrator_url.clone(), 39 | config: config.clone(), 40 | })); 41 | 42 | // nodes 43 | let mut nodes = Vec::new(); 44 | for i in 0..config.config.num_nodes_with_stake.into() { 45 | // Calculate our libp2p advertise address, which we will later derive the 46 | // bind address from for example purposes. 47 | let advertise_address = gen_local_address::(i); 48 | let builder_address = gen_local_address::(i); 49 | let orchestrator_url = orchestrator_url.clone(); 50 | let node = spawn(async move { 51 | infra::main_entry_point::( 52 | ValidatorArgs { 53 | url: orchestrator_url, 54 | advertise_address: Some(advertise_address.to_string()), 55 | builder_address: Some(builder_address), 56 | network_config_file: None, 57 | }, 58 | ) 59 | .await; 60 | }); 61 | nodes.push(node); 62 | } 63 | futures::future::join_all(nodes).await; 64 | } 65 | -------------------------------------------------------------------------------- /crates/examples/libp2p/multi-validator.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! A multi-validator using libp2p 8 | use clap::Parser; 9 | use hotshot::helpers::initialize_logging; 10 | use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; 11 | use hotshot_orchestrator::client::{MultiValidatorArgs, ValidatorArgs}; 12 | use tokio::spawn; 13 | use tracing::instrument; 14 | 15 | use crate::types::{Network, NodeImpl, ThisRun}; 16 | 17 | /// types used for this example 18 | pub mod types; 19 | 20 | /// general infra used for this example 21 | #[path = "../infra/mod.rs"] 22 | pub mod infra; 23 | 24 | #[tokio::main] 25 | #[instrument] 26 | async fn main() { 27 | // Initialize logging 28 | initialize_logging(); 29 | 30 | let args = MultiValidatorArgs::parse(); 31 | tracing::debug!("connecting to orchestrator at {:?}", args.url); 32 | let mut nodes = Vec::new(); 33 | for node_index in 0..args.num_nodes { 34 | let args = args.clone(); 35 | 36 | let node = spawn(async move { 37 | infra::main_entry_point::( 38 | ValidatorArgs::from_multi_args(args, node_index), 39 | ) 40 | .await; 41 | }); 42 | nodes.push(node); 43 | } 44 | let _result = futures::future::join_all(nodes).await; 45 | } 46 | -------------------------------------------------------------------------------- /crates/examples/libp2p/types.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use std::fmt::Debug; 8 | 9 | use hotshot::traits::implementations::Libp2pNetwork; 10 | use hotshot_example_types::{ 11 | auction_results_provider_types::TestAuctionResultsProvider, state_types::TestTypes, 12 | storage_types::TestStorage, 13 | }; 14 | use hotshot_types::traits::node_implementation::NodeImplementation; 15 | use serde::{Deserialize, Serialize}; 16 | 17 | use crate::infra::Libp2pDaRun; 18 | 19 | /// dummy struct so we can choose types 20 | #[derive(Clone, Debug, Deserialize, Serialize, Hash, PartialEq, Eq)] 21 | pub struct NodeImpl {} 22 | 23 | /// Convenience type alias 24 | pub type Network = Libp2pNetwork; 25 | 26 | impl NodeImplementation for NodeImpl { 27 | type Network = Network; 28 | type Storage = TestStorage; 29 | type AuctionResultsProvider = TestAuctionResultsProvider; 30 | } 31 | /// convenience type alias 32 | pub type ThisRun = Libp2pDaRun; 33 | -------------------------------------------------------------------------------- /crates/examples/libp2p/validator.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! A validator using libp2p 8 | 9 | use clap::Parser; 10 | use hotshot::helpers::initialize_logging; 11 | use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; 12 | use hotshot_orchestrator::client::ValidatorArgs; 13 | use local_ip_address::local_ip; 14 | use tracing::{debug, instrument}; 15 | 16 | use crate::types::{Network, NodeImpl, ThisRun}; 17 | 18 | /// types used for this example 19 | pub mod types; 20 | 21 | /// general infra used for this example 22 | #[path = "../infra/mod.rs"] 23 | pub mod infra; 24 | 25 | #[tokio::main] 26 | #[instrument] 27 | async fn main() { 28 | // Initialize logging 29 | initialize_logging(); 30 | 31 | let mut args = ValidatorArgs::parse(); 32 | 33 | // If we did not set the advertise address, use our local IP and port 8000 34 | let local_ip = local_ip().expect("failed to get local IP"); 35 | args.advertise_address = Some(args.advertise_address.unwrap_or(format!("{local_ip}:8000"))); 36 | 37 | debug!("connecting to orchestrator at {:?}", args.url); 38 | infra::main_entry_point::(args).await; 39 | } 40 | -------------------------------------------------------------------------------- /crates/examples/orchestrator.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! An orchestrator 8 | 9 | use hotshot::helpers::initialize_logging; 10 | use hotshot_example_types::state_types::TestTypes; 11 | use tracing::instrument; 12 | 13 | use crate::infra::{read_orchestrator_init_config, run_orchestrator, OrchestratorArgs}; 14 | 15 | /// general infra used for this example 16 | #[path = "./infra/mod.rs"] 17 | pub mod infra; 18 | 19 | #[tokio::main] 20 | #[instrument] 21 | async fn main() { 22 | // Initialize logging 23 | initialize_logging(); 24 | 25 | let (config, orchestrator_url) = read_orchestrator_init_config::(); 26 | run_orchestrator::(OrchestratorArgs:: { 27 | url: orchestrator_url.clone(), 28 | config: config.clone(), 29 | }) 30 | .await; 31 | } 32 | -------------------------------------------------------------------------------- /crates/examples/push-cdn/multi-validator.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! A multi validator 8 | use clap::Parser; 9 | use hotshot::helpers::initialize_logging; 10 | use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; 11 | use hotshot_orchestrator::client::{MultiValidatorArgs, ValidatorArgs}; 12 | use tokio::spawn; 13 | use tracing::instrument; 14 | 15 | use crate::types::{Network, NodeImpl, ThisRun}; 16 | 17 | /// types used for this example 18 | pub mod types; 19 | 20 | /// general infra used for this example 21 | #[path = "../infra/mod.rs"] 22 | pub mod infra; 23 | 24 | #[tokio::main] 25 | #[instrument] 26 | async fn main() { 27 | // Initialize logging 28 | initialize_logging(); 29 | 30 | let args = MultiValidatorArgs::parse(); 31 | tracing::debug!("connecting to orchestrator at {:?}", args.url); 32 | let mut nodes = Vec::new(); 33 | for node_index in 0..args.num_nodes { 34 | let args = args.clone(); 35 | 36 | let node = spawn(async move { 37 | infra::main_entry_point::( 38 | ValidatorArgs::from_multi_args(args, node_index), 39 | ) 40 | .await; 41 | }); 42 | nodes.push(node); 43 | } 44 | let _result = futures::future::join_all(nodes).await; 45 | } 46 | -------------------------------------------------------------------------------- /crates/examples/push-cdn/types.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use hotshot::traits::{implementations::PushCdnNetwork, NodeImplementation}; 8 | use hotshot_example_types::{ 9 | auction_results_provider_types::TestAuctionResultsProvider, state_types::TestTypes, 10 | storage_types::TestStorage, 11 | }; 12 | use hotshot_types::traits::node_implementation::NodeType; 13 | use serde::{Deserialize, Serialize}; 14 | 15 | use crate::infra::PushCdnDaRun; 16 | 17 | #[derive(Clone, Deserialize, Serialize, Hash, PartialEq, Eq)] 18 | /// Convenience type alias 19 | pub struct NodeImpl {} 20 | 21 | /// Convenience type alias 22 | pub type Network = PushCdnNetwork<::SignatureKey>; 23 | 24 | impl NodeImplementation for NodeImpl { 25 | type Network = Network; 26 | type Storage = TestStorage; 27 | type AuctionResultsProvider = TestAuctionResultsProvider; 28 | } 29 | 30 | /// Convenience type alias 31 | pub type ThisRun = PushCdnDaRun; 32 | -------------------------------------------------------------------------------- /crates/examples/push-cdn/validator.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! A validator 8 | use clap::Parser; 9 | use hotshot::helpers::initialize_logging; 10 | use hotshot_example_types::{node_types::TestVersions, state_types::TestTypes}; 11 | use hotshot_orchestrator::client::ValidatorArgs; 12 | use tracing::{debug, instrument}; 13 | 14 | use crate::types::{Network, NodeImpl, ThisRun}; 15 | 16 | /// types used for this example 17 | pub mod types; 18 | 19 | /// general infra used for this example 20 | #[path = "../infra/mod.rs"] 21 | pub mod infra; 22 | 23 | #[tokio::main] 24 | #[instrument] 25 | async fn main() { 26 | // Initialize logging 27 | initialize_logging(); 28 | 29 | let args = ValidatorArgs::parse(); 30 | debug!("connecting to orchestrator at {:?}", args.url); 31 | infra::main_entry_point::(args).await; 32 | } 33 | -------------------------------------------------------------------------------- /crates/fakeapi/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "hotshot-fakeapi" 3 | documentation = { workspace = true } 4 | version = { workspace = true } 5 | authors = { workspace = true } 6 | edition = { workspace = true } 7 | rust-version = { workspace = true } 8 | homepage = { workspace = true } 9 | repository = { workspace = true } 10 | 11 | [dependencies] 12 | anyhow = { workspace = true } 13 | async-lock = { workspace = true } 14 | async-trait = { workspace = true } 15 | futures = { workspace = true } 16 | hotshot-example-types = { path = "../example-types" } 17 | hotshot-types = { path = "../types" } 18 | rand = { workspace = true } 19 | tide-disco = { workspace = true } 20 | tokio = { workspace = true } 21 | toml = { workspace = true } 22 | vbs = { workspace = true } 23 | workspace-hack = { version = "0.1", path = "../workspace-hack" } 24 | 25 | [lints] 26 | workspace = true 27 | -------------------------------------------------------------------------------- /crates/fakeapi/apis/solver.toml: -------------------------------------------------------------------------------- 1 | [meta] 2 | NAME = "fake-solver" 3 | DESCRIPTION = "Fake Solver for testing within hotshot" 4 | FORMAT_VERSION = "0.1.0" 5 | 6 | # GET the auction result - non permissioned 7 | [route.get_auction_results_non_permissioned] 8 | PATH = ["auction_results/:view_number"] 9 | ":view_number" = "Integer" 10 | METHOD = "GET" 11 | DOC = """ 12 | GET a fake auction result from the fake Solver. Returns a json object containing a list of 13 | builder URLs corresponding to other running instances of fake-builder, or an empty list if no 14 | values are present. This endpoint is open access. 15 | """ 16 | 17 | # GET the auction result - permissioned 18 | [route.get_auction_results_permissioned] 19 | PATH = ["auction_results/:view_number/:signature"] 20 | ":view_number" = "Integer" 21 | ":signature" = "TaggedBase64" 22 | METHOD = "GET" 23 | DOC = """ 24 | GET a fake auction result from the fake Solver. Returns a json object containing a list of 25 | builder URLs corresponding to other running instances of fake-builder, or an empty list if no 26 | values are present. This endpoint checks the leader provided in the signature. 27 | """ 28 | -------------------------------------------------------------------------------- /crates/fakeapi/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Fake APIs 2 | 3 | /// Fake solver 4 | pub mod fake_solver; 5 | -------------------------------------------------------------------------------- /crates/hotshot-stake-table/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "hotshot-stake-table" 3 | description = "Stake table implementations for HotShot" 4 | version = { workspace = true } 5 | authors = { workspace = true } 6 | edition = { workspace = true } 7 | rust-version = { workspace = true } 8 | 9 | [dependencies] 10 | ark-bn254 = "0.4" 11 | ark-ed-on-bn254 = "0.4" 12 | ark-ff = "0.4" 13 | ark-serialize = { workspace = true } 14 | ark-std = { workspace = true } 15 | digest = { workspace = true } 16 | hotshot-types = { path = "../types" } 17 | jf-crhf = { workspace = true } 18 | jf-rescue = { workspace = true } 19 | jf-signature = { workspace = true, features = ["bls", "schnorr"] } 20 | jf-utils = { workspace = true } 21 | primitive-types = { workspace = true } 22 | serde = { workspace = true, features = ["rc"] } 23 | tagged-base64 = { workspace = true } 24 | workspace-hack = { version = "0.1", path = "../workspace-hack" } 25 | 26 | [dev-dependencies] 27 | rand_chacha = { workspace = true } 28 | 29 | [features] 30 | default = ["parallel"] 31 | std = ["ark-std/std", "ark-serialize/std", "ark-ff/std"] 32 | parallel = ["jf-utils/parallel", "ark-ff/parallel"] 33 | 34 | [lints] 35 | workspace = true 36 | -------------------------------------------------------------------------------- /crates/hotshot-stake-table/src/config.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Configuration file for stake table 8 | 9 | /// Capacity of a stake table 10 | pub const STAKE_TABLE_CAPACITY: usize = 200; 11 | -------------------------------------------------------------------------------- /crates/hotshot-stake-table/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! This crate contains some stake table implementations for `HotShot` system. 8 | pub mod config; 9 | pub mod mt_based; 10 | pub mod utils; 11 | pub mod vec_based; 12 | -------------------------------------------------------------------------------- /crates/hotshot-stake-table/src/mt_based/config.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Config file for stake table 8 | use ark_ff::PrimeField; 9 | use ark_std::vec; 10 | use jf_rescue::crhf::FixedLengthRescueCRHF; 11 | use jf_signature::bls_over_bn254; 12 | 13 | use crate::utils::ToFields; 14 | 15 | /// Branch of merkle tree. 16 | /// Set to 3 because we are currently using RATE-3 rescue hash function 17 | pub(crate) const TREE_BRANCH: usize = 3; 18 | 19 | /// Internal type of Merkle node value(commitment) 20 | pub(crate) type FieldType = ark_bn254::Fq; 21 | /// Hash algorithm used in Merkle tree, using a RATE-3 rescue 22 | pub(crate) type Digest = FixedLengthRescueCRHF; 23 | 24 | impl ToFields for FieldType { 25 | const SIZE: usize = 1; 26 | fn to_fields(&self) -> Vec { 27 | vec![*self] 28 | } 29 | } 30 | 31 | impl ToFields for bls_over_bn254::VerKey { 32 | const SIZE: usize = 2; 33 | fn to_fields(&self) -> Vec { 34 | #[allow(clippy::ignored_unit_patterns)] 35 | let bytes = jf_utils::to_bytes!(&self.to_affine()).unwrap(); 36 | let x = ::from_le_bytes_mod_order(&bytes[..32]); 37 | let y = ::from_le_bytes_mod_order(&bytes[32..]); 38 | vec![x, y] 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /crates/hotshot-stake-table/src/utils.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Utilities to help building a stake table. 8 | 9 | use ark_ff::{Field, PrimeField}; 10 | use primitive_types::U256; 11 | 12 | /// A trait that converts into a field element. 13 | pub trait ToFields { 14 | /// The number of field elements needed to represent the given struct. 15 | const SIZE: usize; 16 | 17 | /// Convert the given struct into a list of field elements. 18 | fn to_fields(&self) -> Vec; 19 | } 20 | 21 | /// convert a U256 to a field element. 22 | pub(crate) fn u256_to_field(v: &U256) -> F { 23 | let mut bytes = vec![0u8; 32]; 24 | v.to_little_endian(&mut bytes); 25 | F::from_le_bytes_mod_order(&bytes) 26 | } 27 | -------------------------------------------------------------------------------- /crates/hotshot-stake-table/src/vec_based/config.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Config file for stake table 8 | use ark_ff::PrimeField; 9 | use ark_std::vec; 10 | /// Schnorr verification key as auxiliary information 11 | pub use hotshot_types::light_client::StateVerKey; 12 | /// BLS verification key as indexing key 13 | pub use jf_signature::bls_over_bn254::VerKey as QCVerKey; 14 | use jf_utils::to_bytes; 15 | 16 | use crate::utils::ToFields; 17 | /// Type for commitment 18 | pub type FieldType = ark_ed_on_bn254::Fq; 19 | 20 | /// Hashable representation of a key 21 | /// NOTE: commitment is only used in light client contract. 22 | /// For this application, we needs only hash the Schnorr verification key. 23 | impl ToFields for StateVerKey { 24 | const SIZE: usize = 2; 25 | 26 | fn to_fields(&self) -> Vec { 27 | let p = self.to_affine(); 28 | vec![p.x, p.y] 29 | } 30 | } 31 | 32 | impl ToFields for QCVerKey { 33 | const SIZE: usize = 3; 34 | 35 | fn to_fields(&self) -> Vec { 36 | #[allow(clippy::ignored_unit_patterns)] 37 | match to_bytes!(&self.to_affine()) { 38 | Ok(bytes) => { 39 | vec![ 40 | FieldType::from_le_bytes_mod_order(&bytes[..31]), 41 | FieldType::from_le_bytes_mod_order(&bytes[31..62]), 42 | FieldType::from_le_bytes_mod_order(&bytes[62..]), 43 | ] 44 | } 45 | Err(_) => unreachable!(), 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /crates/hotshot/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = { workspace = true } 3 | description = "HotShot consesus module" 4 | edition = { workspace = true } 5 | name = "hotshot" 6 | readme = "README.md" 7 | version = { workspace = true } 8 | rust-version = { workspace = true } 9 | 10 | [features] 11 | default = ["docs", "doc-images"] 12 | example-upgrade = ["hotshot-task-impls/example-upgrade"] 13 | rewind = ["hotshot-task-impls/rewind"] 14 | 15 | # Build the extended documentation 16 | docs = [] 17 | doc-images = [] 18 | hotshot-testing = [] 19 | 20 | [dependencies] 21 | anyhow = { workspace = true } 22 | async-broadcast = { workspace = true } 23 | async-lock = { workspace = true } 24 | async-trait = { workspace = true } 25 | bimap = "0.6" 26 | bincode = { workspace = true } 27 | blake3 = { workspace = true } 28 | cdn-broker = { workspace = true, features = ["global-permits"] } 29 | cdn-client = { workspace = true } 30 | cdn-marshal = { workspace = true } 31 | chrono = { workspace = true } 32 | committable = { workspace = true } 33 | dashmap = { workspace = true } 34 | derive_more = { workspace = true } 35 | either = { workspace = true } 36 | futures = { workspace = true } 37 | hotshot-task = { path = "../task" } 38 | hotshot-task-impls = { path = "../task-impls", version = "0.5.36", default-features = false } 39 | hotshot-types = { path = "../types" } 40 | libp2p-identity = { workspace = true } 41 | libp2p-networking = { workspace = true } 42 | lru = { workspace = true } 43 | num_enum = "0.7" 44 | parking_lot.workspace = true 45 | portpicker = "0.1" 46 | primitive-types = { workspace = true } 47 | rand = { workspace = true } 48 | serde = { workspace = true, features = ["rc"] } 49 | sha2 = { workspace = true } 50 | time = { workspace = true } 51 | 52 | tokio = { workspace = true } 53 | tracing = { workspace = true } 54 | tracing-subscriber = { workspace = true } 55 | url = { workspace = true } 56 | utils = { path = "../utils" } 57 | vbs = { workspace = true } 58 | workspace-hack = { version = "0.1", path = "../workspace-hack" } 59 | 60 | [dev-dependencies] 61 | blake3 = { workspace = true } 62 | 63 | [lints] 64 | workspace = true 65 | -------------------------------------------------------------------------------- /crates/hotshot/src/documentation.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | // This is prosaic documentation, we don't need clippy 8 | #![allow( 9 | clippy::all, 10 | clippy::pedantic, 11 | missing_docs, 12 | clippy::missing_docs_in_private_items, 13 | non_camel_case_types 14 | )] 15 | -------------------------------------------------------------------------------- /crates/hotshot/src/helpers.rs: -------------------------------------------------------------------------------- 1 | use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter}; 2 | 3 | /// Initializes logging 4 | pub fn initialize_logging() { 5 | // Parse the `RUST_LOG_SPAN_EVENTS` environment variable 6 | let span_event_filter = match std::env::var("RUST_LOG_SPAN_EVENTS") { 7 | Ok(val) => val 8 | .split(',') 9 | .map(|s| match s.trim() { 10 | "new" => FmtSpan::NEW, 11 | "enter" => FmtSpan::ENTER, 12 | "exit" => FmtSpan::EXIT, 13 | "close" => FmtSpan::CLOSE, 14 | "active" => FmtSpan::ACTIVE, 15 | "full" => FmtSpan::FULL, 16 | _ => FmtSpan::NONE, 17 | }) 18 | .fold(FmtSpan::NONE, |acc, x| acc | x), 19 | Err(_) => FmtSpan::NONE, 20 | }; 21 | 22 | // Conditionally initialize in `json` mode 23 | if std::env::var("RUST_LOG_FORMAT") == Ok("json".to_string()) { 24 | let _ = tracing_subscriber::fmt() 25 | .with_env_filter(EnvFilter::from_default_env()) 26 | .with_span_events(span_event_filter) 27 | .json() 28 | .try_init(); 29 | } else { 30 | let _ = tracing_subscriber::fmt() 31 | .with_env_filter(EnvFilter::from_default_env()) 32 | .with_span_events(span_event_filter) 33 | .try_init(); 34 | }; 35 | } 36 | -------------------------------------------------------------------------------- /crates/hotshot/src/traits.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | /// Sortition trait 8 | pub mod election; 9 | mod networking; 10 | mod node_implementation; 11 | 12 | pub use hotshot_types::traits::{BlockPayload, ValidatedState}; 13 | pub use libp2p_networking::network::NetworkNodeConfigBuilder; 14 | pub use networking::{NetworkError, NetworkReliability}; 15 | pub use node_implementation::{NodeImplementation, TestableNodeImplementation}; 16 | 17 | /// Module for publicly usable implementations of the traits 18 | pub mod implementations { 19 | pub use super::networking::{ 20 | combined_network::{CombinedNetworks, UnderlyingCombinedNetworks}, 21 | libp2p_network::{ 22 | derive_libp2p_keypair, derive_libp2p_multiaddr, derive_libp2p_peer_id, GossipConfig, 23 | Libp2pMetricsValue, Libp2pNetwork, PeerInfoVec, RequestResponseConfig, 24 | }, 25 | memory_network::{MasterMap, MemoryNetwork}, 26 | push_cdn_network::{ 27 | CdnMetricsValue, KeyPair, ProductionDef, PushCdnNetwork, TestingDef, Topic as CdnTopic, 28 | WrappedSignatureKey, 29 | }, 30 | }; 31 | } 32 | -------------------------------------------------------------------------------- /crates/hotshot/src/traits/election/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! elections used for consensus 8 | 9 | /// leader completely randomized every view 10 | pub mod randomized_committee; 11 | 12 | /// quorum randomized every view, with configurable overlap 13 | pub mod randomized_committee_members; 14 | 15 | /// static (round robin) committee election 16 | pub mod static_committee; 17 | 18 | /// static (round robin leader for 2 consecutive views) committee election 19 | pub mod static_committee_leader_two_views; 20 | /// two static (round robin) committees for even and odd epochs 21 | pub mod two_static_committees; 22 | 23 | /// general helpers 24 | pub mod helpers; 25 | -------------------------------------------------------------------------------- /crates/hotshot/src/traits/networking.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Network access compatibility 8 | //! 9 | //! This module contains a trait abstracting over network access, as well as implementations of that 10 | //! trait. Currently this includes 11 | //! - [`MemoryNetwork`](memory_network::MemoryNetwork), an in memory testing-only implementation 12 | //! - [`Libp2pNetwork`](libp2p_network::Libp2pNetwork), a production-ready networking implementation built on top of libp2p-rs. 13 | 14 | pub mod combined_network; 15 | pub mod libp2p_network; 16 | pub mod memory_network; 17 | /// The Push CDN network 18 | pub mod push_cdn_network; 19 | 20 | pub use hotshot_types::traits::network::{NetworkError, NetworkReliability}; 21 | -------------------------------------------------------------------------------- /crates/hotshot/src/traits/node_implementation.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Composite trait for node behavior 8 | //! 9 | //! This module defines the [`NodeImplementation`] trait, which is a composite trait used for 10 | //! describing the overall behavior of a node, as a composition of implementations of the node trait. 11 | 12 | pub use hotshot_types::traits::node_implementation::{ 13 | NodeImplementation, TestableNodeImplementation, 14 | }; 15 | -------------------------------------------------------------------------------- /crates/hotshot/src/types.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | mod event; 8 | mod handle; 9 | 10 | pub use event::{Event, EventType}; 11 | pub use handle::SystemContextHandle; 12 | pub use hotshot_types::{ 13 | message::Message, 14 | signature_key::{BLSPrivKey, BLSPubKey}, 15 | traits::signature_key::SignatureKey, 16 | }; 17 | -------------------------------------------------------------------------------- /crates/hotshot/src/types/event.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Events that a [`SystemContext`](crate::SystemContext) instance can emit 8 | 9 | pub use hotshot_types::event::{Event, EventType}; 10 | -------------------------------------------------------------------------------- /crates/libp2p-networking/.cargo/config: -------------------------------------------------------------------------------- 1 | [net] 2 | git-fetch-with-cli = true -------------------------------------------------------------------------------- /crates/libp2p-networking/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /result 3 | /outfile_0 4 | /out*.txt 5 | -------------------------------------------------------------------------------- /crates/libp2p-networking/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | description = "Libp2p Networking Layer" 3 | name = "libp2p-networking" 4 | version = { workspace = true } 5 | edition = { workspace = true } 6 | authors = { workspace = true } 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [features] 10 | default = ["webui"] 11 | webui = [] 12 | 13 | [dev-dependencies] 14 | hotshot-example-types = { path = "../example-types" } 15 | 16 | [dependencies] 17 | anyhow = { workspace = true } 18 | async-lock = { workspace = true } 19 | async-trait = { workspace = true } 20 | bincode = { workspace = true } 21 | blake3 = { workspace = true } 22 | cbor4ii = { workspace = true } 23 | delegate = "0.13" 24 | derive_builder.workspace = true 25 | derive_more = { workspace = true } 26 | futures = { workspace = true } 27 | hotshot-types = { path = "../types" } 28 | lazy_static = { workspace = true } 29 | libp2p = { workspace = true, features = ["tokio"] } 30 | libp2p-identity = { workspace = true } 31 | libp2p-swarm-derive = { workspace = true } 32 | pin-project = "1" 33 | rand = { workspace = true } 34 | serde = { workspace = true } 35 | tokio = { workspace = true } 36 | tracing = { workspace = true } 37 | tracing-subscriber = { workspace = true } 38 | workspace-hack = { version = "0.1", path = "../workspace-hack" } 39 | 40 | [lints] 41 | workspace = true 42 | -------------------------------------------------------------------------------- /crates/libp2p-networking/flamegraph.sh: -------------------------------------------------------------------------------- 1 | sudo nix develop -c flamegraph -- $(fd -I "counter*" -t x | rg debug) test_request_response_one_round 2 | -------------------------------------------------------------------------------- /crates/libp2p-networking/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Library for p2p communication 8 | 9 | /// Network logic 10 | pub mod network; 11 | 12 | /// symbols needed to implement a networking instance over libp2p-netorking 13 | pub mod reexport { 14 | pub use libp2p::{request_response::ResponseChannel, Multiaddr}; 15 | pub use libp2p_identity::PeerId; 16 | } 17 | -------------------------------------------------------------------------------- /crates/libp2p-networking/src/network/behaviours/dht/store/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod persistent; 2 | pub mod validated; 3 | -------------------------------------------------------------------------------- /crates/libp2p-networking/src/network/behaviours/exponential_backoff.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use std::time::{Duration, Instant}; 8 | 9 | /// Track (with exponential backoff) 10 | /// sending of some sort of message 11 | #[derive(Debug, Clone, Eq, Hash, PartialEq)] 12 | pub struct ExponentialBackoff { 13 | /// Value to reset to when reset is called 14 | reset_val: Duration, 15 | /// factor to back off by 16 | backoff_factor: u32, 17 | /// the current timeout amount 18 | timeout: Duration, 19 | /// when we started the timeout 20 | started: Option, 21 | } 22 | 23 | impl ExponentialBackoff { 24 | /// Create new backoff 25 | #[must_use] 26 | pub fn new(backoff_factor: u32, next_timeout: Duration) -> Self { 27 | ExponentialBackoff { 28 | backoff_factor, 29 | timeout: next_timeout * backoff_factor, 30 | reset_val: next_timeout, 31 | started: None, 32 | } 33 | } 34 | 35 | /// reset backoff 36 | pub fn reset(&mut self) { 37 | self.timeout = self.reset_val; 38 | } 39 | 40 | /// start next timeout 41 | /// result: whether or not we succeeded 42 | /// if we succeeded, reset the timeout 43 | /// else increment the timeout by a factor 44 | /// of `timeout` 45 | pub fn start_next(&mut self, result: bool) { 46 | // success 47 | if result { 48 | self.timeout = self.reset_val; 49 | self.started = Some(Instant::now()); 50 | } 51 | // failure 52 | else { 53 | // note we want to prevent overflow. 54 | if let Some(r) = self.timeout.checked_mul(self.backoff_factor) { 55 | self.timeout = r; 56 | } 57 | self.started = Some(Instant::now()); 58 | } 59 | } 60 | 61 | /// Return the timeout duration and start the next timeout. 62 | pub fn next_timeout(&mut self, result: bool) -> Duration { 63 | let timeout = self.timeout; 64 | self.start_next(result); 65 | timeout 66 | } 67 | /// Whether or not the timeout is expired 68 | #[must_use] 69 | pub fn is_expired(&self) -> bool { 70 | if let Some(then) = self.started { 71 | then.elapsed() > self.timeout 72 | } else { 73 | true 74 | } 75 | } 76 | /// Marked as expired regardless of time left. 77 | pub fn expire(&mut self) { 78 | self.started = None; 79 | } 80 | } 81 | 82 | impl Default for ExponentialBackoff { 83 | fn default() -> Self { 84 | Self { 85 | reset_val: Duration::from_millis(500), 86 | backoff_factor: 2, 87 | timeout: Duration::from_millis(500), 88 | started: None, 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /crates/libp2p-networking/src/network/behaviours/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | /// Wrapper around `RequestResponse` 8 | pub mod direct_message; 9 | 10 | /// exponential backoff type 11 | pub mod exponential_backoff; 12 | 13 | /// Wrapper around Kademlia 14 | pub mod dht; 15 | -------------------------------------------------------------------------------- /crates/macros/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "hotshot-macros" 3 | version = { workspace = true } 4 | edition = { workspace = true } 5 | description = "Macros for hotshot tests" 6 | 7 | [dependencies] 8 | derive_builder.workspace = true 9 | proc-macro2 = "1" 10 | # proc macro stuff 11 | quote = "1" 12 | syn = { version = "2", features = ["full", "extra-traits"] } 13 | workspace-hack = { version = "0.1", path = "../workspace-hack" } 14 | 15 | [lib] 16 | proc-macro = true 17 | [lints] 18 | workspace = true 19 | -------------------------------------------------------------------------------- /crates/orchestrator/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "hotshot-orchestrator" 3 | version = { workspace = true } 4 | edition = { workspace = true } 5 | 6 | [dependencies] 7 | anyhow = { workspace = true } 8 | async-lock = { workspace = true } 9 | blake3 = { workspace = true } 10 | clap = { workspace = true } 11 | csv = "1" 12 | futures = { workspace = true } 13 | hotshot-types = { path = "../types" } 14 | libp2p-identity = { workspace = true } 15 | multiaddr = { workspace = true } 16 | serde = { workspace = true } 17 | surf-disco = { workspace = true } 18 | tide-disco = { workspace = true } 19 | tokio = { workspace = true } 20 | toml = { workspace = true } 21 | tracing = { workspace = true } 22 | vbs = { workspace = true } 23 | workspace-hack = { version = "0.1", path = "../workspace-hack" } 24 | 25 | [lints] 26 | workspace = true 27 | -------------------------------------------------------------------------------- /crates/orchestrator/README.md: -------------------------------------------------------------------------------- 1 | # Orchestrator 2 | 3 | This crate implements an orchestrator that coordinates starting the network with a particular configuration. It is useful for testing and benchmarking. Like the web server, the orchestrator is built using [Tide Disco](https://github.com/EspressoSystems/tide-disco). 4 | 5 | To run the orchestrator: `just example orchestrator http://0.0.0.0:3333 ./crates/orchestrator/run-config.toml` -------------------------------------------------------------------------------- /crates/orchestrator/staging-config.toml: -------------------------------------------------------------------------------- 1 | rounds = 10 2 | indexed_da = false 3 | transactions_per_round = 10 4 | manual_start_password = "tuktu6-tohnaX-gihxib" 5 | node_index = 0 6 | seed = [ 7 | 0, 8 | 0, 9 | 0, 10 | 0, 11 | 0, 12 | 0, 13 | 0, 14 | 0, 15 | 0, 16 | 0, 17 | 0, 18 | 0, 19 | 0, 20 | 0, 21 | 0, 22 | 0, 23 | 0, 24 | 0, 25 | 0, 26 | 0, 27 | 0, 28 | 0, 29 | 0, 30 | 0, 31 | 0, 32 | 0, 33 | 0, 34 | 0, 35 | 0, 36 | 0, 37 | 0, 38 | 0 39 | ] 40 | transaction_size = 100 41 | builder = "Simple" 42 | 43 | [config] 44 | start_threshold = [ 8, 10 ] 45 | num_nodes_with_stake = 10 46 | staked_da_nodes = 10 47 | fixed_leader_for_gpuvid = 1 48 | next_view_timeout = 15_000 49 | num_bootstrap = 5 50 | builder_urls = [ "https://builder.staging.testnet.espresso.network/" ] 51 | 52 | [config.view_sync_timeout] 53 | secs = 15 54 | nanos = 0 55 | 56 | [config.builder_timeout] 57 | secs = 8 58 | nanos = 0 59 | 60 | [config.data_request_delay] 61 | secs = 5 62 | nanos = 0 63 | 64 | [config.upgrade] 65 | start_proposing_view = 1 66 | stop_proposing_view = 0 67 | start_voting_view = 1 68 | stop_voting_view = 0 69 | start_proposing_time = 1 70 | stop_proposing_time = 0 71 | start_voting_time = 1 72 | stop_voting_time = 0 73 | 74 | [combined_network_config.delay_duration] 75 | secs = 5 76 | nanos = 0 77 | -------------------------------------------------------------------------------- /crates/request-response/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "request-response" 3 | version.workspace = true 4 | authors.workspace = true 5 | edition.workspace = true 6 | rust-version.workspace = true 7 | homepage.workspace = true 8 | documentation.workspace = true 9 | repository.workspace = true 10 | 11 | [lints] 12 | workspace = true 13 | 14 | [dev-dependencies] 15 | serde.workspace = true 16 | 17 | [dependencies] 18 | anyhow.workspace = true 19 | async-trait.workspace = true 20 | hotshot-types = { path = "../types" } 21 | byteorder = { version = "1", default-features = false } 22 | bincode.workspace = true 23 | blake3.workspace = true 24 | rand.workspace = true 25 | tokio.workspace = true 26 | parking_lot.workspace = true 27 | derive_more.workspace = true 28 | tracing.workspace = true 29 | async-broadcast.workspace = true 30 | derive_builder.workspace = true 31 | thiserror.workspace = true 32 | tokio-util = { version = "0.7", default-features = false, features = ["rt"] } -------------------------------------------------------------------------------- /crates/request-response/src/data_source.rs: -------------------------------------------------------------------------------- 1 | //! This file contains the [`DataSource`] trait. This trait allows the [`RequestResponseProtocol`] 2 | //! to calculate/derive a response for a specific request. In the confirmation layer the implementer 3 | //! would be something like a [`FeeMerkleTree`] for fee catchup 4 | 5 | use anyhow::Result; 6 | use async_trait::async_trait; 7 | 8 | use super::request::Request; 9 | 10 | /// The trait that allows the [`RequestResponseProtocol`] to calculate/derive a response for a specific request 11 | #[async_trait] 12 | pub trait DataSource: Send + Sync + 'static + Clone { 13 | /// Calculate/derive the response for a specific request 14 | async fn derive_response_for(&self, request: &R) -> Result; 15 | } 16 | -------------------------------------------------------------------------------- /crates/request-response/src/network.rs: -------------------------------------------------------------------------------- 1 | //! This file contains the [`Sender`] and [`Receiver`] traits. These traits are **used** by the 2 | //! [`RequestResponseProtocol`] to send and receive messages from a network or other source. 3 | //! 4 | //! For HotShot I've gone ahead and done a blanket implementation for a [`Sender`] for all 5 | //! [`ConnectedNetwork`]s. The reason it's not done for the [`Receiver`] is because both 6 | //! HS and the confirmation layer will receive messages from a single point and _then_ decide 7 | //! what to do with them (as opposed to having some sort of filtering mechanism). So for 8 | //! [`Receiver`] I've done a blanket implementation for channels that send [`Vec`]s. 9 | 10 | use std::{ops::Deref, sync::Arc}; 11 | 12 | use anyhow::{Context, Result}; 13 | use async_trait::async_trait; 14 | use hotshot_types::traits::{network::ConnectedNetwork, signature_key::SignatureKey}; 15 | use tokio::sync::mpsc; 16 | 17 | /// A type alias for a shareable byte array 18 | pub type Bytes = Arc>; 19 | 20 | /// The [`Sender`] trait is used to allow the [`RequestResponseProtocol`] to send messages to a specific recipient 21 | #[async_trait] 22 | pub trait Sender: Send + Sync + 'static + Clone { 23 | /// Send a message to the specified recipient 24 | async fn send_message(&self, message: &Bytes, recipient: K) -> Result<()>; 25 | } 26 | 27 | /// The [`Receiver`] trait is used to allow the [`RequestResponseProtocol`] to receive messages from a network 28 | /// or other source. 29 | #[async_trait] 30 | pub trait Receiver: Send + Sync + 'static { 31 | /// Receive a message. Returning an error here means the receiver will _NEVER_ receive any more messages 32 | async fn receive_message(&mut self) -> Result; 33 | } 34 | 35 | /// A blanket implementation of the [`Sender`] trait for all types that dereference to [`ConnectedNetwork`] 36 | #[async_trait] 37 | impl Sender for T 38 | where 39 | T: Deref> + Send + Sync + 'static + Clone, 40 | K: SignatureKey + 'static, 41 | { 42 | async fn send_message(&self, message: &Bytes, recipient: K) -> Result<()> { 43 | // Just send the message to the recipient 44 | self.direct_message(message.to_vec(), recipient) 45 | .await 46 | .with_context(|| "failed to send message") 47 | } 48 | } 49 | 50 | /// An implementation of the [`Receiver`] trait for the [`mpsc::Receiver`] type. Allows us to send messages 51 | /// to a channel and have the protocol receive them. 52 | #[async_trait] 53 | impl Receiver for mpsc::Receiver { 54 | async fn receive_message(&mut self) -> Result { 55 | // Just receive a message from the channel 56 | self.recv().await.ok_or(anyhow::anyhow!("channel closed")) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /crates/request-response/src/recipient_source.rs: -------------------------------------------------------------------------------- 1 | use async_trait::async_trait; 2 | use hotshot_types::traits::signature_key::SignatureKey; 3 | 4 | use super::request::Request; 5 | 6 | /// A trait that allows the [`RequestResponseProtocol`] to get the recipients that a specific message should 7 | /// expect responses from. In `HotShot` this would go on top of the [`Membership`] trait and determine 8 | /// which nodes are able (quorum/DA) to respond to which requests 9 | #[async_trait] 10 | pub trait RecipientSource: Send + Sync + 'static { 11 | /// Get all the recipients that the specific request should expect responses from 12 | async fn get_recipients_for(&self, request: &R) -> Vec; 13 | } 14 | -------------------------------------------------------------------------------- /crates/request-response/src/request.rs: -------------------------------------------------------------------------------- 1 | //! This file contains the [`Request`] and [`Response`] traits. Any upstream 2 | //! that wants to use the [`RequestResponseProtocol`] needs to implement these 3 | //! traits for their specific types. 4 | 5 | use std::fmt::Debug; 6 | 7 | use anyhow::Result; 8 | use async_trait::async_trait; 9 | 10 | use super::Serializable; 11 | 12 | /// A trait for a request. Associates itself with a response type. 13 | #[async_trait] 14 | pub trait Request: Send + Sync + Serializable + 'static + Clone + Debug { 15 | /// The response type associated with this request 16 | type Response: Response; 17 | 18 | /// Validate the request, returning an error if it is not valid 19 | /// 20 | /// # Errors 21 | /// If the request is not valid 22 | async fn validate(&self) -> Result<()>; 23 | } 24 | 25 | /// A trait that a response needs to implement 26 | #[async_trait] 27 | pub trait Response: 28 | Send + Sync + Serializable + Clone + Debug + PartialEq + Eq 29 | { 30 | /// Validate the response, making sure it is valid for the given request 31 | /// 32 | /// # Errors 33 | /// If the response is not valid for the given request 34 | async fn validate(&self, request: &R) -> Result<()>; 35 | } 36 | -------------------------------------------------------------------------------- /crates/request-response/src/util.rs: -------------------------------------------------------------------------------- 1 | use std::collections::VecDeque; 2 | 3 | /// A [`VecDeque`] with a maximum size 4 | pub struct BoundedVecDeque { 5 | /// The inner [`VecDeque`] 6 | inner: VecDeque, 7 | /// The maximum size of the [`VecDeque`] 8 | max_size: usize, 9 | } 10 | 11 | impl BoundedVecDeque { 12 | /// Create a new bounded [`VecDeque`] with the given maximum size 13 | pub fn new(max_size: usize) -> Self { 14 | Self { 15 | inner: VecDeque::new(), 16 | max_size, 17 | } 18 | } 19 | 20 | /// Push an item into the bounded [`VecDeque`], removing the oldest item if the 21 | /// maximum size is reached 22 | pub fn push(&mut self, item: T) { 23 | if self.inner.len() >= self.max_size { 24 | self.inner.pop_front(); 25 | } 26 | self.inner.push_back(item); 27 | } 28 | } 29 | 30 | #[cfg(test)] 31 | mod tests { 32 | use super::*; 33 | 34 | #[test] 35 | fn test_bounded_vec_deque() { 36 | let mut deque = BoundedVecDeque::new(3); 37 | deque.push(1); 38 | deque.push(2); 39 | deque.push(3); 40 | deque.push(4); 41 | deque.push(5); 42 | assert_eq!(deque.inner.len(), 3); 43 | assert_eq!(deque.inner, vec![3, 4, 5]); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /crates/task-impls/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = { workspace = true } 3 | description = "Async task implementations for consensus" 4 | edition = { workspace = true } 5 | name = "hotshot-task-impls" 6 | version = { workspace = true } 7 | 8 | [features] 9 | example-upgrade = [] 10 | rewind = [] 11 | 12 | [dependencies] 13 | anyhow = { workspace = true } 14 | async-broadcast = { workspace = true } 15 | async-lock = { workspace = true } 16 | async-trait = { workspace = true } 17 | bincode = { workspace = true } 18 | chrono = { workspace = true } 19 | committable = { workspace = true } 20 | either = { workspace = true } 21 | futures = { workspace = true } 22 | hotshot-builder-api = { path = "../builder-api" } 23 | hotshot-task = { path = "../task" } 24 | hotshot-types = { path = "../types" } 25 | jf-vid = { workspace = true } 26 | lru = { workspace = true } 27 | rand = { workspace = true } 28 | serde = { workspace = true } 29 | sha2 = { workspace = true } 30 | surf-disco = { workspace = true } 31 | tagged-base64 = { workspace = true } 32 | thiserror = { workspace = true } 33 | time = { workspace = true } 34 | tokio = { workspace = true } 35 | tracing = { workspace = true } 36 | url = { workspace = true } 37 | utils = { path = "../utils" } 38 | vbs = { workspace = true } 39 | vec1 = { workspace = true } 40 | workspace-hack = { version = "0.1", path = "../workspace-hack" } 41 | 42 | [lints] 43 | workspace = true 44 | -------------------------------------------------------------------------------- /crates/task-impls/HotShot_event_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/crates/task-impls/HotShot_event_architecture.png -------------------------------------------------------------------------------- /crates/task-impls/README.md: -------------------------------------------------------------------------------- 1 | HotShot uses an event-based architecture. This architecture is made of 4 main tasks: Network Task, View Sync Task, Consensus Task, and DA Task. The Network Task handles all incoming and outgoing messages. It forwards incoming messages to the correct task and listens for outgoing messages from the other tasks. The View Sync Task coordinates the view sync protocol. It listens for timeout events from the Consensus Task. Once a certain threshold of timeouts seen has been reached, the View Sync Task starts the View Sync protocol to bring the network back into agreement on which view it should be in. The Consensus Task handles the core HotShot consensus logic. It manages replicas that listen for quorum proposals and vote on them, leaders who send quorum proposals, and next leaders who listen for quorum votes and form QCs. The DA task handles the data availability protocol of HotShot. It listens for DA proposals, sends DA proposals, and forms a Data Availability Certificate (DAC) 2 | 3 | A diagram of how events interact with each task is below: 4 | ![HotShot Event Architecture](HotShot_event_architecture.png) 5 | 6 | For more information about each event see `./src/events.rs` 7 | -------------------------------------------------------------------------------- /crates/task-impls/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! The consensus layer for hotshot. This currently implements sequencing 8 | //! consensus in an event driven way 9 | 10 | /// The task which implements the core state logic of consensus. 11 | pub mod consensus; 12 | 13 | /// The task which handles the logic for the quorum vote. 14 | pub mod quorum_vote; 15 | 16 | /// The task which implements the main parts of data availability. 17 | pub mod da; 18 | 19 | /// The task which implements all transaction handling 20 | pub mod transactions; 21 | 22 | /// Defines the events passed between tasks 23 | pub mod events; 24 | 25 | /// The task which implements the network. 26 | pub mod network; 27 | 28 | /// Defines the types to run unit tests for a task. 29 | pub mod harness; 30 | 31 | /// The task which implements view synchronization 32 | pub mod view_sync; 33 | 34 | /// The task which implements verifiable information dispersal 35 | pub mod vid; 36 | 37 | /// Generic task for collecting votes 38 | pub mod vote_collection; 39 | 40 | /// Task for handling upgrades 41 | pub mod upgrade; 42 | 43 | /// Implementations for builder client 44 | /// Should contain builder task in the future 45 | pub mod builder; 46 | 47 | /// Helper functions used by any task 48 | pub mod helpers; 49 | 50 | /// Task which responses to requests from the network 51 | pub mod response; 52 | 53 | /// Task for requesting the network for things 54 | pub mod request; 55 | 56 | /// Task for handling logic for quorum proposals 57 | pub mod quorum_proposal; 58 | 59 | /// Task for handling QuorumProposalRecv events 60 | pub mod quorum_proposal_recv; 61 | 62 | /// Task for storing and replaying all received tasks by a node 63 | pub mod rewind; 64 | -------------------------------------------------------------------------------- /crates/task-impls/src/rewind.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use std::{fs::OpenOptions, io::Write, sync::Arc}; 8 | 9 | use async_broadcast::{Receiver, Sender}; 10 | use async_trait::async_trait; 11 | use hotshot_task::task::TaskState; 12 | use hotshot_types::traits::node_implementation::NodeType; 13 | use utils::anytrace::Result; 14 | 15 | use crate::events::HotShotEvent; 16 | 17 | /// The task state for the `Rewind` task is used to capture all events received 18 | /// by a particular node, in the order they've been received. 19 | pub struct RewindTaskState { 20 | /// All events received by this node since the beginning of time. 21 | pub events: Vec>>, 22 | 23 | /// The id of this node 24 | pub id: u64, 25 | } 26 | 27 | impl RewindTaskState { 28 | /// Handles all events, storing them to the private state 29 | pub fn handle(&mut self, event: &Arc>) { 30 | self.events.push(Arc::clone(event)); 31 | } 32 | } 33 | 34 | #[async_trait] 35 | impl TaskState for RewindTaskState { 36 | type Event = HotShotEvent; 37 | 38 | async fn handle_event( 39 | &mut self, 40 | event: Arc, 41 | _sender: &Sender>, 42 | _receiver: &Receiver>, 43 | ) -> Result<()> { 44 | self.handle(&event); 45 | Ok(()) 46 | } 47 | 48 | fn cancel_subtasks(&mut self) { 49 | tracing::info!("Node ID {} Recording {} events", self.id, self.events.len()); 50 | let filename = format!("rewind_{}.log", self.id); 51 | let mut file = match OpenOptions::new() 52 | .write(true) 53 | .create(true) 54 | .truncate(true) 55 | .open(&filename) 56 | { 57 | Ok(file) => file, 58 | Err(e) => { 59 | tracing::error!("Failed to write file {}; error = {}", filename, e); 60 | return; 61 | } 62 | }; 63 | 64 | for (event_number, event) in self.events.iter().enumerate() { 65 | // We do not want to die here, so we log and move on capturing as many events as we can. 66 | if let Err(e) = writeln!(file, "{event_number}: {event}") { 67 | tracing::error!( 68 | "Failed to write event number {event_number} and event {event}; error = {e}" 69 | ); 70 | } 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /crates/task/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = { workspace = true } 3 | name = "hotshot-task" 4 | version = { workspace = true } 5 | edition = { workspace = true } 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | async-broadcast = { workspace = true } 10 | async-trait = { workspace = true } 11 | futures = { workspace = true } 12 | tokio = { workspace = true, features = [ 13 | "time", 14 | "rt-multi-thread", 15 | "macros", 16 | "sync", 17 | ] } 18 | tracing = { workspace = true } 19 | utils = { path = "../utils" } 20 | workspace-hack = { version = "0.1", path = "../workspace-hack" } 21 | 22 | [lints] 23 | workspace = true 24 | -------------------------------------------------------------------------------- /crates/task/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Task primitives for `HotShot` 8 | 9 | /// Simple Dependency types 10 | pub mod dependency; 11 | /// Task which can uses dependencies 12 | pub mod dependency_task; 13 | /// Basic task types 14 | pub mod task; 15 | -------------------------------------------------------------------------------- /crates/testing/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /out*.txt 3 | -------------------------------------------------------------------------------- /crates/testing/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "hotshot-testing" 3 | version = { workspace = true } 4 | edition = { workspace = true } 5 | description = "Types and traits for the HotShot consesus module" 6 | authors = { workspace = true } 7 | 8 | [features] 9 | default = [] 10 | # NOTE this is used to activate the slow tests we don't wish to run in CI 11 | slow-tests = [] 12 | rewind = ["hotshot/rewind"] 13 | broken_3_chain_fixed = [] 14 | 15 | [dependencies] 16 | anyhow = { workspace = true } 17 | async-broadcast = { workspace = true } 18 | async-lock = { workspace = true } 19 | async-trait = { workspace = true } 20 | automod = "1.0.14" 21 | bitvec = { workspace = true } 22 | committable = { workspace = true } 23 | either = { workspace = true } 24 | futures = { workspace = true } 25 | hotshot = { path = "../hotshot", features = ["hotshot-testing"] } 26 | hotshot-builder-api = { path = "../builder-api" } 27 | hotshot-example-types = { path = "../example-types" } 28 | hotshot-fakeapi = { path = "../fakeapi" } 29 | hotshot-macros = { path = "../macros" } 30 | hotshot-task = { path = "../task" } 31 | hotshot-task-impls = { path = "../task-impls", version = "0.5.36", default-features = false } 32 | hotshot-types = { path = "../types" } 33 | itertools = "0.14.0" 34 | jf-vid = { workspace = true } 35 | lru = { workspace = true } 36 | portpicker = { workspace = true } 37 | primitive-types = { workspace = true } 38 | rand = { workspace = true } 39 | reqwest = { workspace = true } 40 | serde = { workspace = true } 41 | sha2 = { workspace = true } 42 | tagged-base64 = { workspace = true } 43 | thiserror = { workspace = true } 44 | tide-disco = { workspace = true } 45 | tokio = { workspace = true } 46 | tracing = { workspace = true } 47 | url = { workspace = true } 48 | vbs = { workspace = true } 49 | vec1 = { workspace = true } 50 | workspace-hack = { version = "0.1", path = "../workspace-hack" } 51 | -------------------------------------------------------------------------------- /crates/testing/README.md: -------------------------------------------------------------------------------- 1 | # Purpose 2 | 3 | Infrastructure and integration tests for hotshot. Since a lot of our tests can take a while to run, they've been split into groups to allow for parallelization in CI. 4 | 5 | # Usage 6 | 7 | The overall control flow is: 8 | 9 | ```ignore 10 | TestBuilder::default().build() -> TestLauncher::launch() -> TestRunner::execute() 11 | | | | 12 | - easy override setup fn - more explicit overrides - executes the test 13 | | | for networking, storage, 14 | - easy override correctness fn | hooks/overrides etc 15 | | 16 | - easily add in hooks 17 | | 18 | - easily override launching 19 | ``` 20 | 21 | Easily overriding setup/correctness checks/hooks and launching is all done by anonymous functions. Fairly sane and configurable setup and correct check functions may be generated from the round builder. The intended workflow should look like: 22 | 23 | ```rust 24 | use std::sync::Arc; 25 | use futures::FutureExt; 26 | use hotshot_example_types::test_types::StaticNodeImplType; 27 | use hotshot_example_types::round::RoundHook; 28 | use hotshot_example_types::test_types::StaticCommitteeTestTypes; 29 | use hotshot_example_types::test_builder::TestBuilder; 30 | use hotshot_example_types::test_builder::TestMetadata; 31 | 32 | async { 33 | // specify general characteristics of the test in TestMetadata 34 | let metadata = TestMetadata { 35 | total_nodes: 10, 36 | start_nodes: 10, 37 | num_succeeds: 5, 38 | failure_threshold: 10, 39 | ..Default::default() 40 | }; 41 | 42 | // construct the builder 43 | let mut test_builder = TestBuilder { 44 | metadata, 45 | /// we could build a check 46 | check: None, 47 | /// or a round setup if we want 48 | setup: None 49 | }; 50 | 51 | // construct the launcher 52 | // this may be used to manually override any of the round functions 53 | let test_launcher = test_builder.build::(); 54 | 55 | /// now let's add in a custom hook to print some debugging information at the beginning 56 | /// of each view 57 | let hook = 58 | RoundHook(Arc::new(move |_runner, ctx| { 59 | async move { 60 | tracing::error!("Context for this view is {:#?}", ctx); 61 | Ok(()) 62 | } 63 | .boxed_local() 64 | })); 65 | 66 | /// add the hook, launch the test, then run it. 67 | test_launcher.push_hook(hook).launch().run_test().await.unwrap(); 68 | 69 | }; 70 | ``` 71 | 72 | See TODO for examples. 73 | -------------------------------------------------------------------------------- /crates/testing/src/byzantine/mod.rs: -------------------------------------------------------------------------------- 1 | /// Byzantine definitions and implementations of different behaviours 2 | pub mod byzantine_behaviour; 3 | -------------------------------------------------------------------------------- /crates/testing/src/completion_task.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use std::time::Duration; 8 | 9 | use async_broadcast::{Receiver, Sender}; 10 | use hotshot_task_impls::helpers::broadcast_event; 11 | use tokio::{spawn, task::JoinHandle, time::timeout}; 12 | 13 | use crate::test_task::TestEvent; 14 | 15 | /// Completion task state 16 | pub struct CompletionTask { 17 | pub tx: Sender, 18 | 19 | pub rx: Receiver, 20 | /// Duration of the task. 21 | pub duration: Duration, 22 | } 23 | 24 | impl CompletionTask { 25 | pub fn run(mut self) -> JoinHandle<()> { 26 | spawn(async move { 27 | if timeout(self.duration, self.wait_for_shutdown()) 28 | .await 29 | .is_err() 30 | { 31 | broadcast_event(TestEvent::Shutdown, &self.tx).await; 32 | } 33 | }) 34 | } 35 | async fn wait_for_shutdown(&mut self) { 36 | while let Ok(event) = self.rx.recv_direct().await { 37 | if matches!(event, TestEvent::Shutdown) { 38 | tracing::error!("Completion Task shutting down"); 39 | return; 40 | } 41 | } 42 | } 43 | } 44 | /// Description for a time-based completion task. 45 | #[derive(Clone, Debug)] 46 | pub struct TimeBasedCompletionTaskDescription { 47 | /// Duration of the task. 48 | pub duration: Duration, 49 | } 50 | 51 | /// Description for a completion task. 52 | #[derive(Clone, Debug)] 53 | pub enum CompletionTaskDescription { 54 | /// Time-based completion task. 55 | TimeBasedCompletionTaskBuilder(TimeBasedCompletionTaskDescription), 56 | } 57 | -------------------------------------------------------------------------------- /crates/testing/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Testing infrastructure for `HotShot` 8 | 9 | /// Helpers for initializing system context handle and building tasks. 10 | pub mod helpers; 11 | 12 | /// builder 13 | pub mod test_builder; 14 | 15 | /// launcher 16 | pub mod test_launcher; 17 | 18 | /// runner 19 | pub mod test_runner; 20 | 21 | /// task that's consuming events and asserting safety 22 | pub mod overall_safety_task; 23 | 24 | /// task that checks leaves received across all nodes from decide events for consistency 25 | pub mod consistency_task; 26 | 27 | /// task that's submitting transactions to the stream 28 | pub mod txn_task; 29 | 30 | /// task that decides when things are complete 31 | pub mod completion_task; 32 | 33 | /// task to spin nodes up and down 34 | pub mod spinning_task; 35 | 36 | /// the `TestTask` struct and associated trait/functions 37 | pub mod test_task; 38 | 39 | /// task for checking if view sync got activated 40 | pub mod view_sync_task; 41 | 42 | /// Test implementation of block builder 43 | pub mod block_builder; 44 | 45 | /// predicates to use in tests 46 | pub mod predicates; 47 | 48 | /// scripting harness for tests 49 | pub mod script; 50 | 51 | /// view generator for tests 52 | pub mod view_generator; 53 | 54 | /// byzantine framework for tests 55 | pub mod byzantine; 56 | -------------------------------------------------------------------------------- /crates/testing/src/node_ctx.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use std::{collections::HashMap, sync::Arc}; 8 | 9 | use hotshot::{traits::TestableNodeImplementation, HotShotError}; 10 | use hotshot_types::traits::node_implementation::NodeType; 11 | 12 | /// context for a round 13 | // TODO eventually we want these to just be futures 14 | // that we poll when things are event driven 15 | // this context will be passed around 16 | #[derive(Debug, Clone)] 17 | pub struct NodeCtx> { 18 | /// results from previous rounds 19 | pub round_results: HashMap>, 20 | } 21 | 22 | impl> Default 23 | for NodeCtx 24 | { 25 | fn default() -> Self { 26 | Self { 27 | round_results: Default::default(), 28 | } 29 | } 30 | } 31 | 32 | /// Status of a view. 33 | #[derive(Debug, Clone)] 34 | pub enum ViewStatus> { 35 | /// The view is in progress. 36 | InProgress(InProgress), 37 | /// The view is failed. 38 | ViewFailed(ViewFailed), 39 | /// The view is a success. 40 | ViewSuccess(ViewSuccess), 41 | } 42 | 43 | /// In-progress status of a view. 44 | #[derive(Debug, Clone)] 45 | pub struct InProgress {} 46 | 47 | /// Failed status of a view. 48 | #[derive(Debug, Clone)] 49 | pub struct ViewFailed(pub Arc>); 50 | 51 | /// Success status of a view. 52 | #[derive(Debug, Clone)] 53 | pub struct ViewSuccess { 54 | /// state after decide event 55 | pub agreed_state: (), 56 | 57 | /// block after decide event 58 | pub agreed_block: LeafBlockPayload>, 59 | 60 | /// leaf after decide event 61 | pub agreed_leaf: Leaf, 62 | } 63 | -------------------------------------------------------------------------------- /crates/testing/src/predicates/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | pub mod event; 8 | pub mod upgrade_with_proposal; 9 | pub mod upgrade_with_vote; 10 | 11 | use async_trait::async_trait; 12 | 13 | #[derive(Eq, PartialEq, Copy, Clone, Debug)] 14 | pub enum PredicateResult { 15 | Pass, 16 | 17 | Fail, 18 | 19 | Incomplete, 20 | } 21 | 22 | impl From for PredicateResult { 23 | fn from(boolean: bool) -> Self { 24 | match boolean { 25 | true => PredicateResult::Pass, 26 | false => PredicateResult::Fail, 27 | } 28 | } 29 | } 30 | 31 | #[async_trait] 32 | pub trait Predicate: std::fmt::Debug { 33 | async fn evaluate(&self, input: &INPUT) -> PredicateResult; 34 | async fn info(&self) -> String; 35 | } 36 | -------------------------------------------------------------------------------- /crates/testing/src/predicates/upgrade_with_proposal.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use std::sync::Arc; 8 | 9 | use async_trait::async_trait; 10 | use hotshot_example_types::node_types::{MemoryImpl, TestTypes, TestVersions}; 11 | use hotshot_task_impls::quorum_proposal::QuorumProposalTaskState; 12 | use hotshot_types::simple_certificate::UpgradeCertificate; 13 | 14 | use crate::predicates::{Predicate, PredicateResult}; 15 | 16 | type QuorumProposalTaskTestState = QuorumProposalTaskState; 17 | 18 | type UpgradeCertCallback = 19 | Arc>>) -> bool + Send + Sync>; 20 | 21 | pub struct UpgradeCertPredicate { 22 | check: UpgradeCertCallback, 23 | info: String, 24 | } 25 | 26 | impl std::fmt::Debug for UpgradeCertPredicate { 27 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 28 | write!(f, "{}", self.info) 29 | } 30 | } 31 | 32 | #[async_trait] 33 | impl Predicate for UpgradeCertPredicate { 34 | async fn evaluate(&self, input: &QuorumProposalTaskTestState) -> PredicateResult { 35 | let upgrade_cert = input 36 | .upgrade_lock 37 | .decided_upgrade_certificate 38 | .read() 39 | .await 40 | .clone(); 41 | PredicateResult::from((self.check)(upgrade_cert.into())) 42 | } 43 | 44 | async fn info(&self) -> String { 45 | self.info.clone() 46 | } 47 | } 48 | 49 | pub fn no_decided_upgrade_certificate() -> Box { 50 | let info = "expected decided_upgrade_certificate to be None".to_string(); 51 | let check: UpgradeCertCallback = Arc::new(move |s| s.is_none()); 52 | Box::new(UpgradeCertPredicate { info, check }) 53 | } 54 | 55 | pub fn decided_upgrade_certificate() -> Box { 56 | let info = "expected decided_upgrade_certificate to be Some(_)".to_string(); 57 | let check: UpgradeCertCallback = Arc::new(move |s| s.is_some()); 58 | Box::new(UpgradeCertPredicate { info, check }) 59 | } 60 | -------------------------------------------------------------------------------- /crates/testing/src/predicates/upgrade_with_vote.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use std::sync::Arc; 8 | 9 | use async_trait::async_trait; 10 | use hotshot_example_types::node_types::{MemoryImpl, TestTypes, TestVersions}; 11 | use hotshot_task_impls::quorum_vote::QuorumVoteTaskState; 12 | use hotshot_types::simple_certificate::UpgradeCertificate; 13 | 14 | use crate::predicates::{Predicate, PredicateResult}; 15 | type QuorumVoteTaskTestState = QuorumVoteTaskState; 16 | 17 | type UpgradeCertCallback = 18 | Arc>>) -> bool + Send + Sync>; 19 | 20 | pub struct UpgradeCertPredicate { 21 | check: UpgradeCertCallback, 22 | info: String, 23 | } 24 | 25 | impl std::fmt::Debug for UpgradeCertPredicate { 26 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 27 | write!(f, "{}", self.info) 28 | } 29 | } 30 | 31 | #[async_trait] 32 | impl Predicate for UpgradeCertPredicate { 33 | async fn evaluate(&self, input: &QuorumVoteTaskTestState) -> PredicateResult { 34 | let upgrade_cert = input 35 | .upgrade_lock 36 | .decided_upgrade_certificate 37 | .read() 38 | .await 39 | .clone(); 40 | PredicateResult::from((self.check)(upgrade_cert.into())) 41 | } 42 | 43 | async fn info(&self) -> String { 44 | self.info.clone() 45 | } 46 | } 47 | 48 | pub fn no_decided_upgrade_certificate() -> Box { 49 | let info = "expected decided_upgrade_certificate to be None".to_string(); 50 | let check: UpgradeCertCallback = Arc::new(move |s| s.is_none()); 51 | Box::new(UpgradeCertPredicate { info, check }) 52 | } 53 | 54 | pub fn decided_upgrade_certificate() -> Box { 55 | let info = "expected decided_upgrade_certificate to be Some(_)".to_string(); 56 | let check: UpgradeCertCallback = Arc::new(move |s| s.is_some()); 57 | Box::new(UpgradeCertPredicate { info, check }) 58 | } 59 | -------------------------------------------------------------------------------- /crates/testing/src/test_helpers.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use committable::Committable; 8 | use hotshot_example_types::{node_types::TestTypes, state_types::TestValidatedState}; 9 | use hotshot_types::{ 10 | data::Leaf, 11 | utils::{View, ViewInner}, 12 | }; 13 | /// This function will create a fake [`View`] from a provided [`Leaf`]. 14 | pub fn create_fake_view_with_leaf(leaf: Leaf) -> View { 15 | create_fake_view_with_leaf_and_state(leaf, TestValidatedState::default()) 16 | } 17 | 18 | /// This function will create a fake [`View`] from a provided [`Leaf`] and `state`. 19 | pub fn create_fake_view_with_leaf_and_state( 20 | leaf: Leaf, 21 | state: TestValidatedState, 22 | ) -> View { 23 | View { 24 | view_inner: ViewInner::Leaf { 25 | leaf: leaf.commit(), 26 | state: state.into(), 27 | delta: None, 28 | }, 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_1.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | mod tests_1 { 8 | automod::dir!("tests/tests_1"); 9 | } 10 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_1/gen_key_pair.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | #![allow(clippy::panic)] 8 | 9 | #[cfg(test)] 10 | mod tests { 11 | use core::panic; 12 | use std::{env, fs::File, io::prelude::*}; 13 | 14 | use hotshot::types::{BLSPubKey, SignatureKey}; 15 | use hotshot_types::{validator_config::ValidatorConfigFile, ValidatorConfig}; 16 | #[test] 17 | fn gen_key_pair_gen_from_config_file() { 18 | let config_file = ValidatorConfigFile::from_file("config/ValidatorConfigFile.toml"); 19 | let my_own_validator_config = ValidatorConfig::::from(config_file.clone()); 20 | if config_file.seed == [0u8; 32] && config_file.node_id == 0 { 21 | assert_eq!( 22 | my_own_validator_config.public_key, 23 | ::from_private(&my_own_validator_config.private_key) 24 | ); 25 | } 26 | 27 | let current_working_dir = match env::current_dir() { 28 | Ok(dir) => dir, 29 | Err(e) => { 30 | panic!("get_current_working_dir error: {:?}", e); 31 | } 32 | }; 33 | let filename = current_working_dir.into_os_string().into_string().unwrap() 34 | + "/../../config/ValidatorConfigOutput"; 35 | match File::create(filename) { 36 | Err(why) => panic!("couldn't create file for output key pairs: {}", why), 37 | Ok(mut file) => match write!(file, "{my_own_validator_config:?}",) { 38 | Err(why) => panic!("couldn't generate key pairs and write to the file: {}", why), 39 | Ok(()) => println!("successfully wrote to file for output key pairs"), 40 | }, 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_1/transaction_task.rs: -------------------------------------------------------------------------------- 1 | use hotshot::tasks::task_state::CreateTaskState; 2 | use hotshot_example_types::{ 3 | block_types::TestMetadata, 4 | node_types::{MemoryImpl, TestConsecutiveLeaderTypes, TestVersions}, 5 | }; 6 | use hotshot_task_impls::{ 7 | events::HotShotEvent, harness::run_harness, transactions::TransactionTaskState, 8 | }; 9 | use hotshot_testing::helpers::build_system_handle; 10 | use hotshot_types::{ 11 | data::{null_block, EpochNumber, PackedBundle, ViewNumber}, 12 | traits::{ 13 | election::Membership, 14 | node_implementation::{ConsensusTime, Versions}, 15 | }, 16 | }; 17 | use vbs::version::StaticVersionType; 18 | 19 | #[cfg(test)] 20 | #[tokio::test(flavor = "multi_thread")] 21 | async fn test_transaction_task_leader_two_views_in_a_row() { 22 | hotshot::helpers::initialize_logging(); 23 | 24 | // Build the API for node 2. 25 | let node_id = 2; 26 | let handle = 27 | build_system_handle::(node_id) 28 | .await 29 | .0; 30 | 31 | let mut input = Vec::new(); 32 | let mut output = Vec::new(); 33 | 34 | let current_view = ViewNumber::new(4); 35 | input.push(HotShotEvent::ViewChange( 36 | current_view, 37 | Some(EpochNumber::new(1)), 38 | )); 39 | input.push(HotShotEvent::ViewChange( 40 | current_view + 1, 41 | Some(EpochNumber::new(1)), 42 | )); 43 | input.push(HotShotEvent::Shutdown); 44 | 45 | // current view 46 | let mut exp_packed_bundle = PackedBundle::new( 47 | vec![].into(), 48 | TestMetadata { 49 | num_transactions: 0, 50 | }, 51 | current_view, 52 | Some(EpochNumber::new(1)), 53 | vec1::vec1![ 54 | null_block::builder_fee::( 55 | handle 56 | .hotshot 57 | .memberships 58 | .read() 59 | .await 60 | .total_nodes(Some(EpochNumber::new(0))), 61 | ::Base::VERSION, 62 | *ViewNumber::new(4), 63 | ) 64 | .unwrap() 65 | ], 66 | None, 67 | ); 68 | output.push(HotShotEvent::BlockRecv(exp_packed_bundle.clone())); 69 | 70 | // next view 71 | exp_packed_bundle.view_number = current_view + 1; 72 | output.push(HotShotEvent::BlockRecv(exp_packed_bundle)); 73 | 74 | let transaction_state = 75 | TransactionTaskState::::create_from( 76 | &handle, 77 | ) 78 | .await; 79 | run_harness(input, output, transaction_state, false).await; 80 | } 81 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_1/view_sync_task.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use hotshot::tasks::task_state::CreateTaskState; 8 | use hotshot_example_types::node_types::{MemoryImpl, TestTypes, TestVersions}; 9 | use hotshot_task_impls::{ 10 | events::HotShotEvent, harness::run_harness, view_sync::ViewSyncTaskState, 11 | }; 12 | use hotshot_testing::helpers::build_system_handle; 13 | use hotshot_types::{ 14 | data::ViewNumber, simple_vote::ViewSyncPreCommitData2, 15 | traits::node_implementation::ConsensusTime, 16 | }; 17 | 18 | #[cfg(test)] 19 | #[tokio::test(flavor = "multi_thread")] 20 | async fn test_view_sync_task() { 21 | hotshot::helpers::initialize_logging(); 22 | 23 | // Build the API for node 5. 24 | let handle = build_system_handle::(5) 25 | .await 26 | .0; 27 | 28 | let vote_data = ViewSyncPreCommitData2 { 29 | relay: 0, 30 | round: ::View::new(4), 31 | epoch: None, 32 | }; 33 | let vote = hotshot_types::simple_vote::ViewSyncPreCommitVote2::::create_signed_vote( 34 | vote_data, 35 | ::View::new(4), 36 | hotshot_types::traits::consensus_api::ConsensusApi::public_key(&handle), 37 | hotshot_types::traits::consensus_api::ConsensusApi::private_key(&handle), 38 | &handle.hotshot.upgrade_lock, 39 | ) 40 | .await 41 | .expect("Failed to create a ViewSyncPreCommitVote!"); 42 | 43 | tracing::error!("Vote in test is {:?}", vote.clone()); 44 | 45 | let mut input = Vec::new(); 46 | let mut output = Vec::new(); 47 | 48 | input.push(HotShotEvent::Timeout(ViewNumber::new(2), None)); 49 | input.push(HotShotEvent::Timeout(ViewNumber::new(3), None)); 50 | 51 | input.push(HotShotEvent::Shutdown); 52 | 53 | output.push(HotShotEvent::ViewChange(ViewNumber::new(3), None)); 54 | output.push(HotShotEvent::ViewSyncPreCommitVoteSend(vote.clone())); 55 | 56 | let view_sync_state = ViewSyncTaskState::::create_from(&handle).await; 57 | run_harness(input, output, view_sync_state, false).await; 58 | } 59 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_2.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | mod tests_2 { 8 | automod::dir!("tests/tests_2"); 9 | } 10 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_3.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | mod tests_3 { 8 | automod::dir!("tests/tests_3"); 9 | } 10 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_3/test_with_failures_half_f.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use hotshot_example_types::{ 8 | node_types::{Libp2pImpl, MemoryImpl, PushCdnImpl, TestVersions}, 9 | state_types::TestTypes, 10 | }; 11 | use hotshot_macros::cross_tests; 12 | use hotshot_testing::{ 13 | block_builder::SimpleBuilderImplementation, 14 | spinning_task::{ChangeNode, NodeAction, SpinningTaskDescription}, 15 | test_builder::TestDescription, 16 | }; 17 | // Test f/2 nodes leaving the network. 18 | cross_tests!( 19 | TestName: test_with_failures_half_f, 20 | Impls: [MemoryImpl, Libp2pImpl, PushCdnImpl], 21 | Types: [TestTypes], 22 | Versions: [TestVersions], 23 | Ignore: false, 24 | Metadata: { 25 | let mut metadata = TestDescription::default_more_nodes(); 26 | metadata.test_config.epoch_height = 0; 27 | metadata.test_config.num_bootstrap = 17; 28 | // The first 14 (i.e., 20 - f) nodes are in the DA committee and we may shutdown the 29 | // remaining 6 (i.e., f) nodes. We could remove this restriction after fixing the 30 | // following issue. 31 | let dead_nodes = vec![ 32 | ChangeNode { 33 | idx: 17, 34 | updown: NodeAction::Down, 35 | }, 36 | ChangeNode { 37 | idx: 18, 38 | updown: NodeAction::Down, 39 | }, 40 | ChangeNode { 41 | idx: 19, 42 | updown: NodeAction::Down, 43 | }, 44 | ]; 45 | 46 | metadata.spinning_properties = SpinningTaskDescription { 47 | node_changes: vec![(5, dead_nodes)] 48 | }; 49 | 50 | metadata.overall_safety_properties.num_failed_views = 3; 51 | // Make sure we keep committing rounds after the bad leaders, but not the full 50 because of the numerous timeouts 52 | metadata.overall_safety_properties.num_successful_views = 22; 53 | metadata 54 | } 55 | ); 56 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_4.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | mod tests_4 { 8 | automod::dir!("tests/tests_4"); 9 | } 10 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_4/byzantine_tests.rs: -------------------------------------------------------------------------------- 1 | // use std::{collections::HashMap, rc::Rc, time::Duration}; 2 | 3 | // use hotshot_example_types::{ 4 | // node_types::{Libp2pImpl, MarketplaceTestVersions, MemoryImpl, PushCdnImpl}, 5 | // state_types::TestTypes, 6 | // }; 7 | // use hotshot_macros::cross_tests; 8 | // use hotshot_testing::{ 9 | // block_builder::SimpleBuilderImplementation, 10 | // byzantine::byzantine_behaviour::ViewDelay, 11 | // completion_task::{CompletionTaskDescription, TimeBasedCompletionTaskDescription}, 12 | // test_builder::{Behaviour, TestDescription}, 13 | // }; 14 | // use hotshot_types::{data::ViewNumber, traits::node_implementation::ConsensusTime}; 15 | 16 | // cross_tests!( 17 | // TestName: view_delay, 18 | // Impls: [MemoryImpl, Libp2pImpl, PushCdnImpl], 19 | // Types: [TestTypes], 20 | // Versions: [MarketplaceTestVersions], 21 | // Ignore: false, 22 | // Metadata: { 23 | 24 | // let behaviour = Rc::new(|node_id| { 25 | // let view_delay = ViewDelay { 26 | // number_of_views_to_delay: node_id/3, 27 | // events_for_view: HashMap::new(), 28 | // stop_view_delay_at_view_number: 25, 29 | // }; 30 | // match node_id { 31 | // 6|10|14 => Behaviour::Byzantine(Box::new(view_delay)), 32 | // _ => Behaviour::Standard, 33 | // } 34 | // }); 35 | 36 | // let mut metadata = TestDescription { 37 | // // allow more time to pass in CI 38 | // completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( 39 | // TimeBasedCompletionTaskDescription { 40 | // duration: Duration::from_secs(60), 41 | // }, 42 | // ), 43 | // behaviour, 44 | // epoch_height: 0, 45 | // ..TestDescription::default() 46 | // }; 47 | 48 | // let num_nodes_with_stake = 15; 49 | // metadata.num_nodes_with_stake = num_nodes_with_stake; 50 | // metadata.da_staked_committee_size = num_nodes_with_stake; 51 | // metadata.overall_safety_properties.num_failed_views = 20; 52 | // metadata.overall_safety_properties.num_successful_views = 20; 53 | // metadata.overall_safety_properties.expected_views_to_fail = HashMap::from([ 54 | // (ViewNumber::new(6), false), 55 | // (ViewNumber::new(10), false), 56 | // (ViewNumber::new(14), false), 57 | // (ViewNumber::new(21), false), 58 | // (ViewNumber::new(25), false), 59 | // ]); 60 | // metadata 61 | // }, 62 | // ); 63 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_4/test_with_failures_f.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use hotshot_example_types::{ 8 | node_types::{Libp2pImpl, MemoryImpl, PushCdnImpl, TestVersions}, 9 | state_types::TestTypes, 10 | }; 11 | use hotshot_macros::cross_tests; 12 | use hotshot_testing::{ 13 | block_builder::SimpleBuilderImplementation, 14 | spinning_task::{ChangeNode, NodeAction, SpinningTaskDescription}, 15 | test_builder::TestDescription, 16 | }; 17 | // Test f nodes leaving the network. 18 | cross_tests!( 19 | TestName: test_with_failures_f, 20 | Impls: [MemoryImpl, Libp2pImpl, PushCdnImpl], 21 | Types: [TestTypes], 22 | Versions: [TestVersions], 23 | Ignore: false, 24 | Metadata: { 25 | let mut metadata = TestDescription::default_more_nodes(); 26 | metadata.test_config.epoch_height = 0; 27 | metadata.overall_safety_properties.num_failed_views = 6; 28 | // Make sure we keep committing rounds after the bad leaders, but not the full 50 because of the numerous timeouts 29 | metadata.overall_safety_properties.num_successful_views = 20; 30 | metadata.test_config.num_bootstrap = 14; 31 | // The first 14 (i.e., 20 - f) nodes are in the DA committee and we may shutdown the 32 | // remaining 6 (i.e., f) nodes. We could remove this restriction after fixing the 33 | // following issue. 34 | let dead_nodes = vec![ 35 | ChangeNode { 36 | idx: 14, 37 | updown: NodeAction::Down, 38 | }, 39 | ChangeNode { 40 | idx: 15, 41 | updown: NodeAction::Down, 42 | }, 43 | ChangeNode { 44 | idx: 16, 45 | updown: NodeAction::Down, 46 | }, 47 | ChangeNode { 48 | idx: 17, 49 | updown: NodeAction::Down, 50 | }, 51 | ChangeNode { 52 | idx: 18, 53 | updown: NodeAction::Down, 54 | }, 55 | ChangeNode { 56 | idx: 19, 57 | updown: NodeAction::Down, 58 | }, 59 | ]; 60 | 61 | metadata.spinning_properties = SpinningTaskDescription { 62 | node_changes: vec![(5, dead_nodes)] 63 | }; 64 | 65 | metadata 66 | } 67 | ); 68 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_5.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | mod tests_5 { 8 | automod::dir!("tests/tests_5"); 9 | } 10 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_5/broken_3_chain.rs: -------------------------------------------------------------------------------- 1 | #![cfg(feature = "broken_3_chain_fixed")] 2 | use std::time::Duration; 3 | 4 | use hotshot_example_types::node_types::{PushCdnImpl, TestTypes}; 5 | use hotshot_testing::{ 6 | block_builder::SimpleBuilderImplementation, 7 | completion_task::{CompletionTaskDescription, TimeBasedCompletionTaskDescription}, 8 | overall_safety_task::OverallSafetyPropertiesDescription, 9 | spinning_task::{ChangeNode, SpinningTaskDescription, UpDown}, 10 | test_builder::{TestDescription, TimingData}, 11 | }; 12 | use tracing::instrument; 13 | 14 | /// Broken 3-chain test 15 | 16 | #[tokio::test(flavor = "multi_thread")] 17 | #[instrument] 18 | async fn broken_3_chain() { 19 | hotshot::helpers::initialize_logging(); 20 | 21 | let mut metadata: TestDescription = TestDescription { 22 | overall_safety_properties: OverallSafetyPropertiesDescription { 23 | check_leaf: true, 24 | ..Default::default() 25 | }, 26 | completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( 27 | TimeBasedCompletionTaskDescription { 28 | duration: Duration::from_secs(240), 29 | }, 30 | ), 31 | timing_data: TimingData { 32 | next_view_timeout: 4000, 33 | ..Default::default() 34 | }, 35 | ..TestDescription::default_multiple_rounds() 36 | }; 37 | 38 | let dead_nodes = vec![ 39 | ChangeNode { 40 | idx: 3, 41 | updown: UpDown::NetworkDown, 42 | }, 43 | ChangeNode { 44 | idx: 6, 45 | updown: UpDown::NetworkDown, 46 | }, 47 | ChangeNode { 48 | idx: 9, 49 | updown: UpDown::NetworkDown, 50 | }, 51 | ]; 52 | 53 | metadata.spinning_properties = SpinningTaskDescription { 54 | node_changes: vec![(3, dead_nodes)], 55 | }; 56 | metadata.num_nodes_with_stake = 10; 57 | metadata.da_staked_committee_size = 10; 58 | metadata.start_nodes = 10; 59 | metadata.overall_safety_properties.num_failed_views = 100; 60 | // Check whether we see at least 10 decides 61 | metadata.overall_safety_properties.num_successful_views = 10; 62 | 63 | metadata 64 | .gen_launcher(0) 65 | .launch() 66 | .run_test::() 67 | .await; 68 | } 69 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_5/push_cdn.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use std::time::Duration; 8 | 9 | use hotshot_example_types::node_types::{PushCdnImpl, TestTypes, TestVersions}; 10 | use hotshot_testing::{ 11 | block_builder::SimpleBuilderImplementation, 12 | completion_task::{CompletionTaskDescription, TimeBasedCompletionTaskDescription}, 13 | overall_safety_task::OverallSafetyPropertiesDescription, 14 | test_builder::{TestDescription, TimingData}, 15 | }; 16 | use tracing::instrument; 17 | 18 | /// Push CDN network test 19 | 20 | #[tokio::test(flavor = "multi_thread")] 21 | #[instrument] 22 | async fn push_cdn_network() { 23 | hotshot::helpers::initialize_logging(); 24 | 25 | let mut metadata: TestDescription = TestDescription { 26 | timing_data: TimingData { 27 | next_view_timeout: 10_000, 28 | ..Default::default() 29 | }, 30 | overall_safety_properties: OverallSafetyPropertiesDescription { 31 | num_failed_views: 0, 32 | num_successful_views: 35, 33 | ..Default::default() 34 | }, 35 | completion_task_description: CompletionTaskDescription::TimeBasedCompletionTaskBuilder( 36 | TimeBasedCompletionTaskDescription { 37 | duration: Duration::from_secs(60), 38 | }, 39 | ), 40 | ..TestDescription::default() 41 | }; 42 | 43 | metadata.test_config.epoch_height = 0; 44 | 45 | metadata 46 | .gen_launcher() 47 | .launch() 48 | .run_test::() 49 | .await; 50 | } 51 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_5/test_with_failures.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use hotshot_example_types::{ 8 | node_types::{Libp2pImpl, MemoryImpl, PushCdnImpl, TestVersions}, 9 | state_types::TestTypes, 10 | }; 11 | use hotshot_macros::cross_tests; 12 | use hotshot_testing::{ 13 | block_builder::SimpleBuilderImplementation, 14 | spinning_task::{ChangeNode, NodeAction, SpinningTaskDescription}, 15 | test_builder::TestDescription, 16 | }; 17 | 18 | // Test one node leaving the network. 19 | cross_tests!( 20 | TestName: test_with_failures_one, 21 | Impls: [MemoryImpl, Libp2pImpl, PushCdnImpl], 22 | Types: [TestTypes], 23 | Versions: [TestVersions], 24 | Ignore: false, 25 | Metadata: { 26 | let mut metadata = TestDescription::default_more_nodes(); 27 | metadata.test_config.epoch_height = 0; 28 | metadata.test_config.num_bootstrap = 19; 29 | // The first 14 (i.e., 20 - f) nodes are in the DA committee and we may shutdown the 30 | // remaining 6 (i.e., f) nodes. We could remove this restriction after fixing the 31 | // following issue. 32 | let dead_nodes = vec![ChangeNode { 33 | idx: 19, 34 | updown: NodeAction::Down, 35 | }]; 36 | 37 | metadata.spinning_properties = SpinningTaskDescription { 38 | node_changes: vec![(5, dead_nodes)] 39 | }; 40 | metadata.overall_safety_properties.num_failed_views = 1; 41 | metadata.overall_safety_properties.num_successful_views = 25; 42 | metadata 43 | } 44 | ); 45 | -------------------------------------------------------------------------------- /crates/testing/tests/tests_6.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | mod tests_6 { 8 | automod::dir!("tests/tests_6"); 9 | } 10 | -------------------------------------------------------------------------------- /crates/types/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = ["Espresso Systems "] 3 | description = "Types and traits for the HotShot consesus module" 4 | edition = "2021" 5 | name = "hotshot-types" 6 | version = "0.1.11" 7 | 8 | [dependencies] 9 | anyhow = { workspace = true } 10 | ark-bn254 = { workspace = true } 11 | ark-ed-on-bn254 = { workspace = true } 12 | ark-ff = { workspace = true } 13 | ark-serialize = { workspace = true } 14 | ark-srs = { version = "0.3.1" } 15 | ark-std = { workspace = true } 16 | async-lock = { workspace = true } 17 | async-trait = { workspace = true } 18 | bincode = { workspace = true } 19 | bitvec = { workspace = true } 20 | blake3 = { workspace = true } 21 | clap = { workspace = true } 22 | committable = { workspace = true } 23 | derive_more = { workspace = true, features = ["debug"] } 24 | digest = { workspace = true, features = ["rand_core"] } 25 | displaydoc = { version = "0.2.5", default-features = false } 26 | dyn-clone = "1.0.17" 27 | either = { workspace = true } 28 | futures = { workspace = true, features = ["alloc"] } 29 | jf-pcs = { workspace = true } 30 | jf-signature = { workspace = true, features = ["bls", "schnorr"] } 31 | jf-utils = { workspace = true } 32 | jf-vid = { workspace = true } 33 | lazy_static = { workspace = true } 34 | libp2p-identity = { workspace = true } 35 | memoize = { workspace = true } 36 | mnemonic = "1" 37 | multiaddr = { workspace = true } 38 | primitive-types = { workspace = true } 39 | rand = { workspace = true } 40 | rand_chacha = { workspace = true } 41 | serde = { workspace = true } 42 | serde-inline-default = { workspace = true } 43 | serde_bytes = { workspace = true } 44 | serde_json = { workspace = true } 45 | sha2 = { workspace = true } 46 | tagged-base64 = { workspace = true } 47 | thiserror = { workspace = true } 48 | time = { workspace = true } 49 | tokio = { workspace = true } 50 | toml = { workspace = true } 51 | tracing = { workspace = true } 52 | typenum = { workspace = true } 53 | url = { workspace = true } 54 | utils = { path = "../utils" } 55 | vbs = { workspace = true } 56 | vec1 = { workspace = true } 57 | workspace-hack = { version = "0.1", path = "../workspace-hack" } 58 | 59 | [features] 60 | gpu-vid = ["jf-vid/gpu-vid"] 61 | test-srs = ["jf-vid/test-srs"] 62 | 63 | [lints] 64 | workspace = true 65 | -------------------------------------------------------------------------------- /crates/types/src/bundle.rs: -------------------------------------------------------------------------------- 1 | //! This module provides the `Bundle` type 2 | 3 | use serde::{Deserialize, Serialize}; 4 | 5 | use crate::traits::{ 6 | block_contents::BuilderFee, node_implementation::NodeType, signature_key::BuilderSignatureKey, 7 | BlockPayload, 8 | }; 9 | 10 | #[derive(Clone, Debug, Serialize, Deserialize)] 11 | #[serde(bound = "TYPES: NodeType")] 12 | /// The Bundle for a portion of a block, provided by a downstream 13 | /// builder that exists in a bundle auction. 14 | /// This type is maintained by HotShot 15 | pub struct Bundle { 16 | /// The bundle transactions sent by the builder. 17 | pub transactions: Vec<>::Transaction>, 18 | 19 | /// The signature over the bundle. 20 | pub signature: ::BuilderSignature, 21 | 22 | /// The fee for sequencing 23 | pub sequencing_fee: BuilderFee, 24 | } 25 | -------------------------------------------------------------------------------- /crates/types/src/error.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Error type for `HotShot` 8 | //! 9 | //! This module provides [`HotShotError`], which is an enum representing possible faults that can 10 | //! occur while interacting with this crate. 11 | 12 | use committable::Commitment; 13 | use serde::{Deserialize, Serialize}; 14 | use thiserror::Error; 15 | 16 | use crate::{data::Leaf2, traits::node_implementation::NodeType}; 17 | 18 | /// Error type for `HotShot` 19 | #[derive(Debug, Error)] 20 | #[non_exhaustive] 21 | pub enum HotShotError { 22 | /// The consensus state machine is in an invalid state 23 | #[error("Invalid state: {0}")] 24 | InvalidState(String), 25 | 26 | /// Leaf was not present in storage 27 | #[error("Missing leaf with commitment: {0}")] 28 | MissingLeaf(Commitment>), 29 | 30 | /// Failed to serialize data 31 | #[error("Failed to serialize: {0}")] 32 | FailedToSerialize(String), 33 | 34 | /// Failed to deserialize data 35 | #[error("Failed to deserialize: {0}")] 36 | FailedToDeserialize(String), 37 | 38 | /// The view timed out 39 | #[error("View {view_number} timed out: {state:?}")] 40 | ViewTimedOut { 41 | /// The view number that timed out 42 | view_number: TYPES::View, 43 | /// The state that the round was in when it timed out 44 | state: RoundTimedoutState, 45 | }, 46 | } 47 | 48 | /// Contains information about what the state of the hotshot-consensus was when a round timed out 49 | #[derive(Debug, Clone, Serialize, Deserialize)] 50 | #[non_exhaustive] 51 | pub enum RoundTimedoutState { 52 | /// Leader is in a Prepare phase and is waiting for a HighQc 53 | LeaderWaitingForHighQc, 54 | /// Leader is in a Prepare phase and timed out before the round min time is reached 55 | LeaderMinRoundTimeNotReached, 56 | /// Leader is waiting for prepare votes 57 | LeaderWaitingForPrepareVotes, 58 | /// Leader is waiting for precommit votes 59 | LeaderWaitingForPreCommitVotes, 60 | /// Leader is waiting for commit votes 61 | LeaderWaitingForCommitVotes, 62 | 63 | /// Replica is waiting for a prepare message 64 | ReplicaWaitingForPrepare, 65 | /// Replica is waiting for a pre-commit message 66 | ReplicaWaitingForPreCommit, 67 | /// Replica is waiting for a commit message 68 | ReplicaWaitingForCommit, 69 | /// Replica is waiting for a decide message 70 | ReplicaWaitingForDecide, 71 | 72 | /// HotShot-testing tried to collect round events, but it timed out 73 | TestCollectRoundEventsTimedOut, 74 | } 75 | -------------------------------------------------------------------------------- /crates/types/src/request_response.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Types for the request/response implementations. This module incorporates all 8 | //! of the shared types for all of the network backends. 9 | 10 | use committable::{Committable, RawCommitmentBuilder}; 11 | use serde::{Deserialize, Serialize}; 12 | 13 | use crate::traits::{node_implementation::NodeType, signature_key::SignatureKey}; 14 | 15 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] 16 | /// A signed request for a proposal. 17 | pub struct ProposalRequestPayload { 18 | /// The view number that we're requesting a proposal for. 19 | pub view_number: TYPES::View, 20 | 21 | /// Our public key. The ensures that the recipient can reply to 22 | /// us directly. 23 | pub key: TYPES::SignatureKey, 24 | } 25 | 26 | impl Committable for ProposalRequestPayload { 27 | fn commit(&self) -> committable::Commitment { 28 | RawCommitmentBuilder::new("signed proposal request commitment") 29 | .u64_field("view number", *self.view_number) 30 | .var_size_bytes(&self.key.to_bytes()) 31 | .finalize() 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /crates/types/src/stake_table.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Types and structs related to the stake table 8 | 9 | use primitive_types::U256; 10 | use serde::{Deserialize, Serialize}; 11 | 12 | use crate::traits::signature_key::{SignatureKey, StakeTableEntryType}; 13 | 14 | /// Stake table entry 15 | #[derive(Serialize, Deserialize, PartialEq, Debug, Clone, Hash, Eq)] 16 | #[serde(bound(deserialize = ""))] 17 | pub struct StakeTableEntry { 18 | /// The public key 19 | pub stake_key: K, 20 | /// The associated stake amount 21 | pub stake_amount: U256, 22 | } 23 | 24 | impl StakeTableEntryType for StakeTableEntry { 25 | /// Get the stake amount 26 | fn stake(&self) -> U256 { 27 | self.stake_amount 28 | } 29 | 30 | /// Get the public key 31 | fn public_key(&self) -> K { 32 | self.stake_key.clone() 33 | } 34 | } 35 | 36 | impl StakeTableEntry { 37 | /// Get the public key 38 | pub fn key(&self) -> &K { 39 | &self.stake_key 40 | } 41 | } 42 | 43 | // TODO(Chengyu): add stake table snapshot here 44 | -------------------------------------------------------------------------------- /crates/types/src/traits.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Common traits for the `HotShot` protocol 8 | pub mod auction_results_provider; 9 | pub mod block_contents; 10 | pub mod consensus_api; 11 | pub mod election; 12 | pub mod metrics; 13 | pub mod network; 14 | pub mod node_implementation; 15 | pub mod qc; 16 | pub mod signature_key; 17 | pub mod stake_table; 18 | pub mod states; 19 | pub mod storage; 20 | 21 | pub use block_contents::{BlockPayload, EncodeBytes}; 22 | pub use states::ValidatedState; 23 | -------------------------------------------------------------------------------- /crates/types/src/traits/auction_results_provider.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! This module defines the interaction layer with the Solver via the [`AuctionResultsProvider`] trait, 8 | //! which handles connecting to, and fetching the allocation results from, the Solver. 9 | 10 | use anyhow::Result; 11 | use async_trait::async_trait; 12 | 13 | use super::node_implementation::NodeType; 14 | 15 | /// The AuctionResultsProvider trait is the sole source of Solver-originated state and interaction, 16 | /// and returns the results of the Solver's allocation via the associated type. The associated type, 17 | /// `AuctionResult`, also implements the `HasUrls` trait, which requires that the output 18 | /// type has the requisite fields available. 19 | #[async_trait] 20 | pub trait AuctionResultsProvider: Send + Sync + Clone { 21 | /// Fetches the auction result for a view. Does not cache the result, 22 | /// subsequent calls will invoke additional wasted calls. 23 | async fn fetch_auction_result(&self, view_number: TYPES::View) -> Result; 24 | } 25 | -------------------------------------------------------------------------------- /crates/types/src/traits/consensus_api.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | //! Contains the [`ConsensusApi`] trait. 8 | 9 | use std::{num::NonZeroUsize, time::Duration}; 10 | 11 | use async_trait::async_trait; 12 | 13 | use crate::{ 14 | event::Event, 15 | traits::{ 16 | node_implementation::{NodeImplementation, NodeType}, 17 | signature_key::SignatureKey, 18 | }, 19 | }; 20 | 21 | /// The API that tasks use to talk to the system 22 | /// TODO we plan to drop this 23 | #[async_trait] 24 | pub trait ConsensusApi>: Send + Sync { 25 | /// Total number of nodes in the network. Also known as `n`. 26 | fn total_nodes(&self) -> NonZeroUsize; 27 | 28 | /// The maximum amount of time a leader can wait to get a block from a builder. 29 | fn builder_timeout(&self) -> Duration; 30 | 31 | /// Get a reference to the public key. 32 | fn public_key(&self) -> &TYPES::SignatureKey; 33 | 34 | /// Get a reference to the private key. 35 | fn private_key(&self) -> &::PrivateKey; 36 | 37 | /// Notify the system of an event within `hotshot-consensus`. 38 | async fn send_event(&self, event: Event); 39 | } 40 | -------------------------------------------------------------------------------- /crates/types/src/upgrade_config.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | /// Constants associated with the upgrade process. 8 | pub struct UpgradeConstants { 9 | /// The offset for how far in the future we will send out a `QuorumProposal` with an `UpgradeCertificate` we form. This is also how far in advance of sending a `QuorumProposal` we begin collecting votes on an `UpgradeProposal`. 10 | pub propose_offset: u64, 11 | 12 | /// The offset for how far in the future the upgrade certificate we attach should be decided on (or else discarded). 13 | pub decide_by_offset: u64, 14 | 15 | /// The offset for how far in the future the upgrade actually begins. 16 | pub begin_offset: u64, 17 | 18 | /// The offset for how far in the future the upgrade ends. 19 | pub finish_offset: u64, 20 | } 21 | 22 | #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] 23 | #[serde(bound(deserialize = ""))] 24 | /// Holds configuration for the upgrade task. 25 | pub struct UpgradeConfig { 26 | /// View to start proposing an upgrade 27 | pub start_proposing_view: u64, 28 | /// View to stop proposing an upgrade. To prevent proposing an upgrade, set stop_proposing_view <= start_proposing_view. 29 | pub stop_proposing_view: u64, 30 | /// View to start voting on an upgrade 31 | pub start_voting_view: u64, 32 | /// View to stop voting on an upgrade. To prevent voting on an upgrade, set stop_voting_view <= start_voting_view. 33 | pub stop_voting_view: u64, 34 | /// Unix time in seconds at which we start proposing an upgrade 35 | pub start_proposing_time: u64, 36 | /// Unix time in seconds at which we stop proposing an upgrade. To prevent proposing an upgrade, set stop_proposing_time <= start_proposing_time. 37 | pub stop_proposing_time: u64, 38 | /// Unix time in seconds at which we start voting on an upgrade 39 | pub start_voting_time: u64, 40 | /// Unix time in seconds at which we stop voting on an upgrade. To prevent voting on an upgrade, set stop_voting_time <= start_voting_time. 41 | pub stop_voting_time: u64, 42 | } 43 | 44 | // Explicitly implementing `Default` for clarity. 45 | #[allow(clippy::derivable_impls)] 46 | impl Default for UpgradeConfig { 47 | fn default() -> Self { 48 | UpgradeConfig { 49 | start_proposing_view: u64::MAX, 50 | stop_proposing_view: 0, 51 | start_voting_view: u64::MAX, 52 | stop_voting_view: 0, 53 | start_proposing_time: u64::MAX, 54 | stop_proposing_time: 0, 55 | start_voting_time: u64::MAX, 56 | stop_voting_time: 0, 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /crates/types/src/validator_config.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021-2024 Espresso Systems (espressosys.com) 2 | // This file is part of the HotShot repository. 3 | 4 | // You should have received a copy of the MIT License 5 | // along with the HotShot repository. If not, see . 6 | 7 | use std::{env, fs, path::PathBuf}; 8 | 9 | use toml; 10 | use tracing::error; 11 | 12 | use crate::{traits::signature_key::SignatureKey, ValidatorConfig}; 13 | 14 | /// Holds configuration for a validator node 15 | #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, Default)] 16 | #[serde(bound(deserialize = ""))] 17 | pub struct ValidatorConfigFile { 18 | /// The validator's seed 19 | pub seed: [u8; 32], 20 | /// The validator's index, which can be treated as another input to the seed 21 | pub node_id: u64, 22 | // The validator's stake, commented for now 23 | // pub stake_value: u64, 24 | /// Whether or not we are DA 25 | pub is_da: bool, 26 | } 27 | 28 | impl ValidatorConfigFile { 29 | /// read the validator config from a file 30 | /// # Panics 31 | /// Panics if unable to get the current working directory 32 | pub fn from_file(dir_str: &str) -> Self { 33 | let current_working_dir = match env::current_dir() { 34 | Ok(dir) => dir, 35 | Err(e) => { 36 | error!("get_current_working_dir error: {:?}", e); 37 | PathBuf::from("") 38 | } 39 | }; 40 | let filename = 41 | current_working_dir.into_os_string().into_string().unwrap() + "/../../" + dir_str; 42 | let contents = fs::read_to_string(filename.clone()).expect("Could not read file"); 43 | let data: ValidatorConfigFile = 44 | toml::from_str(&contents).expect("Unable to load data from file"); 45 | data 46 | } 47 | } 48 | 49 | impl From for ValidatorConfig { 50 | fn from(val: ValidatorConfigFile) -> Self { 51 | // here stake_value is set to 1, since we don't input stake_value from ValidatorConfigFile for now 52 | ValidatorConfig::generated_from_seed_indexed(val.seed, val.node_id, 1, val.is_da) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /crates/types/src/vid/advz.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/crates/types/src/vid/advz.rs -------------------------------------------------------------------------------- /crates/types/src/vid/avidm.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/crates/types/src/vid/avidm.rs -------------------------------------------------------------------------------- /crates/utils/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "utils" 3 | version = { workspace = true } 4 | edition = { workspace = true } 5 | description = "Utils" 6 | 7 | [dependencies] 8 | tracing = { workspace = true } 9 | workspace-hack = { version = "0.1", path = "../workspace-hack" } 10 | 11 | [lints] 12 | workspace = true 13 | -------------------------------------------------------------------------------- /crates/utils/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! General (not HotShot-specific) utilities 2 | 3 | /// Error utilities, intended to function as a replacement to `anyhow`. 4 | pub mod anytrace; 5 | -------------------------------------------------------------------------------- /crates/workspace-hack/.gitattributes: -------------------------------------------------------------------------------- 1 | # Avoid putting conflict markers in the generated Cargo.toml file, since their presence breaks 2 | # Cargo. 3 | # Also do not check out the file as CRLF on Windows, as that's what hakari needs. 4 | Cargo.toml merge=binary -crlf 5 | -------------------------------------------------------------------------------- /crates/workspace-hack/build.rs: -------------------------------------------------------------------------------- 1 | // A build script is required for cargo to consider build dependencies. 2 | fn main() {} 3 | -------------------------------------------------------------------------------- /crates/workspace-hack/src/lib.rs: -------------------------------------------------------------------------------- 1 | // This is a stub lib.rs. 2 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | (import 2 | ( 3 | let 4 | lock = builtins.fromJSON (builtins.readFile ./flake.lock); 5 | in 6 | fetchTarball { 7 | url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; 8 | sha256 = lock.nodes.flake-compat.locked.narHash; 9 | } 10 | ) 11 | { 12 | src = ./.; 13 | }).defaultNix.default 14 | -------------------------------------------------------------------------------- /docker/cdn-broker.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y curl libcurl4 wait-for-it tini \ 5 | && rm -rf /var/lib/apt/lists/* 6 | 7 | ARG TARGETARCH 8 | ARG ASYNC_EXECUTOR 9 | 10 | COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/cdn-broker /usr/local/bin/cdn-broker 11 | 12 | # logging 13 | ENV RUST_LOG="warn" 14 | 15 | # log format. JSON no ansi 16 | ENV RUST_LOG_FORMAT="json" 17 | 18 | ENTRYPOINT ["cdn-broker"] 19 | -------------------------------------------------------------------------------- /docker/cdn-marshal.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y curl libcurl4 wait-for-it tini \ 5 | && rm -rf /var/lib/apt/lists/* 6 | 7 | ARG TARGETARCH 8 | ARG ASYNC_EXECUTOR 9 | 10 | COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/cdn-marshal /usr/local/bin/cdn-marshal 11 | 12 | # logging 13 | ENV RUST_LOG="warn" 14 | 15 | # log format. JSON no ansi 16 | ENV RUST_LOG_FORMAT="json" 17 | 18 | ENTRYPOINT ["cdn-marshal"] 19 | -------------------------------------------------------------------------------- /docker/orchestrator.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y curl libcurl4 wait-for-it tini \ 5 | && rm -rf /var/lib/apt/lists/* 6 | 7 | ARG TARGETARCH 8 | ARG ASYNC_EXECUTOR 9 | 10 | COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/orchestrator /usr/local/bin/orchestrator 11 | 12 | # logging 13 | ENV RUST_LOG="warn" 14 | 15 | # log format. JSON no ansi 16 | ENV RUST_LOG_FORMAT="json" 17 | 18 | ENTRYPOINT ["orchestrator"] 19 | -------------------------------------------------------------------------------- /docker/validator-cdn-local.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y curl libcurl4 wait-for-it tini \ 5 | && rm -rf /var/lib/apt/lists/* 6 | 7 | 8 | COPY --chmod=0755 ./target/release-lto/examples/validator-push-cdn /usr/local/bin/validator-push-cdn 9 | 10 | # logging 11 | ENV RUST_LOG="warn" 12 | 13 | # log format. JSON no ansi 14 | ENV RUST_LOG_FORMAT="json" 15 | 16 | ENTRYPOINT ["validator-push-cdn"] 17 | -------------------------------------------------------------------------------- /docker/validator-cdn.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y curl libcurl4 wait-for-it tini \ 5 | && rm -rf /var/lib/apt/lists/* 6 | 7 | ARG TARGETARCH 8 | ARG ASYNC_EXECUTOR 9 | 10 | COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/validator-push-cdn /usr/local/bin/validator-push-cdn 11 | 12 | # logging 13 | ENV RUST_LOG="warn" 14 | 15 | # log format. JSON no ansi 16 | ENV RUST_LOG_FORMAT="json" 17 | 18 | ENTRYPOINT ["validator-push-cdn"] 19 | -------------------------------------------------------------------------------- /docker/validator-combined.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y curl libcurl4 wait-for-it tini \ 5 | && rm -rf /var/lib/apt/lists/* 6 | 7 | ARG TARGETARCH 8 | ARG ASYNC_EXECUTOR 9 | 10 | COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/validator-combined /usr/local/bin/validator-combined 11 | 12 | # logging 13 | ENV RUST_LOG="warn" 14 | 15 | # log format. JSON no ansi 16 | ENV RUST_LOG_FORMAT="json" 17 | 18 | ENTRYPOINT ["validator-combined"] 19 | -------------------------------------------------------------------------------- /docker/validator-libp2p.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:jammy 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y curl libcurl4 wait-for-it tini \ 5 | && rm -rf /var/lib/apt/lists/* 6 | 7 | ARG TARGETARCH 8 | ARG ASYNC_EXECUTOR 9 | 10 | COPY --chmod=0755 ./target/${ASYNC_EXECUTOR}/${TARGETARCH}/release/examples/validator-libp2p /usr/local/bin/validator-libp2p 11 | 12 | # logging 13 | ENV RUST_LOG="warn" 14 | 15 | # log format. JSON no ansi 16 | ENV RUST_LOG_FORMAT="json" 17 | 18 | ENTRYPOINT ["validator-libp2p"] 19 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | This directory contains documentation for the HotShot protocol. `/diagrams` contains state diagrams representing the code logic of HotShot. The implementation of this logic in the `hotshot` crate should match these diagrams. `espresso-sequencer-paper.pdf` is an academic description and analysis of the HotShot protocol. -------------------------------------------------------------------------------- /docs/diagrams/README.md: -------------------------------------------------------------------------------- 1 | To edit or view the source of these diagrams, open `HotShotFlow.drawio` using the draw.io application. Use the Template diagram for style guidance. If you edit the diagrams, be sure to replace the `png` images of the diagrams in the `/diagrams` directory. -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/BlockFromBuilderRecv.md: -------------------------------------------------------------------------------- 1 | # BlockPayloadFromBuilderRecv 2 | 3 | ![BlockPayloadFromBuilderRecv](/docs/diagrams/images/HotShotFlow-BlockPayloadFromBuilderRecv.drawio.png "BlockPayloadFromBuilderRecv") 4 | 5 | * This diagram assumes we trust the builder to give our node a valid block payload. In the future we can specify logic if we do not trust the builder (and therefore need some fair exchange of signatures in place) 6 | * The builder sends its data in two parts: the block payload and vid commitment, since the vid commitment can be costly for the builder to calculate. This task handles the former. -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/General.md: -------------------------------------------------------------------------------- 1 | ## General Notes 2 | * "Valid" certificates meet the following criteria: 3 | * Aggregated signature is valid against the data the certificate commits to 4 | * Aggregated signature represents a proper threshold for that certificate 5 | 6 | ## Data Structures 7 | ``` 8 | state_map { 9 | HashMap (VidCommitment -> Leaf) 10 | TODO 11 | } 12 | ``` 13 | 14 | ``` 15 | latest_known_view: u64 16 | ``` 17 | 18 | ``` 19 | latest_voted_view: u64 20 | ``` 21 | 22 | ``` 23 | latest_da_voted_view: u64 24 | ``` 25 | -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/OptimisticDACertificateRecv.md: -------------------------------------------------------------------------------- 1 | # OptimisticDaCertificateRecv 2 | 3 | ![OptimisticDaCertificateRecv](/docs/diagrams/images/HotShotFlow-OptimisticDaCertificateRecv.drawio.png "OptimisticDaCertificateRecv") 4 | 5 | ## Basic Message Validation 6 | 7 | ## Optimistic DA Certificate Processing 8 | 9 | -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/OptimisticDAProposalRecv.md: -------------------------------------------------------------------------------- 1 | # OptimisticDAProposalRecv 2 | 3 | ![OptimisticDAProposalRecv](/docs/diagrams/images/HotShotFlow-OptimisticDAProposalRecv.drawio.png "QuorumProposalRecv") 4 | 5 | ## Basic Message Validation 6 | * It is possible for some applications built on top of HotShot to listen for DA proposals but not vote on them (such as builders). 7 | 8 | ## DA Proposal Validation and Processing 9 | * "Valid" DA proposal data is a no-op for now. More validation is done in the `VoteOnQuorumProposal` task. 10 | -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/Timeout.md: -------------------------------------------------------------------------------- 1 | # Timeout 2 | 3 | ![Timeout](/docs/diagrams/images/HotShotFlow-Timeout.drawio.png "Timeout") 4 | 5 | * The Timeout event signals that the node has not seen evidence for the view referenced in the event within the expected time window. For example: if a node is in view 10 and does not receive evidence to change to view 11 within the expected time window, a `Timeout(11, _)` event will be created. 6 | * `evidenced` is a boolean value indicating whether the previous view had view change evidence. Using our example above, `Timeout(11, true)` means that the node saw valid evidence for view 10. This parameter is used to decide whether a node should send a timeout vote or trigger view sync. A timeout vote is sent upon the first view timeout after a successful view. View sync, on the other hand, is triggered on the second view timeout after a successful view. 7 | * It is important to update our `latest_voted_view` variable to indicate that the node should no longer vote for the previous view. This is to prevent conflicting certificates from forming in the case that nodes timeout, but receive a late `QuorumProposal` after the timeout. For safety, only one view change evidence certificate can be formed per view. 8 | -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/VIDDataFromBuilderRecv.md: -------------------------------------------------------------------------------- 1 | # VidDataFromBuilderRecv 2 | 3 | ![VidDataFromBuilderRecv](/docs/diagrams/images/HotShotFlow-VidDataFromBuilderRecv.drawio.png "VidDataFromBuilderRecv") 4 | 5 | * This diagram assumes we trust the builder to give our node a valid block payload. In the future we can specify logic if we do not trust the builder (and therefore need some fair exchange of signatures in place) 6 | * The builder sends its data in two parts: the block payload and vid commitment, since the vid commitment can be costly for the builder to calculate. This task handles the latter task. 7 | * In the current PBS design (which will become more holistic in the future) the builder is expected to calculate the vid commitment on the node's behalf. The node uses this commitment to then calculate the vid shares. -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/VIDShareRecv.md: -------------------------------------------------------------------------------- 1 | # VidShareRecv 2 | 3 | ![VidShareRecv](/docs/diagrams/images/HotShotFlow-VidShareRecv.drawio.png "VidShareRecv") 4 | 5 | ## Basic Message Validation 6 | 7 | ## VID Share Processing and Validation 8 | * A "valid" VID share is one that successfully passes the `VidScheme::verify_share` function 9 | 10 | -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/ViewChange.md: -------------------------------------------------------------------------------- 1 | # ViewChange 2 | 3 | ![ViewChange](/docs/diagrams/images/HotShotFlow-ViewChange.drawio.png "ViewChange") 4 | 5 | * This task updates the `latest_known_view` a node is aware of. It also handles canceling any in-progress tasks from previous views (such as timeout tasks waiting to publish a timeout event) 6 | * In the optimistic case, a leader will request their block from the builder 1 view ahead of their leader slot. However, it's possible to skip views due to a node catching up after being offline or the network participating in view sync. In this case, we allow the leader to request their block during their leader view. -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/ViewSyncCommitCertificateRecv.md: -------------------------------------------------------------------------------- 1 | # ViewSyncCommitCertificateRecv 2 | 3 | ![ViewSyncCommitCertificateRecv](/docs/diagrams/images/HotShotFlow-ViewSyncCommitCertificateRecv.drawio.png "ViewSyncCommitCertificateRecv") -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/ViewSyncFinalizeCertificateRecv.md: -------------------------------------------------------------------------------- 1 | # ViewSyncFinalizeCertificateRecv 2 | 3 | ![ViewSyncFinalizeCertificateRecv](/docs/diagrams/images/HotShotFlow-ViewSyncFinalizeCertificateRecv.drawio.png "ViewSyncFinalizeCertificateRecv") -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/ViewSyncPreCommitCertificateRecv.md: -------------------------------------------------------------------------------- 1 | # ViewSyncPreCommitCertificateRecv 2 | 3 | ![ViewSyncPreCommitCertificateRecv](/docs/diagrams/images/HotShotFlow-ViewSyncPreCommitCertificateRecv.drawio.png "ViewSyncPreCommitCertificateRecv") 4 | 5 | * A `ViewSyncPreCommitCertificate`, unlike most other certificates, only requires f + 1 votes to be valid. -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/ViewSyncTimeout.md: -------------------------------------------------------------------------------- 1 | # ViewSyncTimeout 2 | 3 | ![ViewSyncTimeout](/docs/diagrams/images/HotShotFlow-ViewSyncTimeout.drawio.png "ViewSyncTimeout") 4 | 5 | * The `ViewSyncTimeout` event is similar to the `Timeout` event, except that the former is used exclusively in the context of view sync. 6 | * When a view sync round times out a node will increment its `latest_known_relay` value by 1, and then resend its latest view sync vote. -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/ViewSyncTrigger.md: -------------------------------------------------------------------------------- 1 | # ViewSyncTrigger 2 | 3 | ![ViewSyncTrigger](/docs/diagrams/images/HotShotFlow-ViewSyncTrigger.drawio.png "ViewSyncTrigger") 4 | 5 | * The [NK20](https://arxiv.org/pdf/2002.07539.pdf) view synchronization protocol is triggered on the second view timeout after a successful view. This signals that the network was unable to form a timeout certificate and may be out of sync. 6 | -------------------------------------------------------------------------------- /docs/diagrams/event_discriptions/VoteOnQuorumProposal.md: -------------------------------------------------------------------------------- 1 | # VoteOnQuorumProposal 2 | 3 | ![VoteOnQuorumProposal](/docs/diagrams/images/HotShotFlow-VoteOnQuorumProposal.drawio.png "VoteOnQuorumProposal") 4 | 5 | ## Mutually Verify Share, Cert, and Proposal 6 | * Nodes must verify that the `OptimisticDaCertificate`, their `VidShare`, and the `QuorumProposal` all commit to the same data. It is only possible to do this verification once we've received all the necessary data. 7 | * The event/task dependency infrastructure should only spawn this task if the view for all 3 dependencies is the same. It is an internal error if this is not the case. 8 | -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-BlockPayloadFromBuilderRecv.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-BlockPayloadFromBuilderRecv.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-OptimisticDACertificateRecv.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-OptimisticDACertificateRecv.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-OptimisticDAProposalRecv.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-OptimisticDAProposalRecv.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-QuorumProposalRecv.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-QuorumProposalRecv.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-QuorumProposalSend.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-QuorumProposalSend.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-Timeout.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-Timeout.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-VIDDataFromBuilderRecv.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-VIDDataFromBuilderRecv.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-VIDShareRecv.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-VIDShareRecv.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-ViewChange.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-ViewChange.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-ViewSyncCommitCertificateRecv.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-ViewSyncCommitCertificateRecv.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-ViewSyncFinalizeCertificateRecv.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-ViewSyncFinalizeCertificateRecv.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-ViewSyncPreCommitCertificateRecv.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-ViewSyncPreCommitCertificateRecv.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-ViewSyncTimeout.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-ViewSyncTimeout.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-ViewSyncTrigger.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-ViewSyncTrigger.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-VoteOnQuorumProposal.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-VoteOnQuorumProposal.drawio.png -------------------------------------------------------------------------------- /docs/diagrams/images/HotShotFlow-VoteRecv.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/diagrams/images/HotShotFlow-VoteRecv.drawio.png -------------------------------------------------------------------------------- /docs/espresso-sequencer-paper.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EspressoSystems/HotShot/4706499649f020efccea5ffb1af5b0774000dde4/docs/espresso-sequencer-paper.pdf -------------------------------------------------------------------------------- /pull_request_template.md: -------------------------------------------------------------------------------- 1 | Closes # 2 | 3 | 4 | 5 | ### This PR: 6 | 7 | 8 | 9 | 10 | 11 | 12 | ### This PR does not: 13 | 14 | 15 | 16 | ### Key places to review: 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 28 | -------------------------------------------------------------------------------- /repl.nix: -------------------------------------------------------------------------------- 1 | # usage: nix repl repl.nix 2 | let 3 | flake = builtins.getFlake (toString ./.); 4 | in 5 | { 6 | inherit flake; 7 | self = flake.inputs.self; 8 | pkgs = flake.inputs.nixpkgs.legacyPackages; 9 | } 10 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "1.82" 3 | components = [ "cargo", "clippy", "rust-src", "rustc", "rustfmt", "llvm-tools-preview" ] 4 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | group_imports = "StdExternalCrate" 2 | imports_granularity = "Crate" 3 | edition = "2021" 4 | -------------------------------------------------------------------------------- /scripts/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.output 3 | benchmarks_results/results.csv -------------------------------------------------------------------------------- /scripts/auto-integration/.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | -------------------------------------------------------------------------------- /scripts/auto-integration/README.md: -------------------------------------------------------------------------------- 1 | # Auto-Integration 2 | 3 | To use this set up a virtualenv as 4 | ```shell 5 | python -m venv venv 6 | source venv/bin/activate 7 | pip install -r requirements.txt 8 | ``` 9 | 10 | ``` 11 | Help settings 12 | Usage: run-integration.py [OPTIONS] 13 | 14 | Watches the native demo and reports any HotShot failures. 15 | 16 | ╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ 17 | │ --view-threshold TEXT [default: 10000] │ 18 | │ --host-ip TEXT [default: localhost] │ 19 | │ --install-completion Install completion for the current shell. │ 20 | │ --show-completion Show completion for the current shell, to copy it or customize the installation. │ 21 | │ --help Show this message and exit. │ 22 | ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ 23 | ``` 24 | 25 | View threshold is the number of views that this will run for. 26 | -------------------------------------------------------------------------------- /scripts/auto-integration/requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2024.7.4 2 | charset-normalizer==3.3.2 3 | click==8.1.7 4 | idna==3.7 5 | markdown-it-py==3.0.0 6 | mdurl==0.1.2 7 | Pygments==2.18.0 8 | requests==2.32.3 9 | rich==13.7.1 10 | shellingham==1.5.4 11 | toml==0.10.2 12 | typer==0.12.3 13 | typing_extensions==4.12.0 14 | urllib3==2.2.2 15 | -------------------------------------------------------------------------------- /scripts/benchmark_scripts/benchmarks_start_cdn_broker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # make sure the following line is added to `~/.bashrc` 4 | source "$HOME/.cargo/env" 5 | 6 | 7 | if [ -z "$1" ]; then 8 | echo "No arguments provided. Usage: $0 " 9 | exit 1 10 | fi 11 | 12 | echo "Argument 1: $1" 13 | keydb_address="$1" 14 | 15 | just example cdn-broker -- -d $keydb_address \ 16 | --public-bind-endpoint 0.0.0.0:1740 \ 17 | --public-advertise-endpoint local_ip:1740 \ 18 | --private-bind-endpoint 0.0.0.0:1741 \ 19 | --private-advertise-endpoint local_ip:1741 & 20 | -------------------------------------------------------------------------------- /scripts/benchmark_scripts/benchmarks_start_leader_gpu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # make sure the following line is added to `~/.bashrc` 4 | source "$HOME/.cargo/env" 5 | 6 | # Check if at least two arguments are provided 7 | if [ $# -lt 2 ]; then 8 | echo "Usage: $0 " 9 | exit 1 10 | fi 11 | 12 | echo "Argument 1: $1" 13 | echo "Argument 2: $2" 14 | fixed_leader_for_gpuvid="$1" 15 | orchestrator_url="$2" 16 | 17 | just example_gpuvid_leader multi-validator-push-cdn -- $fixed_leader_for_gpuvid $orchestrator_url & 18 | 19 | -------------------------------------------------------------------------------- /scripts/benchmarks_results/README.md: -------------------------------------------------------------------------------- 1 | ## How to run the benchmarks 2 | 3 | - To run it locally, check out `crates/examples/push-cdn/README.md`. 4 | - To run it in AWS, take a look at `scripts/benchmark_scripts/aws_ecs_benchmarks_cdn.sh` and change value of parameters as you'd like, make sure you've installed everything needed in the script and have access to needed servers (and have hotshot on those servers), then start `key-db` in one `tmux` session, in another session, enter `HotShot`, run `./scripts/benchmark_scripts/aws_ecs_benchmarks_cdn.sh [YOUR_NAME] [REMOTE_BROKER_HOST_PUBLIC_IP_1] [REMOTE_BROKER_HOST_PUBLIC_IP_2] ...`. If you want to run leaders on GPU, for the last step, run `./scripts/benchmark_scripts/aws_ecs_benchmarks_cdn_gpu.sh [YOUR_NAME] [REMOTE_GPU_HOST] [REMOTE_BROKER_HOST_PUBLIC_IP_1] [REMOTE_BROKER_HOST_PUBLIC_IP_2] ...` instead (e.g. `./scripts/benchmark_scripts/aws_ecs_benchmarks_cdn_gpu.sh sishan 11.111.223.224 3.111.223.224`). 5 | - When running on a large group of nodes (1000 etc.), it might take too long for all nodes to post "start", you can use `manual_start` when you think there're enough nodes connected: 6 | ``` 7 | export ORCHESTRATOR_MANUAL_START_PASSWORD=password 8 | curl -X POST http://172.31.8.82:4444/v0/api/manual_start -d 'password' 9 | ``` 10 | 11 | ## How to view the results 12 | 13 | - Three ways to gather the results 14 | - (recommended) check out`scripts/benchmarks_results/results.csv` on EC2, where you should have organized overall results. 15 | - check out Datadog under `host:/hotshot/` where you have stats for each individual validator but it's hard to track since they’re distributed. 16 | - wait for the output of orchestrator in local terminal, where the results are not that organized if you do multiple runs, also hard to track. 17 | 18 | - Explanation on confusing arguments 19 | - `partial_results`: Whether the results are partially collected. It's set to "One" when the results are collected for one node; "HalfDA" when the results are collected for the number equals to DA_committee_number / 2; "Half" when the results are collected for half running nodes; "Full" if the results are successfully collected from all nodes. The reason is sometimes we'll get high throughput however not all the nodes can terminate successfully (I suspect the reason is that some of them fall behind when fetching large transactions). -------------------------------------------------------------------------------- /scripts/count_fds.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # USAGE: periodically print number of file descriptors in use 3 | # will work with gnu coreutils 4 | 5 | for i in {0..100000} 6 | do 7 | echo NUM FDS: $(lsof | grep -i "udp" | grep -i "libp2p" | tr -s ' ' | cut -d" " -f11 | sort | uniq | wc -l) 8 | sleep 0.2s 9 | done 10 | -------------------------------------------------------------------------------- /scripts/flakiness.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | N_ITERATIONS=100 4 | OUT_DIR="$(pwd)/flakiness" 5 | TEST_THREADS=1 6 | 7 | # Make script killable with Ctrl+C 8 | trap "echo; exit" INT 9 | 10 | die() { 11 | echo "${1}"; 12 | exit 1; 13 | } 14 | 15 | # Parse arguments 16 | while [ $# -gt 0 ]; do 17 | case $1 in 18 | -h|--help) 19 | echo "./flakiness -n [number of iterations] -o [output directory] -j [threads]" 20 | exit 0; 21 | ;; 22 | -n) 23 | N_ITERATIONS="$2" 24 | shift # past argument 25 | shift # past value 26 | ;; 27 | -o) 28 | OUT_DIR="$2" 29 | shift # past argument 30 | shift # past value 31 | ;; 32 | -j) 33 | TEST_THREADS="$2" 34 | shift # past argument 35 | shift # past value 36 | ;; 37 | -*) 38 | echo "Unknown option $1" 39 | exit 1 40 | ;; 41 | esac 42 | done 43 | 44 | # Don't clobber previous results 45 | if [ -d "${OUT_DIR}" ]; then 46 | if [ "$(ls -A "${OUT_DIR}")" ]; then 47 | die "${OUT_DIR} is not empty" 48 | fi 49 | fi 50 | 51 | # Check for nextest 52 | cargo nextest -V > /dev/null 2>&1 || die "cargo-nextest is not installed" 53 | 54 | # Workaround for nextest in nix shell 55 | DYLD_FALLBACK_LIBRARY_PATH=$(rustc --print sysroot)/lib 56 | export DYLD_FALLBACK_LIBRARY_PATH 57 | 58 | for ITERATION in $(seq -w "${N_ITERATIONS}"); 59 | do 60 | echo "Running iteration ${ITERATION}/${N_ITERATIONS}"; 61 | export CARGO_TARGET_DIR="target_dirs/nix_rustc" 62 | if ! cargo nextest run \ 63 | --no-fail-fast \ 64 | --failure-output final \ 65 | --test-threads "${TEST_THREADS}" \ 66 | --hide-progress-bar > "${OUT_DIR}/${ITERATION}.log" 2>&1; 67 | then 68 | echo "${ITERATION}" >> "${OUT_DIR}/failed-iterations" 69 | fi 70 | done 71 | -------------------------------------------------------------------------------- /scripts/nix_bump_pr_changes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Utility script to show the github commit diff when a `update_flake_lock_action` PR is made. 4 | # 5 | # To run, pipe the contents of the `Flake lock file updates:` into this file 6 | # 7 | # e.g. `cat updates.txt | ./scripts/nix_bump_pr_changes.py` 8 | # 9 | # The output of this script should be pasted as a reply to that PR 10 | 11 | import sys 12 | import re 13 | 14 | name_commit_regex = re.compile(r"'github:([^\/]+\/[^\/]+)\/([^']+)") 15 | prev = '' 16 | 17 | for line in sys.stdin: 18 | line = line.rstrip() 19 | if line.startswith(" 'github:"): 20 | prev = line 21 | if line.startswith(" → 'github:"): 22 | 23 | [_, repo, start_commit, _] = name_commit_regex.split(prev) 24 | [_, _, end_commit, _] = name_commit_regex.split(line) 25 | print("- [ ] " + repo + ": [repo](https://github.com/" + repo + ") | [commits this PR](https://github.com/" + repo + "/compare/" + start_commit + ".." + end_commit + ")") 26 | 27 | 28 | -------------------------------------------------------------------------------- /scripts/runfail.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Runs a command until it fails. 4 | # Useful for running overnight to see if tests don't fail sporadically. 5 | # 6 | # Usage: 7 | # `./scripts/runfail.sh cargo test --profile=release-lto --features=full-ci --manifest-path testing/Cargo.toml` 8 | 9 | while [ $? -eq 0 ]; do 10 | $@ 11 | done 12 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | (import 2 | ( 3 | let 4 | lock = builtins.fromJSON (builtins.readFile ./flake.lock); 5 | in 6 | fetchTarball { 7 | url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; 8 | sha256 = lock.nodes.flake-compat.locked.narHash; 9 | } 10 | ) 11 | { 12 | src = ./.; 13 | }).shellNix.default 14 | 15 | --------------------------------------------------------------------------------