├── .cargo └── audit.toml ├── .envrc ├── .github ├── CODEOWNERS ├── actions │ └── nix-common-setup │ │ └── action.yml ├── dependabot.yml └── workflows │ ├── audit.yml │ ├── nix.yml │ ├── python-scripts.yml │ ├── release.yml │ ├── test.yml │ └── update-flake-lock.yml ├── .gitignore ├── .pre-commit-config.yaml ├── Cargo.lock ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── catalyst-toolbox ├── Cargo.toml ├── build.rs ├── doc │ ├── ideascale_importer.md │ ├── private-key-and-qr-code.md │ ├── rewards.md │ └── tally-recovery.md ├── resources │ ├── explorer │ │ ├── schema.graphql │ │ └── transaction_by_id.graphql │ └── testing │ │ ├── reviews.csv │ │ ├── valid_assessments.csv │ │ └── veteran_reviews_count.csv ├── scripts │ └── python │ │ ├── csv_merger.py │ │ ├── proposers_rewards.py │ │ └── requirements.txt ├── src │ ├── archive.rs │ ├── bin │ │ ├── catalyst-toolbox.rs │ │ └── cli │ │ │ ├── advisor_reviews │ │ │ └── mod.rs │ │ │ ├── archive │ │ │ └── mod.rs │ │ │ ├── ideascale │ │ │ └── mod.rs │ │ │ ├── kedqr │ │ │ ├── decode │ │ │ │ ├── img.rs │ │ │ │ ├── mod.rs │ │ │ │ └── payload.rs │ │ │ ├── encode │ │ │ │ ├── img.rs │ │ │ │ ├── mod.rs │ │ │ │ └── payload.rs │ │ │ ├── info.rs │ │ │ ├── mod.rs │ │ │ └── verify.rs │ │ │ ├── logs │ │ │ ├── compare.rs │ │ │ ├── mod.rs │ │ │ └── sentry │ │ │ │ ├── download.rs │ │ │ │ ├── mod.rs │ │ │ │ └── stats.rs │ │ │ ├── mod.rs │ │ │ ├── notifications │ │ │ ├── api_params.rs │ │ │ ├── mod.rs │ │ │ └── send.rs │ │ │ ├── recovery │ │ │ ├── mod.rs │ │ │ ├── tally.rs │ │ │ └── votes.rs │ │ │ ├── rewards │ │ │ ├── community_advisors.rs │ │ │ ├── dreps.rs │ │ │ ├── full │ │ │ │ ├── config.rs │ │ │ │ └── mod.rs │ │ │ ├── mod.rs │ │ │ ├── proposers │ │ │ │ ├── mod.rs │ │ │ │ └── util.rs │ │ │ ├── veterans.rs │ │ │ └── voters.rs │ │ │ ├── snapshot │ │ │ └── mod.rs │ │ │ ├── stats │ │ │ ├── archive.rs │ │ │ ├── live.rs │ │ │ ├── mod.rs │ │ │ ├── snapshot.rs │ │ │ └── voters │ │ │ │ ├── active.rs │ │ │ │ ├── initials.rs │ │ │ │ └── mod.rs │ │ │ └── vote_check │ │ │ └── mod.rs │ ├── community_advisors │ │ ├── mod.rs │ │ └── models │ │ │ ├── de.rs │ │ │ └── mod.rs │ ├── http │ │ ├── mock.rs │ │ ├── mod.rs │ │ ├── rate_limit.rs │ │ └── reqwest.rs │ ├── ideascale │ │ ├── fetch.rs │ │ ├── mod.rs │ │ └── models │ │ │ ├── custom_fields.rs │ │ │ ├── de │ │ │ ├── ada_rewards.rs │ │ │ ├── approval.rs │ │ │ ├── challenge_title.rs │ │ │ ├── clean_string.rs │ │ │ └── mod.rs │ │ │ ├── mod.rs │ │ │ └── se.rs │ ├── kedqr │ │ ├── img.rs │ │ ├── mod.rs │ │ └── payload.rs │ ├── lib.rs │ ├── logs │ │ ├── compare.rs │ │ ├── mod.rs │ │ └── sentry.rs │ ├── notifications │ │ ├── mod.rs │ │ ├── requests │ │ │ ├── create_message.rs │ │ │ └── mod.rs │ │ ├── responses │ │ │ ├── create_message.rs │ │ │ └── mod.rs │ │ └── send.rs │ ├── recovery │ │ ├── mod.rs │ │ ├── replay.rs │ │ └── tally.rs │ ├── rewards │ │ ├── community_advisors │ │ │ ├── funding.rs │ │ │ ├── lottery.rs │ │ │ └── mod.rs │ │ ├── dreps.rs │ │ ├── mod.rs │ │ ├── proposers │ │ │ ├── io.rs │ │ │ ├── mod.rs │ │ │ ├── types.rs │ │ │ └── util.rs │ │ ├── veterans.rs │ │ └── voters.rs │ ├── stats │ │ ├── archive │ │ │ ├── calculator.rs │ │ │ ├── loader.rs │ │ │ └── mod.rs │ │ ├── distribution.rs │ │ ├── live │ │ │ ├── harvester.rs │ │ │ ├── mod.rs │ │ │ ├── monitor.rs │ │ │ └── settings.rs │ │ ├── mod.rs │ │ ├── snapshot.rs │ │ └── voters.rs │ ├── utils │ │ ├── csv.rs │ │ ├── mod.rs │ │ └── serde.rs │ ├── vca_reviews │ │ └── mod.rs │ └── vote_check │ │ ├── explorer.rs │ │ └── mod.rs └── tests │ ├── notifications │ ├── main.rs │ └── verifier.rs │ └── tally │ ├── blockchain.rs │ ├── generator.rs │ └── main.rs ├── ci ├── release-info.py └── strip-own-version-from-cargo-lock.pl ├── default.nix ├── flake.lock ├── flake.nix ├── shell.nix └── snapshot-lib ├── Cargo.toml ├── resources └── repsdb │ ├── all_representatives.graphql │ └── schema.graphql └── src ├── influence_cap.rs ├── lib.rs ├── registration.rs ├── voter_hir.rs └── voting_group.rs /.cargo/audit.toml: -------------------------------------------------------------------------------- 1 | [advisories] 2 | ignore = [ 3 | # this is the `time` segfault 4 | # `chrono` depends on `time = "0.1"`, but doesn't use the vulnerable API 5 | "RUSTSEC-2020-0071" 6 | ] 7 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | use flake 2 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # DEVOPS 2 | 3 | /.github/action/nix-common-setup* @input-output-hk/jormungandr-devops 4 | /.github/workflows/nix.yml @input-output-hk/jormungandr-devops 5 | /default.nix @input-output-hk/jormungandr-devops 6 | /flake.lock @input-output-hk/jormungandr-devops 7 | /flake.nix @input-output-hk/jormungandr-devops 8 | /shell.nix @input-output-hk/jormungandr-devops 9 | -------------------------------------------------------------------------------- /.github/actions/nix-common-setup/action.yml: -------------------------------------------------------------------------------- 1 | name: Setup Nix Environment 2 | inputs: 3 | CACHIX_AUTH_TOKEN: 4 | required: true 5 | description: 'Cachix Auth Token' 6 | runs: 7 | using: "composite" 8 | steps: 9 | 10 | - name: Installing Nix 11 | uses: cachix/install-nix-action@v16 12 | with: 13 | nix_path: nixpkgs=channel:nixpkgs-unstable 14 | extra_nix_config: | 15 | accept-flake-config = true 16 | trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= 17 | substituters = https://hydra.iohk.io https://cache.nixos.org/ 18 | 19 | - uses: cachix/cachix-action@v10 20 | with: 21 | name: iog 22 | authToken: '${{ inputs.CACHIX_AUTH_TOKEN }}' 23 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | 4 | - package-ecosystem: github-actions 5 | directory: "/" 6 | schedule: 7 | interval: daily 8 | time: '00:00' 9 | timezone: UTC 10 | open-pull-requests-limit: 10 11 | commit-message: 12 | prefix: "chore" 13 | include: "scope" 14 | -------------------------------------------------------------------------------- /.github/workflows/audit.yml: -------------------------------------------------------------------------------- 1 | name: Security audit 2 | on: 3 | push: 4 | paths: 5 | - Cargo.lock 6 | jobs: 7 | security_audit: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v3 11 | 12 | - id: ls-crates-io-index 13 | name: Get head commit hash of crates.io registry index 14 | run: | 15 | commit=$( 16 | git ls-remote --heads https://github.com/rust-lang/crates.io-index.git master | 17 | cut -f 1 18 | ) 19 | echo "::set-output name=head::$commit" 20 | - name: Cache cargo registry index 21 | uses: actions/cache@v3 22 | with: 23 | path: ~/.cargo/registry/index 24 | key: cargo-index-${{ steps.ls-crates-io-index.outputs.head }} 25 | restore-keys: | 26 | cargo-index- 27 | 28 | - uses: actions-rs/audit-check@v1 29 | with: 30 | token: ${{ secrets.GITHUB_TOKEN }} 31 | -------------------------------------------------------------------------------- /.github/workflows/nix.yml: -------------------------------------------------------------------------------- 1 | name: Nix CI 2 | on: 3 | push: 4 | branches: 5 | - main 6 | - catalyst-fund* 7 | pull_request: 8 | 9 | jobs: 10 | decision: 11 | name: Decide what we need to build 12 | runs-on: ubuntu-latest 13 | 14 | outputs: 15 | packages: ${{ steps.packages.outputs.packages }} 16 | 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v3 20 | 21 | - name: Setup 22 | uses: ./.github/actions/nix-common-setup 23 | with: 24 | CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} 25 | 26 | - name: Packages 27 | id: packages 28 | run: | 29 | packages=$(nix eval .#packages.x86_64-linux --apply builtins.attrNames --json) 30 | echo "PACKAGES -> $packages" 31 | echo "::set-output name=packages::$packages" 32 | 33 | build: 34 | name: Build ${{ matrix.package }} package 35 | needs: 36 | - decision 37 | runs-on: ubuntu-latest 38 | 39 | strategy: 40 | fail-fast: false 41 | matrix: 42 | package: ${{ fromJSON(needs.decision.outputs.packages) }} 43 | exclude: 44 | - package: default 45 | 46 | steps: 47 | 48 | - name: Checkout 49 | uses: actions/checkout@v3 50 | 51 | - name: Setup 52 | uses: ./.github/actions/nix-common-setup 53 | with: 54 | CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} 55 | 56 | - name: Build package 57 | run: | 58 | path=$(nix eval --raw .#packages.x86_64-linux.${{ matrix.package }}) 59 | hash=${path:11:32} 60 | url="https://iog.cachix.org/$hash.narinfo"; 61 | if curl --output /dev/null --silent --head --fail "$url"; then 62 | echo "Nothing to build!!!" 63 | echo "" 64 | echo "See build log with:" 65 | echo " nix log $path" 66 | echo "" 67 | else 68 | nix build .#packages.x86_64-linux.${{ matrix.package }} --show-trace -L 69 | fi 70 | -------------------------------------------------------------------------------- /.github/workflows/python-scripts.yml: -------------------------------------------------------------------------------- 1 | name: Python scripts linters 2 | 3 | on: 4 | push: 5 | 6 | jobs: 7 | lint: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Install linters 11 | run: pip3 install black 12 | 13 | - uses: actions/checkout@v3 14 | 15 | - name: Check formatting (black) 16 | run: black ./catalyst-toolbox/scripts/python --check 17 | -------------------------------------------------------------------------------- /.github/workflows/update-flake-lock.yml: -------------------------------------------------------------------------------- 1 | name: update-flake-lock 2 | on: 3 | workflow_dispatch: # allows manual triggering 4 | schedule: 5 | - cron: '0 0 * * 0' # runs weekly on Sunday at 00:00 6 | 7 | jobs: 8 | lockfile: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout repository 12 | uses: actions/checkout@v3 13 | - name: Install Nix 14 | uses: cachix/install-nix-action@v17 15 | with: 16 | extra_nix_config: | 17 | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} 18 | - name: Update flake.lock 19 | uses: DeterminateSystems/update-flake-lock@v14 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.vscode 2 | /.vs 3 | /.idea 4 | 5 | 6 | qr-code.svg 7 | 8 | /target 9 | /venv 10 | /.direnv 11 | /.pre-commit-config.yaml 12 | /result* -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | /nix/store/mm6z36vc1ss0scr9yf1gf40rp43ddj5j-pre-commit-config.json -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "catalyst-toolbox", 4 | "snapshot-lib" 5 | ] 6 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021 Input Output HK 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /catalyst-toolbox/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "catalyst-toolbox" 3 | version = "0.3.0" 4 | edition = "2021" 5 | 6 | authors = ["danielsanchezq "] 7 | license = "MIT OR Apache-2.0" 8 | description = "Rust based CLI utility for catalyst operations" 9 | 10 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 11 | [[bin]] 12 | name = "catalyst-toolbox" 13 | test = false 14 | bench = false 15 | 16 | [dependencies] 17 | assert_fs = "1" 18 | bech32 = "0.8.1" 19 | csv = "1.1" 20 | wallet = { git = "https://github.com/input-output-hk/chain-wallet-libs.git", branch = "master" } 21 | chain-addr = { git = "https://github.com/input-output-hk/chain-libs.git", branch = "master" } 22 | chain-core = { git = "https://github.com/input-output-hk/chain-libs.git", branch = "master" } 23 | chain-crypto = { git = "https://github.com/input-output-hk/chain-libs.git", branch = "master" } 24 | chain-ser = { git = "https://github.com/input-output-hk/chain-libs.git", branch = "master" } 25 | chain-storage = { git = "https://github.com/input-output-hk/chain-libs.git", branch = "master" } 26 | chain-time = { git = "https://github.com/input-output-hk/chain-libs.git", branch = "master" } 27 | chain-impl-mockchain = { git = "https://github.com/input-output-hk/chain-libs.git", branch = "master" } 28 | time = { version = "0.3", features = ["formatting", "parsing", "macros"] } 29 | itertools = "0.10" 30 | jcli = { git = "https://github.com/input-output-hk/jormungandr.git", branch = "master" } 31 | jormungandr-lib = { git = "https://github.com/input-output-hk/jormungandr.git", branch = "master" } 32 | jormungandr-integration-tests = { git = "https://github.com/input-output-hk/jormungandr.git", branch = "master" } 33 | jormungandr-automation = { git = "https://github.com/input-output-hk/jormungandr.git", branch = "master" } 34 | thor = { git = "https://github.com/input-output-hk/jormungandr.git", branch = "master" } 35 | jortestkit = { git = "https://github.com/input-output-hk/jortestkit.git", branch = "master" } 36 | rayon = "1.5" 37 | rust_decimal = "1.16" 38 | rust_decimal_macros = "1" 39 | futures = "0.3" 40 | once_cell = "1.8" 41 | reqwest = { version = "0.11", features = ["blocking", "json"] } 42 | rand = "0.8.3" 43 | rand_chacha = "0.3" 44 | governor = { version = "0.4", features = ["std", "jitter"], default-features = false} 45 | regex = "1.5" 46 | serde = "1.0" 47 | serde_json = "1.0" 48 | structopt = "0.3" 49 | serde_yaml = "0.8.17" 50 | sscanf = "0.1" 51 | color-eyre = "0.6" 52 | thiserror = "1.0" 53 | tokio = { version = "1.8", features = ["rt", "macros"] } 54 | url = "2.2" 55 | hex = "0.4" 56 | image = "0.23.12" 57 | qrcode = "0.12" 58 | quircs = "0.10.0" 59 | symmetric-cipher = { git = "https://github.com/input-output-hk/chain-wallet-libs.git", branch = "master" } 60 | graphql_client = { version = "0.10" } 61 | gag = "1" 62 | vit-servicing-station-lib = { git = "https://github.com/input-output-hk/vit-servicing-station.git", branch = "master" } 63 | snapshot-lib = { path = "../snapshot-lib" } 64 | fraction = "0.10" 65 | tracing = "0.1" 66 | tracing-subscriber = "0.3" 67 | 68 | [dev-dependencies] 69 | rand_chacha = "0.3" 70 | assert_cmd = "0.10" 71 | predicates = "1" 72 | assert_fs = "1.0.0" 73 | chain-vote = { git = "https://github.com/input-output-hk/chain-libs.git", branch = "master" } 74 | proptest = { git = "https://github.com/input-output-hk/proptest", branch = "master" } 75 | test-strategy = "0.2" 76 | serde_test = "1" 77 | snapshot-lib = { path = "../snapshot-lib", features = ["proptest"] } 78 | vit-servicing-station-tests = { git = "https://github.com/input-output-hk/vit-servicing-station.git", branch = "master" } 79 | 80 | [build-dependencies] 81 | versionisator = "1.0.3" 82 | 83 | [features] 84 | test-api = [] 85 | -------------------------------------------------------------------------------- /catalyst-toolbox/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let pkg_version = if let Ok(date) = std::env::var("DATE") { 3 | format!("{}.{}", env!("CARGO_PKG_VERSION"), date) 4 | } else { 5 | env!("CARGO_PKG_VERSION").to_string() 6 | }; 7 | 8 | println!("cargo:rustc-env=CARGO_PKG_VERSION={}", pkg_version); 9 | 10 | let version = versionisator::Version::new( 11 | env!("CARGO_MANIFEST_DIR"), 12 | env!("CARGO_PKG_NAME").to_string(), 13 | pkg_version, 14 | ); 15 | 16 | println!("cargo:rustc-env=FULL_VERSION={}", version.full()); 17 | println!("cargo:rustc-env=SIMPLE_VERSION={}", version.simple()); 18 | println!("cargo:rustc-env=SOURCE_VERSION={}", version.hash()); 19 | } 20 | -------------------------------------------------------------------------------- /catalyst-toolbox/doc/ideascale_importer.md: -------------------------------------------------------------------------------- 1 | # Ideascale importer 2 | 3 | [Ideascale importer](https://github.com/input-output-hk/catalyst-toolbox#ideascale-import) is the tool we use for dumping fund related data. This data is latter massaged and transformed 4 | into proper formats that is later fed into the vit-servicing-station. 5 | 6 | 7 | ## Knows and how's 8 | 9 | ### Ideascale API endpoints 10 | 11 | There are 4 main endpoints that are pinged for getting the data we need: 12 | 13 | 1. `campaigns/groups` 14 | 2. `stages` 15 | 3. `campaigns/{challenge_id}/ideas/0/100000` 16 | 4. `funnels` 17 | 18 | Please refer to the [ideascale api specification](https://a.ideascale.com/api-docs/index.html) to get specific information 19 | for each endpoint. 20 | 21 | 22 | The query logic work as follows: 23 | 24 | 1. Queries the challenges (`funnels`). 25 | 2. Queries the funds data (`campaigns/groups`). 26 | 3. Filter challenges based on fund id (excluding process improvements one {7666}) 27 | 4. Queries proposals for each challenge (`campaigns/{challenge_id}/ideas/0/100000`) 28 | 5. Queries stages (`stages`) 29 | 30 | With these ideascale data (challenges, fund, proposals, and stages) we can build the intermediary data. 31 | It is a process of picking, mixing and filtering on that we query before. Actual building code can be 32 | found [here](https://github.com/input-output-hk/catalyst-toolbox/blob/main/src/ideascale/mod.rs) in the `build_*` 33 | functions. 34 | 35 | ### Custom fields 36 | 37 | Proposals in ideascale contain some fields that may depend on fund configuration. So we could be as flexible as possible 38 | those were abstracted into a [custom matching struct](https://github.com/input-output-hk/catalyst-toolbox/blob/main/src/ideascale/models/custom_fields.rs). 39 | It has a default implementation which was stable till fund7. But it would be desirable to maintain this as a configuration file 40 | that could be saved as part of the auditable fund data. 41 | 42 | ### Intermediary data 43 | 44 | It is difficult to handle data as coming from ideascale. Usually, will be nested in ways we will not want it and named it 45 | in ways it is not useful. In order to fit the data in the formats we want as output we need to transform it. In this case 46 | managing this transformation solely with serde would be quite convoluted. So, to fix this we have some 47 | `de` (deserialize structures) that we use to load, parse and transform data incoming from ideascale 48 | and `se` (serialize structures) that are the final form of the output of this tool. -------------------------------------------------------------------------------- /catalyst-toolbox/doc/private-key-and-qr-code.md: -------------------------------------------------------------------------------- 1 | # Private Key Transfer Protocol 2 | 3 | The encoding described in this document is used to export a private key 4 | for use in another application. 5 | 6 | In Catalyst, this format is used in the QR code used to pass the private key 7 | from a wallet to the mobile application. 8 | 9 | ## Serialization format 10 | 11 | We will expect the scheme to be updateable. So the first byte of the data 12 | transferred refers to the protocol version in use. Here we reserve 0 for error. 13 | 1 for the protocol that is going to be defined in this document. 14 | 15 | Once reading the first byte is 0b0000_0001 we can assume all the remaining bytes 16 | are part of the scheme. 17 | 18 | The first following 16 bytes are the salt. 19 | Then the following 12 bytes are the nonce. 20 | Then the encrypted data and then the last 16 bytes are for the tag. 21 | 22 | ``` 23 | +---------+----------+----------+----------------+----------+ 24 | | Version | Salt | Nonce | Encrypted Data | Tag | 25 | +---------+----------+----------+----------------+----------+ 26 | | 0x01 | 16 bytes | 12 bytes | | 16 bytes | 27 | +---------+----------+----------+----------------+----------+ 28 | ``` 29 | 30 | The encrypted data unit is expected to be the binary representation of an 31 | extended ed25519 key, 64 bytes in length. 32 | In this application, we do not need the chain code as we will not do 33 | any derivation. 34 | So if 256 bytes of data are encoded, we are expecting 4 Ed25519 Extended keys. 35 | 36 | # Symmetric encryption 37 | 38 | Inputs: 39 | 40 | * Password: a byte array 41 | * Data: a byte array 42 | 43 | Algorithm: 44 | 45 | 1. Generate a SALT of 16 bytes (only for encryption, on decryption the SALT is provided) 46 | 2. Generate a NONCE of 12 bytes (only for encryption, on decryption the NONCE is provided) 47 | 3. Derive the symmetric encryption key from the password: 48 | * Use PBKDF2 HMAC SHA512 49 | * 12983 iterations 50 | * Use the SALT 51 | 4. Encrypt the data (or decrypt) 52 | * Use ChaCha20Poly1305 53 | * Use the symmetric encryption key derived in step 3 54 | * Use the NONCE 55 | 56 | Outputs: encode the result in the format defined in the previous section. 57 | -------------------------------------------------------------------------------- /catalyst-toolbox/doc/rewards.md: -------------------------------------------------------------------------------- 1 | # Rewards data pipeline 2 | 3 | The rewards process is an entangled system of data requirements which will 4 | be listed in the former document. 5 | 6 | 7 | ## Voters rewards 8 | 9 | ### Input 10 | 11 | Currently, (as per Fund7) the tool needs: 12 | 13 | * The block0 file (bin) 14 | * The amount of rewards to distribute 15 | * The threshold of votes a voter need in order to access such rewards 16 | 17 | ### Output 18 | 19 | A Csv is generated with the following headers: 20 | 21 | 22 | ``` 23 | +---------+---------------------------+----------------------------+-------------------------------+ 24 | | Address | Stake of the voter (ADA) | Reward for the voter (ADA) |Reward for the voter (lovelace)| 25 | +---------+---------------------------+----------------------------+-------------------------------+ 26 | ``` 27 | 28 | ## Proposers reward 29 | 30 | Users that propose proposals get a rewards too. We can use the [`proposers_rewards.py`](https://github.com/input-output-hk/catalyst-toolbox#calculate-proposers-rewards) script for that. 31 | The scrip has two modes of operating, online and offline. 32 | The online mode works with the data living in the vit-servicing-station server. 33 | The offline mode need to load that data manually through some json files. 34 | Those json files can be downloaded from the vit-servicing-station at any time during the fund. 35 | 36 | ### Input 37 | 38 | #### Json files needed 39 | 1. challenges: from `https://servicing-station.vit.iohk.io/api/v0/challenges` 40 | 2. active voteplans: from `https://servicing-station.vit.iohk.io/api/v0/vote/active/plans` 41 | 3. proposals: from `https://servicing-station.vit.iohk.io/api/v0/proposals` 42 | 4. excluded proposals: a json file with a list of excluded proposals ids `[id, ..idx]` 43 | 44 | ### Output 45 | 46 | The proposers output is csv with several data on it. 47 | ***Really important***, this output file is used as source of truth for the approved proposals 48 | (not to be mistaken with funded proposals). 49 | 50 | Output csv headers: 51 | * internal_id: proposal internal id (from vss) 52 | * proposal_id: proposal chain id 53 | * proposal: proposal title 54 | * overall_score: proposal impact score 55 | * yes: how many yes votes 56 | * no: how many no votes 57 | * result: yes vs no votes difference 58 | * meets_approval_threshold: **is proposal approved** 59 | * requested_dollars: amount of funding requested 60 | * status: **is proposal funded** 61 | * fund_depletion: fund remaining after proposal depletion (entries are sorted in descending order of 'result') 62 | * not_funded_reason: why wasnt the proposal not funded (if applies, over budget or approval threshold) 63 | * link_to_ideascale: url to ideascale proposal page 64 | 65 | The output files are generated per challenge. So, if we have 30 challenges we would have 30 generated output files 66 | in the same fashion. 67 | 68 | 69 | ## Community advisors rewards 70 | 71 | ### Input 72 | 73 | There are 2 (two) main input files needed for calculating the community advisors rewards: 74 | 75 | 1. Proposers reward result output file (approved proposals): We need this to check which of the proposals were approved. 76 | Notice that the proposers rewards script output is per challenge. So in order to use it we need to aggregate all the csv 77 | into a single file (same headers, order is irrelevant). For this we can use the 78 | [`csv_merger.py`](https://github.com/input-output-hk/catalyst-toolbox/blob/main/catalyst-toolbox/scripts/python/csv_merger.py) script, 79 | or any other handier tool. 80 | 2.Assessments csv (assessments): This is a file that comes from the community. It holds the information with the reviews performed 81 | by the CAs. 82 | 83 | ### Output 84 | 85 | A csv with pairs of anonymize CA ids and the amount of the reward: 86 | 87 | ``` 88 | +----+----------+ 89 | | id | rewards | 90 | +----+----------+ 91 | ``` 92 | 93 | ## Veteran community advisors rewards 94 | 95 | ### Input 96 | 97 | Currently, (as per fund7) it is just a normal distribution based on `(number_of_reviews/total_reviews)*total_rewards` 98 | 99 | For that we just need to know the amount of rewards done by each veteran: 100 | 101 | 1. Veteran reviews count: A csv with pairs of `veteran_id -> total_reviews`. It is also a community based document 102 | (it is provided every fund). 103 | 104 | 105 | ### Output 106 | 107 | A csv with pairs of anonymize veteran CA ids and the amount of the reward, `veteran_id -> total_rewards`. 108 | -------------------------------------------------------------------------------- /catalyst-toolbox/resources/explorer/transaction_by_id.graphql: -------------------------------------------------------------------------------- 1 | query TransactionById($id: String!){ 2 | transaction(id: $id) { 3 | blocks { 4 | id 5 | branches { 6 | id 7 | } 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /catalyst-toolbox/resources/testing/reviews.csv: -------------------------------------------------------------------------------- 1 | id,triplet_id,Idea Title,Idea URL,proposal_id,Question,question_id,Rating Given,Assessor,Assessment Note,Proposer Mark,Acceptable,Constructive Feedback,Abstain,Strict,Lenient,Offense / Profanity,Non Constructive Feedback,Score doesn't match,Copy,Incomplete Reading,Not Related,General Infraction,# of vCAs Reviews,Yellow Card,Red Card 2 | 2244,52-351102,Magento 2 ada payments plugin,http://ideascale.com/t/UM5UZBiVw,351102,This proposal effectively addresses the challenge,1,2,z_assessor_52,"While this is a potentially useful project, it is not as innovative as other proposals in this challenge and the effects are not as far-reaching. Those funds would be better spent on other projects within this challenge.",,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0 3 | 2546,22-351198,Workspace for JavaScript Developers,http://ideascale.com/t/UM5UZBiXV,351198,The information provided is sufficient to audit the progress and the success of the proposal.,3,1,z_assessor_22,"The proposal claims that Cardano projects essentially will look the same. If that was the case, then maybe instead of JavaScript, this should be a Haskell project. This proposal, if implemented in presented manner, would do nothing for the Cardano ecosystem.",,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0 4 | 2866,46-351380,Cashout and Buy ADA in Ethiopia,http://ideascale.com/t/UM5UZBibU,351380,Given experience and plan presented it is highly likely this proposal will be implemented successfully,2,5,z_assessor_46,"The proposal is simple and aims to provide a transparent government-approved exchange service, This has minimal risks, so the proposers will be able to implement the proposal.",x,0,0,1,0,4,0,0,0,0,0,0,0,0,0,0 5 | 2931,57-351403,E-ROW waterways transportation,http://ideascale.com/t/UM5UZBibs,351403,The information provided is sufficient to audit the progress and the success of the proposal.,3,5,z_assessor_57," It includes stages that the project will go through in order to materialize, as well as a division explaining how the funding would be used, in percentage, for what purpose. We are also told the expectation of how the project would be impacting the lives of local residents (according to the data provided, 50K people would be using water transport in 6 months). Therefore, I believe that the project is well planned, very detailed in relation to an action plan and expectations to be achieved. ",,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0 6 | 2972,108-351471,DEFI billing - an untapped market?,http://ideascale.com/t/UM5UZBicz,351471,This proposal effectively addresses the challenge,1,1,z_assessor_108,"There is no technical documentation or WP included in this proposal and the timeline included says the dApp won't go live for 12 months, this is uncompetitive. This is also not DeFi if you need to rely on government utility depts/billing agencies that is CeFi.",,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0 7 | 2980,108-351471,DEFI billing - an untapped market?,http://ideascale.com/t/UM5UZBicz,351471,Given experience and plan presented it is highly likely this proposal will be implemented successfully,2,1,z_assessor_108,"There is no way to verify the experience of the proposer beyond the ""relevant experience"", no GH links, no demo app/website. The roadmap is also for 12 months out for go-live, given the requested budget 1 year out is way too long to have a production dApp.",,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0 8 | 3009,53-351487,WooCommerce Payments Plugin,http://ideascale.com/t/UM5UZBidF,351487,The information provided is sufficient to audit the progress and the success of the proposal.,3,4,z_assessor_53,The only gaps I can see to this proposal are minor and relate to the information provided. This proposal would have benefitted from further explanation of some of the more technical aspects especially in light of voter levels of understanding. Its really the only aspect that I found lacking. ,,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 9 | 3052,46-351489,Hyper-Localizable Wallet for Africa,http://ideascale.com/t/UM5UZBidH,351489,The information provided is sufficient to audit the progress and the success of the proposal.,3,5,z_assessor_46,The proposal is for development of mobile wallet with low data use and in local africal language/ no text wallet. The progress can be audited by monitoring the developmental. Success can also be accurately measured by seeing the number of downloads.,x,0,0,2,0,1,0,0,0,0,0,0,0,0,0,0 10 | 3070,71-351501,Artificial Intelligence/ML API,http://ideascale.com/t/UM5UZBidT,351501,Given experience and plan presented it is highly likely this proposal will be implemented successfully,2,5,z_assessor_71,It is likely for this proposal to be implemented because Qbo is an existing company and the Qbo team has extensive experience in AI and ML robotics. ,x,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0 11 | -------------------------------------------------------------------------------- /catalyst-toolbox/resources/testing/valid_assessments.csv: -------------------------------------------------------------------------------- 1 | id,Idea Title,proposal_id,Idea URL,Assessor,Impact / Alignment Note,Impact / Alignment Rating,Feasibility Note,Feasibility Rating,Auditability Note,Auditability Rating,Excellent,Good 2 | 1,Design & Sim of Voting Influence,53123,http://app.ideascale.com/t/UM5UZBmp2,z_assessor_101,"This proposal addresses the optimization of our CATALYST Voting and funding system. Major part of this is research-based work as well as designing and simulating such improved voting systems. 3 | 4 | This touches the very core of our journey towards VOLTAIRE and is therefore for sure a proposal of high impact. I would have liked to see a more detailed explanation, if and how the proposer team plans to work together with IOHK experts for avoiding overlapping efforts. ",4,"The team behind this proposal is undoubtfully highly qualified. There is good background material added for outlining their field of expertise and their impressive experience. 5 | 6 | In order to make this substantial investment a success, I would have liked to see a better explanation of the aligned with the work done at IOHK/IOG. A lack iof alignment could jeopardize the likelihood of success, even though great expertise is at work here. ",4,"The proposer team has outlined, how they want to approach this rather big topic step-by-step. 7 | 8 | As a CA I did not really find a comprehensive way of how to audit the progress of this proposal, other than kind of trusting that experts from the proposal team will get together with experts from IOHK/IOG (somehow) and implement this all together. Again, i would have liked to find more description about how this implementation is planned (otherwise this project might easily just add new research - which is good - but lack practical deployment).",0,x, -------------------------------------------------------------------------------- /catalyst-toolbox/resources/testing/veteran_reviews_count.csv: -------------------------------------------------------------------------------- 1 | name,vca_link,No. of Reviews 2 | Andreas,https://docs.google.com,50 3 | Antonio,https://docs.google.com,100 4 | Diamond,https://docs.google.com,200 5 | Penney,https://drive.google.com,300 6 | Tomi,https://docs.google.com,350 7 | 8 | -------------------------------------------------------------------------------- /catalyst-toolbox/scripts/python/csv_merger.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import glob 3 | from pathlib import Path 4 | import os 5 | from typing import List, Generator, Iterator, TextIO, Tuple 6 | import typer 7 | from contextlib import contextmanager 8 | 9 | 10 | @contextmanager 11 | def open_files(files: Iterator[str]) -> Generator[Iterator[TextIO], None, None]: 12 | files_objs = [open(file, encoding="utf-8", newline="") for file in files] 13 | yield iter(files_objs) 14 | for file in files_objs: 15 | file.close() 16 | 17 | 18 | def search_file_pattern(pattern: str, base_path: Path) -> Iterator[str]: 19 | yield from map( 20 | lambda file: os.path.join(base_path, file), 21 | glob.iglob(pathname=pattern, root_dir=base_path), 22 | ) 23 | 24 | 25 | def file_as_csv(file: TextIO, delimiter: chr) -> Tuple[List[str], Iterator[List[str]]]: 26 | reader = csv.reader(file, delimiter=delimiter) 27 | return next(reader), reader 28 | 29 | 30 | def merge_csv( 31 | pattern: str, 32 | base_path: Path, 33 | output_file: Path, 34 | input_delimiter: chr, 35 | output_delimiter: chr, 36 | ): 37 | print(pattern) 38 | files = search_file_pattern(pattern, base_path) 39 | with open(output_file, "w", encoding="utf-8", newline="") as out_file: 40 | with open_files(files) as fs: 41 | writer = csv.writer(out_file, delimiter=output_delimiter) 42 | header, first_content = file_as_csv(next(fs), delimiter=input_delimiter) 43 | writer.writerow(header) 44 | writer.writerows(first_content) 45 | for file in fs: 46 | # skip headers and use just content 47 | _, content = file_as_csv(file, delimiter=input_delimiter) 48 | writer.writerows(content) 49 | 50 | 51 | def merge_csv_files( 52 | output_file: Path = typer.Option(...), 53 | pattern: str = typer.Option(...), 54 | base_path: Path = typer.Option(default=Path("./")), 55 | input_delimiter: str = typer.Option(default=","), 56 | output_delimiter: str = typer.Option(default=","), 57 | ): 58 | merge_csv(pattern, base_path, output_file, input_delimiter, output_delimiter) 59 | 60 | 61 | if __name__ == "__main__": 62 | typer.run(merge_csv_files) 63 | -------------------------------------------------------------------------------- /catalyst-toolbox/scripts/python/requirements.txt: -------------------------------------------------------------------------------- 1 | httpx==0.17.1 2 | pydantic==1.8.1 3 | typer==0.3.2 4 | pyYAML==6.0 5 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/archive.rs: -------------------------------------------------------------------------------- 1 | use chain_addr::Discrimination; 2 | use chain_core::{packer::Codec, property::DeserializeFromSlice}; 3 | use chain_impl_mockchain::{ 4 | block::Block, chaintypes::HeaderId, fragment::Fragment, transaction::InputEnum, 5 | }; 6 | use jormungandr_lib::interfaces::{AccountIdentifier, Address}; 7 | 8 | use serde::Serialize; 9 | 10 | use std::{collections::HashMap, path::Path}; 11 | 12 | const MAIN_TAG: &str = "HEAD"; 13 | 14 | #[derive(Debug, thiserror::Error)] 15 | pub enum Error { 16 | #[error(transparent)] 17 | Storage(#[from] chain_storage::Error), 18 | 19 | #[error(transparent)] 20 | Io(#[from] std::io::Error), 21 | 22 | #[error(transparent)] 23 | Csv(#[from] csv::Error), 24 | 25 | #[error("Only accounts inputs are supported not Utxos")] 26 | UnhandledInput, 27 | } 28 | 29 | #[derive(Serialize)] 30 | struct Vote { 31 | fragment_id: String, 32 | caster: Address, 33 | proposal: u8, 34 | time: String, 35 | choice: u8, 36 | raw_fragment: String, 37 | } 38 | 39 | pub fn generate_archive_files(jormungandr_database: &Path, output_dir: &Path) -> Result<(), Error> { 40 | let db = chain_storage::BlockStore::file( 41 | jormungandr_database, 42 | HeaderId::zero_hash() 43 | .as_bytes() 44 | .to_owned() 45 | .into_boxed_slice(), 46 | )?; 47 | 48 | // Tag should be present 49 | let tip_id = db.get_tag(MAIN_TAG)?.unwrap(); 50 | let distance = db.get_block_info(tip_id.as_ref())?.chain_length(); 51 | 52 | let mut vote_plan_files = HashMap::new(); 53 | 54 | let block_iter = db.iter(tip_id.as_ref(), distance)?; 55 | 56 | for iter_res in block_iter { 57 | let block_bin = iter_res?; 58 | let mut codec = Codec::new(block_bin.as_ref()); 59 | let block: Block = DeserializeFromSlice::deserialize_from_slice(&mut codec).unwrap(); 60 | 61 | for fragment in block.fragments() { 62 | if let Fragment::VoteCast(tx) = fragment { 63 | let fragment_id = fragment.hash(); 64 | 65 | let input = tx.as_slice().inputs().iter().next().unwrap().to_enum(); 66 | let caster = if let InputEnum::AccountInput(account_id, _value) = input { 67 | AccountIdentifier::from(account_id) 68 | .into_address(Discrimination::Production, "ca") 69 | } else { 70 | return Err(Error::UnhandledInput); 71 | }; 72 | let certificate = tx.as_slice().payload().into_payload(); 73 | 74 | let writer = vote_plan_files 75 | .entry(certificate.vote_plan().clone()) 76 | .or_insert_with(|| { 77 | let mut path = output_dir.to_path_buf(); 78 | path.push(format!("vote_plan_{}.csv", certificate.vote_plan())); 79 | let file = std::fs::File::create(path).unwrap(); 80 | csv::Writer::from_writer(file) 81 | }); 82 | 83 | let choice = match certificate.payload() { 84 | chain_impl_mockchain::vote::Payload::Public { choice } => choice.as_byte(), 85 | chain_impl_mockchain::vote::Payload::Private { .. } => { 86 | // zeroing data to enable private voting support 87 | // (at least everying exception choice, since it is disabled by desing in private vote) 88 | 0u8 89 | } 90 | }; 91 | 92 | writer.serialize(Vote { 93 | fragment_id: fragment_id.to_string(), 94 | caster, 95 | proposal: certificate.proposal_index(), 96 | time: block.header().block_date().to_string(), 97 | raw_fragment: hex::encode(tx.as_ref()), 98 | choice, 99 | })?; 100 | } 101 | } 102 | } 103 | Ok(()) 104 | } 105 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/catalyst-toolbox.rs: -------------------------------------------------------------------------------- 1 | use structopt::StructOpt; 2 | 3 | pub mod cli; 4 | 5 | fn main() -> color_eyre::Result<()> { 6 | tracing_subscriber::fmt().init(); 7 | color_eyre::install()?; 8 | cli::Cli::from_args().exec()?; 9 | Ok(()) 10 | } 11 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/advisor_reviews/mod.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::utils; 2 | use catalyst_toolbox::vca_reviews::read_vca_reviews_aggregated_file; 3 | 4 | use color_eyre::eyre::bail; 5 | use color_eyre::Report; 6 | use jcli_lib::utils::io::open_file_write; 7 | use std::fmt; 8 | use std::path::PathBuf; 9 | use std::str::FromStr; 10 | use structopt::StructOpt; 11 | 12 | #[derive(Debug)] 13 | pub enum OutputFormat { 14 | Csv, 15 | Json, 16 | } 17 | 18 | #[derive(StructOpt)] 19 | pub enum Reviews { 20 | Export(Export), 21 | } 22 | 23 | #[derive(StructOpt)] 24 | pub struct Export { 25 | /// Path to vca aggregated file 26 | #[structopt(long)] 27 | from: PathBuf, 28 | /// Output file 29 | #[structopt(long)] 30 | to: PathBuf, 31 | /// Output format either csv or json 32 | #[structopt(long, default_value = "csv")] 33 | format: OutputFormat, 34 | } 35 | 36 | impl FromStr for OutputFormat { 37 | type Err = Report; 38 | 39 | fn from_str(s: &str) -> Result { 40 | match s.to_lowercase().as_str() { 41 | "csv" => Ok(Self::Csv), 42 | "json" => Ok(Self::Json), 43 | other => bail!("invalid format: {other}"), 44 | } 45 | } 46 | } 47 | 48 | impl fmt::Display for OutputFormat { 49 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 50 | write!(f, "{:?}", self) 51 | } 52 | } 53 | 54 | impl Reviews { 55 | pub fn exec(self) -> Result<(), Report> { 56 | match self { 57 | Reviews::Export(transform) => transform.exec()?, 58 | }; 59 | Ok(()) 60 | } 61 | } 62 | 63 | impl Export { 64 | pub fn exec(self) -> Result<(), Report> { 65 | let Self { from, to, format } = self; 66 | 67 | let reviews = read_vca_reviews_aggregated_file(&from)?; 68 | match format { 69 | OutputFormat::Csv => { 70 | utils::csv::dump_data_to_csv(reviews.iter(), &to)?; 71 | } 72 | OutputFormat::Json => { 73 | serde_json::to_writer(open_file_write(&Some(to))?, &reviews)?; 74 | } 75 | }; 76 | Ok(()) 77 | } 78 | } 79 | 80 | #[cfg(test)] 81 | mod test { 82 | use super::{Export, OutputFormat}; 83 | use catalyst_toolbox::utils::csv; 84 | use vit_servicing_station_lib::db::models::community_advisors_reviews::AdvisorReview; 85 | 86 | #[test] 87 | fn test_output_csv() { 88 | let resource_input = "resources/testing/valid_assessments.csv"; 89 | let tmp_file = assert_fs::NamedTempFile::new("outfile.csv").unwrap(); 90 | 91 | let export = Export { 92 | from: resource_input.into(), 93 | to: tmp_file.path().into(), 94 | format: OutputFormat::Csv, 95 | }; 96 | 97 | export.exec().unwrap(); 98 | let reviews: Vec = csv::load_data_from_csv::<_, b','>(&tmp_file).unwrap(); 99 | assert_eq!(reviews.len(), 1); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/archive/mod.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::archive::generate_archive_files; 2 | 3 | use color_eyre::Report; 4 | use structopt::StructOpt; 5 | 6 | use std::path::PathBuf; 7 | 8 | #[derive(Debug, StructOpt)] 9 | #[structopt(rename_all = "kebab-case")] 10 | pub struct Archive { 11 | /// The path to the Jormungandr database to dump transactions from. 12 | jormungandr_database: PathBuf, 13 | /// CSV output directory 14 | output_dir: PathBuf, 15 | } 16 | 17 | impl Archive { 18 | pub fn exec(self) -> Result<(), Report> { 19 | generate_archive_files(&self.jormungandr_database, &self.output_dir)?; 20 | Ok(()) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/kedqr/decode/img.rs: -------------------------------------------------------------------------------- 1 | use bech32::{ToBase32, Variant}; 2 | use catalyst_toolbox::kedqr::KeyQrCode; 3 | use catalyst_toolbox::kedqr::QrPin; 4 | use chain_crypto::AsymmetricKey; 5 | use chain_crypto::Ed25519Extended; 6 | use chain_crypto::SecretKey; 7 | use color_eyre::Report; 8 | use std::fs::File; 9 | use std::io::Write; 10 | use std::path::Path; 11 | use std::path::PathBuf; 12 | 13 | pub fn save_secret_from_qr(qr: PathBuf, output: Option, pin: QrPin) -> Result<(), Report> { 14 | let sk = secret_from_qr(&qr, pin)?; 15 | let hrp = Ed25519Extended::SECRET_BECH32_HRP; 16 | let secret_key = bech32::encode(hrp, sk.leak_secret().to_base32(), Variant::Bech32)?; 17 | 18 | match output { 19 | Some(path) => { 20 | // save secret to file, or print to stdout if it fails 21 | let mut file = File::create(path)?; 22 | file.write_all(secret_key.as_bytes())?; 23 | } 24 | None => { 25 | // prints secret to stdout when no path is specified 26 | println!("{}", secret_key); 27 | } 28 | }; 29 | Ok(()) 30 | } 31 | 32 | pub fn secret_from_qr( 33 | qr: impl AsRef, 34 | pin: QrPin, 35 | ) -> Result, Report> { 36 | let img = image::open(qr)?; 37 | let secret = KeyQrCode::decode(img, &pin.password)?; 38 | Ok(secret.first().unwrap().clone()) 39 | } 40 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/kedqr/decode/mod.rs: -------------------------------------------------------------------------------- 1 | mod img; 2 | mod payload; 3 | 4 | use crate::cli::kedqr::QrCodeOpts; 5 | use catalyst_toolbox::kedqr::QrPin; 6 | use color_eyre::Report; 7 | pub use img::{save_secret_from_qr, secret_from_qr}; 8 | pub use payload::{decode_payload, secret_from_payload}; 9 | use std::path::PathBuf; 10 | use structopt::StructOpt; 11 | 12 | /// QCode CLI toolkit 13 | #[derive(Debug, PartialEq, Eq, StructOpt)] 14 | #[structopt(rename_all = "kebab-case")] 15 | pub struct DecodeQrCodeCmd { 16 | /// Path to file containing img or payload. 17 | #[structopt(short, long, parse(from_os_str))] 18 | input: PathBuf, 19 | /// Path to file to save secret output, if not provided console output will be attempted. 20 | #[structopt(short, long, parse(from_os_str))] 21 | output: Option, 22 | /// Pin code. 4-digit number is used on Catalyst. 23 | #[structopt(short, long, parse(try_from_str))] 24 | pin: QrPin, 25 | 26 | #[structopt(flatten)] 27 | opts: QrCodeOpts, 28 | } 29 | 30 | impl DecodeQrCodeCmd { 31 | pub fn exec(self) -> Result<(), Report> { 32 | match self.opts { 33 | QrCodeOpts::Payload => decode_payload(self.input, self.output, self.pin), 34 | QrCodeOpts::Img => save_secret_from_qr(self.input, self.output, self.pin), 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/kedqr/decode/payload.rs: -------------------------------------------------------------------------------- 1 | use bech32::{ToBase32, Variant}; 2 | use catalyst_toolbox::kedqr::{decode, QrPin}; 3 | use chain_crypto::{AsymmetricKey, Ed25519Extended, SecretKey}; 4 | use color_eyre::Report; 5 | use std::{ 6 | fs::{File, OpenOptions}, 7 | io::{BufRead, BufReader, Write}, 8 | path::{Path, PathBuf}, 9 | }; 10 | 11 | pub fn decode_payload(input: PathBuf, output: Option, pin: QrPin) -> Result<(), Report> { 12 | // generate qrcode with key and parsed pin 13 | let secret = secret_from_payload(input, pin)?; 14 | let hrp = Ed25519Extended::SECRET_BECH32_HRP; 15 | let secret_key = bech32::encode(hrp, secret.leak_secret().to_base32(), Variant::Bech32)?; 16 | // process output 17 | match output { 18 | Some(path) => { 19 | // save qr code to file, or print to stdout if it fails 20 | let mut file = File::create(path)?; 21 | file.write_all(secret_key.as_bytes())?; 22 | } 23 | None => { 24 | // prints qr code to stdout when no path is specified 25 | println!("{}", secret_key); 26 | } 27 | } 28 | Ok(()) 29 | } 30 | 31 | pub fn secret_from_payload( 32 | input: impl AsRef, 33 | pin: QrPin, 34 | ) -> Result, Report> { 35 | let input = OpenOptions::new() 36 | .create(false) 37 | .read(true) 38 | .write(false) 39 | .append(false) 40 | .open(&input) 41 | .expect("Could not open input file."); 42 | 43 | let mut reader = BufReader::new(input); 44 | let mut payload_str = String::new(); 45 | let _len = reader 46 | .read_line(&mut payload_str) 47 | .expect("Could not read input file."); 48 | payload_str = payload_str.trim_end().to_string(); 49 | 50 | // use parsed pin from args 51 | let pwd = pin.password; 52 | // generate qrcode with key and parsed pin 53 | Ok(decode(payload_str, &pwd)?) 54 | } 55 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/kedqr/encode/img.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::kedqr::{KeyQrCode, QrPin}; 2 | use chain_crypto::bech32::Bech32; 3 | use chain_crypto::{Ed25519Extended, SecretKey}; 4 | use color_eyre::Report; 5 | use std::{ 6 | fs::OpenOptions, 7 | io::{BufRead, BufReader}, 8 | path::PathBuf, 9 | }; 10 | 11 | pub fn generate_qr(input: PathBuf, output: Option, pin: QrPin) -> Result<(), Report> { 12 | // open input key and parse it 13 | let key_file = OpenOptions::new() 14 | .create(false) 15 | .read(true) 16 | .write(false) 17 | .append(false) 18 | .open(&input) 19 | .expect("Could not open input file."); 20 | 21 | let mut reader = BufReader::new(key_file); 22 | let mut key_str = String::new(); 23 | reader 24 | .read_line(&mut key_str) 25 | .expect("Could not read input file."); 26 | let sk = key_str.trim_end().to_string(); 27 | 28 | let secret_key: SecretKey = 29 | SecretKey::try_from_bech32_str(&sk).expect("Malformed secret key."); 30 | // use parsed pin from args 31 | let pwd = pin.password; 32 | // generate qrcode with key and parsed pin 33 | let qr = KeyQrCode::generate(secret_key, &pwd); 34 | // process output 35 | match output { 36 | Some(path) => { 37 | // save qr code to file, or print to stdout if it fails 38 | let img = qr.to_img(); 39 | if let Err(e) = img.save(path) { 40 | println!("Error: {}", e); 41 | println!(); 42 | println!("{}", qr); 43 | } 44 | } 45 | None => { 46 | // prints qr code to stdout when no path is specified 47 | println!(); 48 | println!("{}", qr); 49 | } 50 | } 51 | Ok(()) 52 | } 53 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/kedqr/encode/mod.rs: -------------------------------------------------------------------------------- 1 | mod img; 2 | mod payload; 3 | 4 | use crate::cli::kedqr::QrCodeOpts; 5 | use catalyst_toolbox::kedqr::QrPin; 6 | use color_eyre::Report; 7 | pub use img::generate_qr; 8 | pub use payload::generate_payload; 9 | use std::path::PathBuf; 10 | use structopt::StructOpt; 11 | 12 | /// QCode CLI toolkit 13 | #[derive(Debug, PartialEq, Eq, StructOpt)] 14 | #[structopt(rename_all = "kebab-case")] 15 | pub struct EncodeQrCodeCmd { 16 | /// Path to file containing ed25519extended bech32 value. 17 | #[structopt(short, long, parse(from_os_str))] 18 | input: PathBuf, 19 | /// Path to file to save qr code output, if not provided console output will be attempted. 20 | #[structopt(short, long, parse(from_os_str))] 21 | output: Option, 22 | /// Pin code. 4-digit number is used on Catalyst. 23 | #[structopt(short, long, parse(try_from_str))] 24 | pin: QrPin, 25 | 26 | #[structopt(flatten)] 27 | opts: QrCodeOpts, 28 | } 29 | 30 | impl EncodeQrCodeCmd { 31 | pub fn exec(self) -> Result<(), Report> { 32 | match self.opts { 33 | QrCodeOpts::Payload => generate_payload(self.input, self.output, self.pin), 34 | QrCodeOpts::Img => generate_qr(self.input, self.output, self.pin), 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/kedqr/encode/payload.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::kedqr::generate; 2 | use catalyst_toolbox::kedqr::QrPin; 3 | use chain_crypto::bech32::Bech32; 4 | use chain_crypto::{Ed25519Extended, SecretKey}; 5 | use color_eyre::Report; 6 | use std::fs::File; 7 | use std::io::Write; 8 | use std::{ 9 | fs::OpenOptions, 10 | io::{BufRead, BufReader}, 11 | path::PathBuf, 12 | }; 13 | 14 | pub fn generate_payload(input: PathBuf, output: Option, pin: QrPin) -> Result<(), Report> { 15 | // open input key and parse it 16 | let key_file = OpenOptions::new() 17 | .create(false) 18 | .read(true) 19 | .write(false) 20 | .append(false) 21 | .open(&input) 22 | .expect("Could not open input file."); 23 | 24 | let mut reader = BufReader::new(key_file); 25 | let mut key_str = String::new(); 26 | let _key_len = reader 27 | .read_line(&mut key_str) 28 | .expect("Could not read input file."); 29 | let sk = key_str.trim_end().to_string(); 30 | 31 | let secret_key: SecretKey = 32 | SecretKey::try_from_bech32_str(&sk).expect("Malformed secret key."); 33 | // use parsed pin from args 34 | let pwd = pin.password; 35 | // generate qrcode with key and parsed pin 36 | let qr = generate(secret_key, &pwd); 37 | // process output 38 | match output { 39 | Some(path) => { 40 | // save qr code to file, or print to stdout if it fails 41 | let mut file = File::create(path)?; 42 | file.write_all(qr.as_bytes())?; 43 | } 44 | None => { 45 | // prints qr code to stdout when no path is specified 46 | println!("{}", qr); 47 | } 48 | } 49 | Ok(()) 50 | } 51 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/kedqr/info.rs: -------------------------------------------------------------------------------- 1 | use super::QrCodeOpts; 2 | use crate::cli::kedqr::decode::{secret_from_payload, secret_from_qr}; 3 | use catalyst_toolbox::kedqr::QrPin; 4 | use chain_addr::{AddressReadable, Discrimination, Kind}; 5 | use chain_core::property::Deserialize; 6 | use chain_crypto::{Ed25519Extended, SecretKey}; 7 | use chain_impl_mockchain::block::Block; 8 | use chain_ser::packer::Codec; 9 | use color_eyre::Report; 10 | use jormungandr_lib::interfaces::{Block0Configuration, Initial}; 11 | use std::path::{Path, PathBuf}; 12 | use structopt::StructOpt; 13 | use url::Url; 14 | #[derive(StructOpt, Debug)] 15 | pub struct InfoForQrCodeCmd { 16 | /// Path to file containing img or payload. 17 | #[structopt(short, long, parse(from_os_str))] 18 | input: PathBuf, 19 | 20 | /// Pin code. 4-digit number is used on Catalyst. 21 | #[structopt(short, long, parse(try_from_str))] 22 | pin: QrPin, 23 | 24 | /// Blockchain block0. Can be either url of local file path 25 | #[structopt(long = "block0")] 26 | pub block0: Option, 27 | 28 | /// Set the discrimination type to testing (default is production). 29 | #[structopt(short, long)] 30 | pub testing: bool, 31 | 32 | #[structopt(flatten)] 33 | opts: QrCodeOpts, 34 | } 35 | 36 | impl InfoForQrCodeCmd { 37 | pub fn exec(self) -> Result<(), Report> { 38 | let secret_key: SecretKey = { 39 | match self.opts { 40 | QrCodeOpts::Payload => secret_from_payload(&self.input, self.pin)?, 41 | QrCodeOpts::Img => secret_from_qr(&self.input, self.pin)?, 42 | } 43 | }; 44 | let kind = Kind::Account(secret_key.to_public()); 45 | let address = chain_addr::Address(test_discrimination(self.testing), kind); 46 | 47 | if let Some(block0_path) = &self.block0 { 48 | let block = { 49 | if Path::new(block0_path).exists() { 50 | let reader = std::fs::OpenOptions::new() 51 | .create(false) 52 | .write(false) 53 | .read(true) 54 | .append(false) 55 | .open(block0_path)?; 56 | Block::deserialize(&mut Codec::new(reader))? 57 | } else if Url::parse(block0_path).is_ok() { 58 | let response = reqwest::blocking::get(block0_path)?; 59 | let block0_bytes = response.bytes()?.to_vec(); 60 | Block::deserialize(&mut Codec::new(&block0_bytes[..]))? 61 | } else { 62 | panic!("invalid block0: should be either path to filesystem or url "); 63 | } 64 | }; 65 | let genesis = Block0Configuration::from_block(&block)?; 66 | 67 | for initial in genesis.initial.iter() { 68 | if let Initial::Fund(initial_utxos) = initial { 69 | if let Some(entry) = initial_utxos 70 | .iter() 71 | .find(|x| x.address == address.clone().into()) 72 | { 73 | println!( 74 | "Address corresponding to input qr found in block0: '{}' with value: '{}'", 75 | AddressReadable::from_address(&test_prefix(self.testing),&address), entry.value); 76 | return Ok(()); 77 | } 78 | } 79 | } 80 | eprintln!("Address corresponding to input qr not found in block0"); 81 | } else { 82 | println!( 83 | "Address: {}", 84 | AddressReadable::from_address(&test_prefix(self.testing), &address) 85 | ); 86 | } 87 | Ok(()) 88 | } 89 | } 90 | 91 | pub fn test_discrimination(testing: bool) -> Discrimination { 92 | match testing { 93 | false => Discrimination::Production, 94 | true => Discrimination::Test, 95 | } 96 | } 97 | 98 | pub fn test_prefix(testing: bool) -> String { 99 | match test_discrimination(testing) { 100 | Discrimination::Production => "ca".to_string(), 101 | Discrimination::Test => "ta".to_string(), 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/kedqr/mod.rs: -------------------------------------------------------------------------------- 1 | mod decode; 2 | mod encode; 3 | mod info; 4 | mod verify; 5 | 6 | use color_eyre::Report; 7 | use structopt::StructOpt; 8 | 9 | #[derive(StructOpt)] 10 | #[structopt(rename_all = "kebab-case")] 11 | pub enum QrCodeCmd { 12 | /// Encode qr code 13 | Encode(encode::EncodeQrCodeCmd), 14 | /// Decode qr code 15 | Decode(decode::DecodeQrCodeCmd), 16 | /// Prints information for qr code 17 | Info(info::InfoForQrCodeCmd), 18 | /// Validates qr code 19 | Verify(verify::VerifyQrCodeCmd), 20 | } 21 | 22 | impl QrCodeCmd { 23 | pub fn exec(self) -> Result<(), Report> { 24 | match self { 25 | Self::Encode(encode) => encode.exec()?, 26 | Self::Decode(decode) => decode.exec()?, 27 | Self::Info(info) => info.exec()?, 28 | Self::Verify(verify) => verify.exec()?, 29 | }; 30 | Ok(()) 31 | } 32 | } 33 | 34 | #[derive(Debug, PartialEq, Eq, StructOpt)] 35 | #[structopt(rename_all = "kebab-case")] 36 | pub enum QrCodeOpts { 37 | Img, 38 | Payload, 39 | } 40 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/kedqr/verify.rs: -------------------------------------------------------------------------------- 1 | use crate::cli::kedqr::decode::secret_from_payload; 2 | use crate::cli::kedqr::decode::secret_from_qr; 3 | use crate::cli::kedqr::QrCodeOpts; 4 | use catalyst_toolbox::kedqr::PinReadMode; 5 | use color_eyre::eyre::Context; 6 | use color_eyre::Report; 7 | use std::path::PathBuf; 8 | use structopt::StructOpt; 9 | 10 | #[derive(StructOpt, Debug)] 11 | pub struct VerifyQrCodeCmd { 12 | #[structopt(long = "folder", required_unless = "file")] 13 | pub folder: Option, 14 | 15 | #[structopt(long = "file", required_unless = "folder")] 16 | pub file: Option, 17 | 18 | #[structopt(short, long, default_value = "1234")] 19 | pub pin: String, 20 | 21 | #[structopt(long = "pin-from-file")] 22 | pub read_pin_from_filename: bool, 23 | 24 | #[structopt(short = "s", long = "stop-at-fail")] 25 | pub stop_at_fail: bool, 26 | 27 | #[structopt(flatten)] 28 | opts: QrCodeOpts, 29 | } 30 | 31 | impl VerifyQrCodeCmd { 32 | pub fn exec(&self) -> Result<(), Report> { 33 | let qr_codes: Vec = { 34 | if let Some(file) = &self.file { 35 | vec![file.to_path_buf()] 36 | } else { 37 | std::fs::read_dir(&self.folder.as_ref().unwrap()) 38 | .unwrap() 39 | .into_iter() 40 | .map(|x| x.unwrap().path()) 41 | .collect() 42 | } 43 | }; 44 | 45 | let mut failed_count = 0; 46 | 47 | for (idx, qr_code) in qr_codes.iter().enumerate() { 48 | let pin = { 49 | if self.read_pin_from_filename { 50 | PinReadMode::FromFileName(qr_code.clone()) 51 | } else { 52 | PinReadMode::Global(self.pin.to_string()) 53 | } 54 | } 55 | .into_qr_pin()?; 56 | 57 | let result = match self.opts { 58 | QrCodeOpts::Payload => secret_from_payload(qr_code, pin), 59 | QrCodeOpts::Img => secret_from_qr(qr_code, pin), 60 | }; 61 | 62 | if let Err(err) = result { 63 | if self.stop_at_fail { 64 | let qr_path = qr_code.to_path_buf().to_string_lossy().to_string(); 65 | let index = idx + 1; 66 | return Err(err).context(format!("qr_code: {qr_path}, index: {index}")); 67 | } else { 68 | failed_count += 1; 69 | } 70 | } 71 | } 72 | println!( 73 | "{} QR read. {} succesfull, {} failed", 74 | qr_codes.len(), 75 | qr_codes.len() - failed_count, 76 | failed_count 77 | ); 78 | Ok(()) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/logs/compare.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::logs::compare::{compare_logs, LogCmpStats}; 2 | use catalyst_toolbox::logs::sentry::{RawLog, SentryFragmentLog}; 3 | use chain_core::property::Fragment; 4 | use color_eyre::Report; 5 | use jcli_lib::utils::io; 6 | use jormungandr_lib::interfaces::{ 7 | load_persistent_fragments_logs_from_folder_path, PersistentFragmentLog, 8 | }; 9 | use serde::de::DeserializeOwned; 10 | use std::path::PathBuf; 11 | use structopt::StructOpt; 12 | 13 | #[derive(StructOpt)] 14 | #[structopt(rename_all = "kebab-case")] 15 | pub struct Compare { 16 | #[structopt(long)] 17 | sentry_logs: PathBuf, 18 | 19 | #[structopt(long)] 20 | permanent_logs: PathBuf, 21 | } 22 | 23 | impl Compare { 24 | pub fn exec(self) -> Result<(), Report> { 25 | let Self { 26 | sentry_logs, 27 | permanent_logs, 28 | } = self; 29 | let sentry_logs: Vec = load_logs_from_file(sentry_logs)?; 30 | 31 | let sentry_logs_data: Vec = sentry_logs 32 | .iter() 33 | .enumerate() 34 | .filter_map( 35 | |(i, raw_log)| match raw_log.get("message").and_then(|v| v.as_str()) { 36 | None => { 37 | // if we could deserialize should be safe to re-serialize it again 38 | eprintln!( 39 | "couldn't load sentry log for entry {}: {}", 40 | i, 41 | serde_json::to_string(raw_log).unwrap() 42 | ); 43 | None 44 | } 45 | Some(value) => match value.parse::() { 46 | Ok(log) => Some(log), 47 | Err(e) => { 48 | eprintln!( 49 | "couldn't load sentry log for entry {} with message '{}' due to: {:?}", 50 | i, value, e 51 | ); 52 | None 53 | } 54 | }, 55 | }, 56 | ) 57 | .collect(); 58 | 59 | let permanent_logs_data: Vec = 60 | load_persistent_fragments_logs_from_folder_path(&permanent_logs)? 61 | .enumerate() 62 | .filter_map(|(i, res)| match res { 63 | Ok(log) => Some(log), 64 | Err(e) => { 65 | eprintln!( 66 | "Error deserializing persistent fragment log entry {}: {:?}", 67 | i, e 68 | ); 69 | None 70 | } 71 | }) 72 | .collect(); 73 | 74 | let cmp_result = compare_logs(&sentry_logs_data, &permanent_logs_data); 75 | print_results(&cmp_result); 76 | Ok(()) 77 | } 78 | } 79 | 80 | pub fn load_logs_from_file(path: PathBuf) -> Result, Report> { 81 | let reader = io::open_file_read(&Some(path))?; 82 | Ok(serde_json::from_reader(reader)?) 83 | } 84 | 85 | pub fn print_results(results: &LogCmpStats) { 86 | let LogCmpStats { 87 | sentry_logs_size, 88 | fragment_logs_size, 89 | duplicated_sentry_logs, 90 | duplicated_fragment_logs, 91 | fragment_ids_differ, 92 | unhandled_fragment_logs, 93 | } = results; 94 | for (unhandled_fragment, e) in unhandled_fragment_logs { 95 | eprintln!( 96 | "unable to load fragment information from fragment id {} due to: {:?}", 97 | unhandled_fragment.id(), 98 | e 99 | ); 100 | } 101 | println!("Sentry logs size {}", sentry_logs_size); 102 | println!("Fragment logs size {}", fragment_logs_size); 103 | println!("Duplicated sentry logs {}", duplicated_sentry_logs); 104 | println!("Duplicated fragments logs {}", duplicated_fragment_logs); 105 | if !fragment_ids_differ.is_empty() { 106 | println!("Non matching (sentry over persistent logs) fragment id's:"); 107 | for id in fragment_ids_differ { 108 | println!("\t{}", id); 109 | } 110 | } else { 111 | println!("All fragment ids match (sentry over persistent logs)"); 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/logs/mod.rs: -------------------------------------------------------------------------------- 1 | mod compare; 2 | mod sentry; 3 | 4 | use color_eyre::Report; 5 | use structopt::StructOpt; 6 | 7 | #[derive(StructOpt)] 8 | #[structopt(rename_all = "kebab-case")] 9 | pub enum Logs { 10 | /// Operate over sentry logs 11 | Sentry(sentry::SentryLogs), 12 | /// Compare Sentry and Persistent fragment logs 13 | Compare(compare::Compare), 14 | } 15 | 16 | impl Logs { 17 | pub fn exec(self) -> Result<(), Report> { 18 | match self { 19 | Logs::Sentry(sentry_logs) => sentry_logs.exec()?, 20 | Logs::Compare(compare) => compare.exec()?, 21 | }; 22 | Ok(()) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/logs/sentry/mod.rs: -------------------------------------------------------------------------------- 1 | mod download; 2 | mod stats; 3 | 4 | use color_eyre::Report; 5 | use structopt::StructOpt; 6 | 7 | #[derive(StructOpt)] 8 | #[structopt(rename_all = "kebab-case")] 9 | pub enum SentryLogs { 10 | /// Download logs from sentry 11 | Download(download::Download), 12 | /// Stats report about logs 13 | Stats(stats::Stats), 14 | } 15 | 16 | impl SentryLogs { 17 | pub fn exec(self) -> Result<(), Report> { 18 | match self { 19 | SentryLogs::Download(download) => download.exec(), 20 | SentryLogs::Stats(stats) => stats.exec(), 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/logs/sentry/stats.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::logs::sentry::{ 2 | RawLog, RegexMatch, SentryLogsStatChecker, SentryLogsStatsExecutor, Stat, 3 | }; 4 | use color_eyre::Report; 5 | use jcli_lib::utils::io::open_file_read; 6 | 7 | use regex::Regex; 8 | use std::path::PathBuf; 9 | use structopt::StructOpt; 10 | 11 | #[derive(StructOpt)] 12 | #[structopt(rename_all = "kebab-case")] 13 | pub struct Scans { 14 | /// Report total successful scans 15 | #[structopt(long)] 16 | scans_ok: bool, 17 | 18 | /// Report total malformed QRs 19 | #[structopt(long)] 20 | malformed_qr: bool, 21 | } 22 | 23 | #[derive(StructOpt)] 24 | #[structopt(rename_all = "kebab-case")] 25 | pub struct Matches { 26 | #[structopt(long, requires("re"))] 27 | key: Option, 28 | #[structopt(long)] 29 | re: Option, 30 | } 31 | 32 | #[derive(StructOpt)] 33 | #[structopt(rename_all = "kebab-case")] 34 | pub struct Stats { 35 | /// Path to the input file 36 | #[structopt(long)] 37 | file: PathBuf, 38 | 39 | /// Report all default stats, overrides single stats options 40 | #[structopt(long)] 41 | all: bool, 42 | 43 | #[structopt(flatten)] 44 | scans: Scans, 45 | 46 | #[structopt(flatten)] 47 | matches: Matches, 48 | } 49 | 50 | impl Scans { 51 | pub fn build_checkers(&self, checkers: &mut Vec, all: bool) { 52 | if self.scans_ok || all { 53 | checkers.push(SentryLogsStatChecker::SuccessfulScans(Default::default())); 54 | } 55 | if self.malformed_qr || all { 56 | checkers.push(SentryLogsStatChecker::MalformedQr(Default::default())); 57 | } 58 | } 59 | } 60 | 61 | impl Matches { 62 | pub fn build_checkers(&self, checkers: &mut Vec, _all: bool) { 63 | if self.key.is_some() { 64 | checkers.push(SentryLogsStatChecker::RegexMatch(RegexMatch::new( 65 | self.re.as_ref().unwrap().clone(), 66 | self.key.as_ref().unwrap().clone(), 67 | ))) 68 | } 69 | } 70 | } 71 | 72 | impl Stats { 73 | fn build_checkers(&self) -> SentryLogsStatsExecutor { 74 | let mut checkers = Vec::new(); 75 | self.scans.build_checkers(&mut checkers, self.all); 76 | self.matches.build_checkers(&mut checkers, self.all); 77 | SentryLogsStatsExecutor::new(checkers) 78 | } 79 | 80 | pub fn exec(self) -> Result<(), Report> { 81 | let mut checker = self.build_checkers(); 82 | let logs_reader = open_file_read(&Some(self.file))?; 83 | let logs: Vec = serde_json::from_reader(logs_reader)?; 84 | checker.process_raw_logs(logs.iter()); 85 | println!("{}", checker); 86 | Ok(()) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/mod.rs: -------------------------------------------------------------------------------- 1 | mod advisor_reviews; 2 | mod archive; 3 | mod ideascale; 4 | mod kedqr; 5 | mod logs; 6 | mod notifications; 7 | mod recovery; 8 | mod rewards; 9 | mod snapshot; 10 | mod stats; 11 | mod vote_check; 12 | 13 | use color_eyre::Report; 14 | use structopt::StructOpt; 15 | 16 | #[derive(StructOpt)] 17 | #[structopt(rename_all = "kebab-case")] 18 | pub struct Cli { 19 | /// display full version details (software version, source version, targets and compiler used) 20 | #[structopt(long = "full-version")] 21 | full_version: bool, 22 | 23 | /// display the sources version, allowing to check the source's hash used to compile this executable. 24 | /// this option is useful for scripting retrieving the logs of the version of this application. 25 | #[structopt(long = "source-version")] 26 | source_version: bool, 27 | 28 | #[structopt(subcommand)] 29 | command: Option, 30 | } 31 | 32 | #[allow(clippy::large_enum_variant)] 33 | #[derive(StructOpt)] 34 | #[structopt(rename_all = "kebab-case")] 35 | pub enum CatalystCommand { 36 | /// Rewards related operations 37 | Rewards(rewards::Rewards), 38 | /// Send push notification to pushwoosh service 39 | Push(notifications::PushNotifications), 40 | /// Tally recovery utility 41 | Recover(recovery::Recover), 42 | /// Download, compare and get stats from sentry and persistent fragment logs 43 | Logs(logs::Logs), 44 | /// Generate qr codes 45 | QrCode(kedqr::QrCodeCmd), 46 | /// Interact with the Ideascale API 47 | Ideascale(ideascale::Ideascale), 48 | /// Advisor reviews related operations 49 | Reviews(advisor_reviews::Reviews), 50 | /// Dump information related to catalyst fund 51 | Archive(archive::Archive), 52 | /// Validate catalyst elections 53 | VoteCheck(vote_check::VoteCheck), 54 | /// Prints voting statistics 55 | Stats(stats::Stats), 56 | /// Process raw registrations to produce initial blockchain setup 57 | Snapshot(snapshot::SnapshotCmd), 58 | } 59 | 60 | impl Cli { 61 | pub fn exec(self) -> Result<(), Report> { 62 | use std::io::Write as _; 63 | if self.full_version { 64 | Ok(writeln!(std::io::stdout(), "{}", env!("FULL_VERSION"))?) 65 | } else if self.source_version { 66 | Ok(writeln!(std::io::stdout(), "{}", env!("SOURCE_VERSION"))?) 67 | } else if let Some(cmd) = self.command { 68 | cmd.exec() 69 | } else { 70 | writeln!(std::io::stderr(), "No command, try `--help'")?; 71 | std::process::exit(1); 72 | } 73 | } 74 | } 75 | 76 | impl CatalystCommand { 77 | pub fn exec(self) -> Result<(), Report> { 78 | use self::CatalystCommand::*; 79 | match self { 80 | Rewards(rewards) => rewards.exec()?, 81 | Push(notifications) => notifications.exec()?, 82 | Recover(recover) => recover.exec()?, 83 | Logs(logs) => logs.exec()?, 84 | QrCode(kedqr) => kedqr.exec()?, 85 | Ideascale(ideascale) => ideascale.exec()?, 86 | Reviews(reviews) => reviews.exec()?, 87 | Archive(archive) => archive.exec()?, 88 | VoteCheck(vote_check) => vote_check.exec()?, 89 | Stats(stats) => stats.exec()?, 90 | Snapshot(snapshot) => snapshot.exec()?, 91 | }; 92 | Ok(()) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/notifications/api_params.rs: -------------------------------------------------------------------------------- 1 | use reqwest::Url; 2 | use structopt::StructOpt; 3 | 4 | pub const DEFAULT_PUSHWOOSH_API_URL: &str = "https://cp.pushwoosh.com/json/1.3/"; 5 | 6 | #[derive(StructOpt)] 7 | #[structopt(rename_all = "kebab-case")] 8 | pub struct ApiParams { 9 | #[structopt(long, default_value = DEFAULT_PUSHWOOSH_API_URL)] 10 | pub api_url: Url, 11 | #[structopt(long)] 12 | pub access_token: String, 13 | } 14 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/notifications/mod.rs: -------------------------------------------------------------------------------- 1 | mod api_params; 2 | mod send; 3 | 4 | use color_eyre::Report; 5 | use structopt::StructOpt; 6 | 7 | #[derive(StructOpt)] 8 | #[structopt(rename_all = "kebab-case")] 9 | pub enum PushNotifications { 10 | Send(send::SendNotification), 11 | } 12 | 13 | impl PushNotifications { 14 | pub fn exec(self) -> Result<(), Report> { 15 | use self::PushNotifications::*; 16 | match self { 17 | Send(cmd) => cmd.exec()?, 18 | }; 19 | Ok(()) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/recovery/mod.rs: -------------------------------------------------------------------------------- 1 | mod tally; 2 | mod votes; 3 | 4 | use color_eyre::Report; 5 | use structopt::StructOpt; 6 | 7 | #[derive(StructOpt)] 8 | #[structopt(rename_all = "kebab-case")] 9 | pub enum Recover { 10 | Tally(tally::ReplayCli), 11 | VotesPrintout(votes::VotesPrintout), 12 | } 13 | 14 | impl Recover { 15 | pub fn exec(self) -> Result<(), Report> { 16 | match self { 17 | Recover::Tally(cmd) => cmd.exec(), 18 | Recover::VotesPrintout(cmd) => cmd.exec(), 19 | } 20 | } 21 | } 22 | 23 | fn set_verbosity(verbosity: usize) { 24 | if verbosity > 0 { 25 | std::env::set_var( 26 | "RUST_LOG", 27 | match verbosity { 28 | 0 => unreachable!(), 29 | 1 => "warn", 30 | 2 => "info", 31 | 3 => "debug", 32 | _ => "trace", 33 | }, 34 | ) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/recovery/tally.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::recovery::Replay; 2 | use chain_core::{packer::Codec, property::Deserialize}; 3 | use chain_impl_mockchain::block::Block; 4 | use color_eyre::{ 5 | eyre::{bail, Context}, 6 | Report, 7 | }; 8 | use jcli_lib::utils::{output_file::OutputFile, output_format::OutputFormat}; 9 | 10 | use std::path::PathBuf; 11 | 12 | use reqwest::Url; 13 | use structopt::StructOpt; 14 | 15 | use super::set_verbosity; 16 | 17 | /// Recover the tally from fragment log files and the initial preloaded block0 binary file. 18 | #[derive(StructOpt)] 19 | #[structopt(rename_all = "kebab")] 20 | pub struct ReplayCli { 21 | /// Path to the block0 binary file 22 | #[structopt(long, conflicts_with = "block0-url")] 23 | block0_path: Option, 24 | 25 | /// Url to a block0 endpoint 26 | #[structopt(long)] 27 | block0_url: Option, 28 | 29 | /// Path to the folder containing the log files used for the tally reconstruction 30 | #[structopt(long)] 31 | logs_path: PathBuf, 32 | 33 | #[structopt(flatten)] 34 | output: OutputFile, 35 | 36 | #[structopt(flatten)] 37 | output_format: OutputFormat, 38 | 39 | /// Verbose mode (-v, -vv, -vvv, etc) 40 | #[structopt(short = "v", long = "verbose", parse(from_occurrences))] 41 | verbose: usize, 42 | } 43 | 44 | fn read_block0(path: PathBuf) -> Result { 45 | let reader = std::fs::File::open(path)?; 46 | Block::deserialize(&mut Codec::new(reader)).context("block0 loading") 47 | } 48 | 49 | fn load_block0_from_url(url: Url) -> Result { 50 | let block0_body = reqwest::blocking::get(url)?.bytes()?; 51 | Block::deserialize(&mut Codec::new(&block0_body[..])).context("block0 loading") 52 | } 53 | 54 | impl ReplayCli { 55 | pub fn exec(self) -> Result<(), Report> { 56 | let Self { 57 | block0_path, 58 | block0_url, 59 | logs_path, 60 | output, 61 | output_format, 62 | verbose, 63 | } = self; 64 | 65 | set_verbosity(verbose); 66 | 67 | let block0 = if let Some(path) = block0_path { 68 | read_block0(path)? 69 | } else if let Some(url) = block0_url { 70 | load_block0_from_url(url)? 71 | } else { 72 | bail!("block0 unavailable"); 73 | }; 74 | 75 | let replay = Replay::new(block0, logs_path, output, output_format); 76 | replay.exec().map_err(Into::into) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/rewards/dreps.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::rewards::voters::calc_voter_rewards; 2 | use catalyst_toolbox::rewards::{Rewards, Threshold}; 3 | use color_eyre::Report; 4 | use jcli_lib::jcli_lib::block::Common; 5 | use jormungandr_lib::{crypto::account::Identifier, interfaces::AccountVotes}; 6 | use snapshot_lib::{registration::MainnetRewardAddress, SnapshotInfo}; 7 | use structopt::StructOpt; 8 | use vit_servicing_station_lib::db::models::proposals::FullProposalInfo; 9 | 10 | use std::collections::{BTreeMap, HashMap}; 11 | use std::path::PathBuf; 12 | 13 | #[derive(StructOpt)] 14 | #[structopt(rename_all = "kebab-case")] 15 | pub struct DrepsRewards { 16 | #[structopt(flatten)] 17 | common: Common, 18 | /// Reward (in dollars) to be distributed proportionally to delegated stake with respect to total stake. 19 | /// The total amount will only be awarded if dreps control all of the stake. 20 | #[structopt(long)] 21 | total_rewards: u64, 22 | 23 | /// Path to a json encoded list of `SnapshotInfo` 24 | #[structopt(long)] 25 | snapshot_info_path: PathBuf, 26 | 27 | /// Path to a json-encoded list of proposal every user has voted for. 28 | /// This can be retrived from the v1/account-votes-all endpoint exposed 29 | /// by a Jormungandr node. 30 | #[structopt(long)] 31 | votes_count_path: PathBuf, 32 | 33 | /// Number of global votes required to be able to receive voter rewards 34 | #[structopt(long, default_value)] 35 | vote_threshold: u64, 36 | 37 | /// Path to a json-encoded map from challenge id to an optional required threshold 38 | /// per-challenge in order to receive rewards. 39 | #[structopt(long)] 40 | per_challenge_threshold: Option, 41 | 42 | /// Path to the list of proposals active in this election. 43 | /// Can be obtained from /api/v0/proposals. 44 | #[structopt(long)] 45 | proposals: PathBuf, 46 | } 47 | 48 | fn write_rewards_results( 49 | common: Common, 50 | rewards: &BTreeMap, 51 | ) -> Result<(), Report> { 52 | let writer = common.open_output()?; 53 | let header = ["Address", "Reward for the voter (lovelace)"]; 54 | let mut csv_writer = csv::Writer::from_writer(writer); 55 | csv_writer.write_record(&header)?; 56 | 57 | for (address, rewards) in rewards.iter() { 58 | let record = [address.to_string(), rewards.trunc().to_string()]; 59 | csv_writer.write_record(&record)?; 60 | } 61 | 62 | Ok(()) 63 | } 64 | 65 | impl DrepsRewards { 66 | pub fn exec(self) -> Result<(), Report> { 67 | let DrepsRewards { 68 | common, 69 | total_rewards, 70 | snapshot_info_path, 71 | votes_count_path, 72 | vote_threshold, 73 | per_challenge_threshold, 74 | proposals, 75 | } = self; 76 | 77 | let proposals = serde_json::from_reader::<_, Vec>( 78 | jcli_lib::utils::io::open_file_read(&Some(proposals))?, 79 | )?; 80 | 81 | let vote_count = super::extract_individual_votes( 82 | proposals.clone(), 83 | serde_json::from_reader::<_, HashMap>>( 84 | jcli_lib::utils::io::open_file_read(&Some(votes_count_path))?, 85 | )?, 86 | )?; 87 | 88 | let snapshot: Vec = serde_json::from_reader( 89 | jcli_lib::utils::io::open_file_read(&Some(snapshot_info_path))?, 90 | )?; 91 | 92 | let additional_thresholds: HashMap = if let Some(file) = per_challenge_threshold 93 | { 94 | serde_json::from_reader(jcli_lib::utils::io::open_file_read(&Some(file))?)? 95 | } else { 96 | HashMap::new() 97 | }; 98 | 99 | let results = calc_voter_rewards( 100 | vote_count, 101 | snapshot, 102 | Threshold::new( 103 | vote_threshold 104 | .try_into() 105 | .expect("vote threshold is too big"), 106 | additional_thresholds, 107 | proposals, 108 | )?, 109 | Rewards::from(total_rewards), 110 | )?; 111 | 112 | write_rewards_results(common, &results)?; 113 | Ok(()) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/rewards/full/config.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | 3 | use rust_decimal::Decimal; 4 | use serde::Deserialize; 5 | 6 | use crate::cli::rewards::community_advisors::{FundSettingOpt, ProposalRewardsSlotsOpt}; 7 | 8 | #[derive(Debug, Deserialize)] 9 | pub(super) struct Config { 10 | pub(super) inputs: Inputs, 11 | pub(super) outputs: Outputs, 12 | pub(super) params: Params, 13 | } 14 | 15 | #[derive(Debug, Deserialize)] 16 | pub(super) struct Inputs { 17 | pub(super) block_file: PathBuf, 18 | pub(super) snapshot_path: PathBuf, 19 | pub(super) vote_count_path: PathBuf, 20 | pub(super) reviews_csv: PathBuf, 21 | pub(super) assessments_path: PathBuf, 22 | pub(super) proposal_bonus_output: Option, 23 | pub(super) approved_proposals_path: PathBuf, 24 | pub(super) active_voteplans: PathBuf, 25 | pub(super) challenges: PathBuf, 26 | pub(super) proposals_path: PathBuf, 27 | pub(super) committee_keys: PathBuf, 28 | pub(super) excluded_proposals: Option, 29 | } 30 | 31 | #[derive(Debug, Deserialize)] 32 | pub(super) struct Outputs { 33 | pub(super) voter_rewards_output: PathBuf, 34 | pub(super) veterans_rewards_output: PathBuf, 35 | pub(super) ca_rewards_output: PathBuf, 36 | pub(super) proposer_rewards_output: PathBuf, 37 | } 38 | 39 | #[derive(Debug, Deserialize)] 40 | pub(super) struct Params { 41 | pub(super) voter_params: VoterParams, 42 | pub(super) proposer_params: ProposerParams, 43 | pub(super) ca_params: CaParams, 44 | pub(super) vca_params: VcaParams, 45 | } 46 | 47 | #[derive(Debug, Deserialize)] 48 | pub(super) struct VoterParams { 49 | pub(super) total_rewards: u64, 50 | pub(super) vote_threshold: u64, 51 | } 52 | 53 | #[derive(Debug, Deserialize)] 54 | pub(super) struct ProposerParams { 55 | pub(super) stake_threshold: f64, 56 | pub(super) approval_threshold: f64, 57 | } 58 | 59 | #[derive(Debug, Deserialize)] 60 | pub(super) struct CaParams { 61 | pub(super) rewards_slots: ProposalRewardsSlotsOpt, 62 | pub(super) fund_settings: FundSettingOpt, 63 | pub(super) seed: String, 64 | } 65 | 66 | #[derive(Debug, Deserialize)] 67 | pub(super) struct VcaParams { 68 | pub(super) total_rewards: u64, 69 | pub(super) rewards_agreement_rate_cutoffs: Vec, 70 | pub(super) rewards_agreement_rate_modifiers: Vec, 71 | pub(super) reputation_agreement_rate_cutoffs: Vec, 72 | pub(super) reputation_agreement_rate_modifiers: Vec, 73 | pub(super) min_rankings: usize, 74 | pub(super) max_rankings_reputation: usize, 75 | pub(super) max_rankings_rewards: usize, 76 | } 77 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/rewards/full/mod.rs: -------------------------------------------------------------------------------- 1 | use std::{fs::File, path::Path}; 2 | 3 | use catalyst_toolbox::{ 4 | http::HttpClient, 5 | rewards::proposers::{OutputFormat, ProposerRewards}, 6 | }; 7 | use color_eyre::Result; 8 | use config::*; 9 | use serde_json::from_reader; 10 | use tracing::info; 11 | 12 | mod config; 13 | 14 | pub(super) fn full_rewards(path: &Path) -> Result<()> { 15 | let config = from_reader(File::open(path)?)?; 16 | let Config { 17 | inputs: 18 | Inputs { 19 | block_file, 20 | vote_count_path, 21 | snapshot_path, 22 | reviews_csv, 23 | assessments_path, 24 | proposal_bonus_output, 25 | approved_proposals_path, 26 | active_voteplans, 27 | challenges, 28 | proposals_path, 29 | committee_keys, 30 | excluded_proposals, 31 | }, 32 | outputs: 33 | Outputs { 34 | voter_rewards_output, 35 | veterans_rewards_output, 36 | ca_rewards_output, 37 | proposer_rewards_output, 38 | }, 39 | params: 40 | Params { 41 | voter_params, 42 | proposer_params, 43 | ca_params, 44 | vca_params, 45 | }, 46 | } = config; 47 | 48 | info!("calculating voter rewards"); 49 | super::voters::voter_rewards( 50 | &voter_rewards_output, 51 | &vote_count_path, 52 | &snapshot_path, 53 | voter_params.vote_threshold, 54 | voter_params.total_rewards, 55 | )?; 56 | 57 | info!("calculating vca rewards"); 58 | super::veterans::vca_rewards( 59 | reviews_csv, 60 | veterans_rewards_output, 61 | vca_params.rewards_agreement_rate_cutoffs, 62 | vca_params.rewards_agreement_rate_modifiers, 63 | vca_params.reputation_agreement_rate_cutoffs, 64 | vca_params.reputation_agreement_rate_modifiers, 65 | vca_params.total_rewards.into(), 66 | vca_params.min_rankings, 67 | vca_params.max_rankings_reputation, 68 | vca_params.max_rankings_rewards, 69 | )?; 70 | 71 | info!("calculating ca rewards"); 72 | super::community_advisors::ca_rewards( 73 | assessments_path, 74 | approved_proposals_path, 75 | ca_params.fund_settings, 76 | ca_params.rewards_slots, 77 | ca_rewards_output, 78 | ca_params.seed, 79 | proposal_bonus_output, 80 | )?; 81 | 82 | info!("calculating proposer rewards"); 83 | super::proposers::rewards( 84 | &ProposerRewards { 85 | output: proposer_rewards_output, 86 | block0: block_file, 87 | total_stake_threshold: proposer_params.stake_threshold, 88 | approval_threshold: proposer_params.approval_threshold, 89 | proposals: Some(proposals_path), 90 | active_voteplans: Some(active_voteplans), 91 | challenges: Some(challenges), 92 | committee_keys: Some(committee_keys), 93 | excluded_proposals, 94 | output_format: OutputFormat::Csv, 95 | vit_station_url: "not used".into(), 96 | }, 97 | &PanickingHttpClient, 98 | )?; 99 | 100 | Ok(()) 101 | } 102 | 103 | struct PanickingHttpClient; 104 | 105 | impl HttpClient for PanickingHttpClient { 106 | fn get(&self, _path: &str) -> Result> 107 | where 108 | T: for<'a> serde::Deserialize<'a>, 109 | { 110 | unimplemented!("this implementation always panics"); 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/rewards/mod.rs: -------------------------------------------------------------------------------- 1 | mod community_advisors; 2 | mod dreps; 3 | mod full; 4 | mod proposers; 5 | mod veterans; 6 | mod voters; 7 | 8 | use std::{collections::HashMap, path::PathBuf}; 9 | 10 | use catalyst_toolbox::{ 11 | http::default_http_client, 12 | rewards::{proposers as proposers_lib, VoteCount}, 13 | }; 14 | use color_eyre::{eyre::eyre, Report}; 15 | use jormungandr_lib::{ 16 | crypto::{account::Identifier, hash::Hash}, 17 | interfaces::AccountVotes, 18 | }; 19 | use structopt::StructOpt; 20 | use vit_servicing_station_lib::db::models::proposals::FullProposalInfo; 21 | 22 | #[derive(StructOpt)] 23 | #[structopt(rename_all = "kebab-case")] 24 | pub enum Rewards { 25 | /// Calculate rewards for voters base on their stake 26 | Voters(voters::VotersRewards), 27 | 28 | /// Calculate rewards for dreps based on their delegated stake 29 | Dreps(dreps::DrepsRewards), 30 | 31 | /// Calculate community advisors rewards 32 | CommunityAdvisors(community_advisors::CommunityAdvisors), 33 | 34 | /// Calculate rewards for veteran community advisors 35 | Veterans(veterans::VeteransRewards), 36 | 37 | /// Calculate full rewards based on a config file 38 | Full { path: PathBuf }, 39 | 40 | /// Calculate rewards for propsers 41 | Proposers(proposers_lib::ProposerRewards), 42 | } 43 | 44 | impl Rewards { 45 | pub fn exec(self) -> Result<(), Report> { 46 | match self { 47 | Rewards::Voters(cmd) => cmd.exec(), 48 | Rewards::CommunityAdvisors(cmd) => cmd.exec(), 49 | Rewards::Veterans(cmd) => cmd.exec(), 50 | Rewards::Dreps(cmd) => cmd.exec(), 51 | Rewards::Full { path } => full::full_rewards(&path), 52 | Rewards::Proposers(proposers) => { 53 | proposers::rewards(&proposers, &default_http_client(None)) 54 | } 55 | } 56 | } 57 | } 58 | 59 | fn extract_individual_votes( 60 | proposals: Vec, 61 | votes: HashMap>, 62 | ) -> Result { 63 | let proposals_per_voteplan = 64 | proposals 65 | .into_iter() 66 | .fold(>>::new(), |mut acc, prop| { 67 | let entry = acc 68 | .entry(prop.voteplan.chain_voteplan_id.clone()) 69 | .or_default(); 70 | entry.push(prop); 71 | entry.sort_by_key(|p| p.voteplan.chain_proposal_index); 72 | acc 73 | }); 74 | 75 | votes 76 | .into_iter() 77 | .try_fold(VoteCount::new(), |mut acc, (account, votes)| { 78 | for vote in &votes { 79 | let voteplan = vote.vote_plan_id; 80 | let props = proposals_per_voteplan 81 | .get(&voteplan.to_string()) 82 | .iter() 83 | .flat_map(|p| p.iter()) 84 | .enumerate() 85 | .filter(|(i, _p)| vote.votes.contains(&(*i as u8))) 86 | .map(|(_, p)| { 87 | Ok::<_, Report>(Hash::from( 88 | <[u8; 32]>::try_from(p.proposal.chain_proposal_id.clone()).map_err( 89 | |v| eyre!("Invalid proposal hash length {}, expected 32", v.len()), 90 | )?, 91 | )) 92 | }) 93 | .collect::, _>>()?; 94 | acc.entry(account.clone()).or_default().extend(props); 95 | } 96 | Ok::<_, Report>(acc) 97 | }) 98 | } 99 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/rewards/proposers/mod.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashSet, fs::File}; 2 | 3 | use catalyst_toolbox::{ 4 | http::HttpClient, 5 | rewards::proposers::{ 6 | io::{load_data, write_results}, 7 | proposer_rewards, ProposerRewards, ProposerRewardsInputs, 8 | }, 9 | }; 10 | use color_eyre::eyre::Result; 11 | 12 | pub fn rewards( 13 | ProposerRewards { 14 | output, 15 | block0, 16 | proposals, 17 | excluded_proposals, 18 | active_voteplans, 19 | challenges, 20 | committee_keys, 21 | total_stake_threshold, 22 | approval_threshold, 23 | output_format, 24 | vit_station_url, 25 | }: &ProposerRewards, 26 | http: &impl HttpClient, 27 | ) -> Result<()> { 28 | let (proposals, voteplans, challenges) = load_data( 29 | http, 30 | vit_station_url, 31 | proposals.as_deref(), 32 | active_voteplans.as_deref(), 33 | challenges.as_deref(), 34 | )?; 35 | 36 | let block0_config = serde_yaml::from_reader(File::open(block0)?)?; 37 | 38 | let excluded_proposals = match excluded_proposals { 39 | Some(path) => serde_json::from_reader(File::open(path)?)?, 40 | None => HashSet::new(), 41 | }; 42 | let committee_keys = match committee_keys { 43 | Some(path) => serde_json::from_reader(File::open(path)?)?, 44 | None => vec![], 45 | }; 46 | 47 | let results = proposer_rewards(ProposerRewardsInputs { 48 | block0_config, 49 | proposals, 50 | voteplans, 51 | challenges, 52 | excluded_proposals, 53 | committee_keys, 54 | total_stake_threshold: *total_stake_threshold, 55 | approval_threshold: *approval_threshold, 56 | })?; 57 | 58 | write_results(output, *output_format, results)?; 59 | 60 | Ok(()) 61 | } 62 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/rewards/proposers/util.rs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/input-output-hk/catalyst-toolbox/b9910d772186dddd3593dd8b315ed38e6d7f4d15/catalyst-toolbox/src/bin/cli/rewards/proposers/util.rs -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/rewards/voters.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::rewards::voters::calc_voter_rewards; 2 | use catalyst_toolbox::rewards::{Rewards, Threshold, VoteCount}; 3 | use catalyst_toolbox::utils::assert_are_close; 4 | 5 | use color_eyre::eyre::eyre; 6 | use color_eyre::{Report, Result}; 7 | use jcli_lib::block::open_output; 8 | use jcli_lib::jcli_lib::block::Common; 9 | 10 | use snapshot_lib::registration::MainnetRewardAddress; 11 | use snapshot_lib::SnapshotInfo; 12 | use structopt::StructOpt; 13 | 14 | use std::collections::BTreeMap; 15 | use std::path::{Path, PathBuf}; 16 | 17 | #[derive(StructOpt)] 18 | #[structopt(rename_all = "kebab-case")] 19 | pub struct VotersRewards { 20 | #[structopt(flatten)] 21 | common: Common, 22 | /// Reward (in LOVELACE) to be distributed 23 | #[structopt(long)] 24 | total_rewards: u64, 25 | 26 | /// Path to a json encoded list of `SnapshotInfo` 27 | #[structopt(long)] 28 | snapshot_info_path: PathBuf, 29 | 30 | /// Path to a json-encoded list of VotePlanStatusFull to consider for voters 31 | /// participation in the election. 32 | /// This can be retrived from the v1/vote/active/plans/full endpoint exposed 33 | /// by a Jormungandr node. 34 | #[structopt(long)] 35 | votes_count_path: PathBuf, 36 | 37 | /// Number of global votes required to be able to receive voter rewards 38 | #[structopt(long, default_value)] 39 | vote_threshold: u64, 40 | } 41 | 42 | fn write_rewards_results( 43 | common: &Option, 44 | rewards: &BTreeMap, 45 | ) -> Result<(), Report> { 46 | let writer = open_output(common)?; 47 | let header = ["Address", "Reward for the voter (lovelace)"]; 48 | let mut csv_writer = csv::Writer::from_writer(writer); 49 | csv_writer.write_record(&header)?; 50 | 51 | for (address, rewards) in rewards.iter() { 52 | let record = [address.to_string(), rewards.trunc().to_string()]; 53 | csv_writer.write_record(&record)?; 54 | } 55 | 56 | Ok(()) 57 | } 58 | 59 | impl VotersRewards { 60 | pub fn exec(self) -> Result<(), Report> { 61 | let VotersRewards { 62 | common, 63 | total_rewards, 64 | snapshot_info_path, 65 | votes_count_path, 66 | vote_threshold, 67 | } = self; 68 | 69 | voter_rewards( 70 | common 71 | .output_file 72 | .as_deref() 73 | .ok_or(eyre!("missing output file"))?, 74 | &votes_count_path, 75 | &snapshot_info_path, 76 | vote_threshold, 77 | total_rewards, 78 | ) 79 | } 80 | } 81 | 82 | pub fn voter_rewards( 83 | output: &Path, 84 | votes_count_path: &Path, 85 | snapshot_path: &Path, 86 | vote_threshold: u64, 87 | total_rewards: u64, 88 | ) -> Result<()> { 89 | let vote_count: VoteCount = serde_json::from_reader(jcli_lib::utils::io::open_file_read( 90 | &Some(votes_count_path), 91 | )?)?; 92 | 93 | let snapshot: Vec = 94 | serde_json::from_reader(jcli_lib::utils::io::open_file_read(&Some(snapshot_path))?)?; 95 | 96 | let results = calc_voter_rewards( 97 | vote_count, 98 | snapshot, 99 | Threshold::new( 100 | vote_threshold as usize, 101 | Default::default(), 102 | Default::default(), 103 | )?, 104 | Rewards::from(total_rewards), 105 | )?; 106 | 107 | let actual_rewards = results.values().sum::(); 108 | assert_are_close(actual_rewards, Rewards::from(total_rewards)); 109 | 110 | write_rewards_results(&Some(output.to_path_buf()), &results)?; 111 | Ok(()) 112 | } 113 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/snapshot/mod.rs: -------------------------------------------------------------------------------- 1 | use color_eyre::Report; 2 | use jcli_lib::utils::{output_file::OutputFile, output_format::OutputFormat}; 3 | use jormungandr_lib::interfaces::Value; 4 | use snapshot_lib::Fraction; 5 | use snapshot_lib::{ 6 | voting_group::{RepsVotersAssigner, DEFAULT_DIRECT_VOTER_GROUP, DEFAULT_REPRESENTATIVE_GROUP}, 7 | RawSnapshot, Snapshot, 8 | }; 9 | use std::fs::File; 10 | use std::io::Write; 11 | use std::path::PathBuf; 12 | use structopt::StructOpt; 13 | 14 | /// Process raw registrations into blockchain initials 15 | #[derive(StructOpt)] 16 | #[structopt(rename_all = "kebab-case")] 17 | pub struct SnapshotCmd { 18 | /// Path to the file containing all CIP-15 compatible registrations in json format. 19 | #[structopt(short, long, parse(from_os_str))] 20 | snapshot: PathBuf, 21 | /// Registrations voting power threshold for eligibility 22 | #[structopt(short, long)] 23 | min_stake_threshold: Value, 24 | 25 | /// Voter group to assign direct voters to. 26 | /// If empty, defaults to "voter" 27 | #[structopt(short, long)] 28 | direct_voters_group: Option, 29 | 30 | /// Voter group to assign representatives to. 31 | /// If empty, defaults to "rep" 32 | #[structopt(long)] 33 | representatives_group: Option, 34 | 35 | /// Voting power cap for each account 36 | #[structopt(short, long)] 37 | voting_power_cap: Fraction, 38 | 39 | #[structopt(flatten)] 40 | output: OutputFile, 41 | 42 | #[structopt(flatten)] 43 | output_format: OutputFormat, 44 | } 45 | 46 | impl SnapshotCmd { 47 | pub fn exec(self) -> Result<(), Report> { 48 | let raw_snapshot: RawSnapshot = serde_json::from_reader(File::open(&self.snapshot)?)?; 49 | let direct_voter = self 50 | .direct_voters_group 51 | .unwrap_or_else(|| DEFAULT_DIRECT_VOTER_GROUP.into()); 52 | let representative = self 53 | .representatives_group 54 | .unwrap_or_else(|| DEFAULT_REPRESENTATIVE_GROUP.into()); 55 | let assigner = RepsVotersAssigner::new(direct_voter, representative); 56 | let initials = Snapshot::from_raw_snapshot( 57 | raw_snapshot, 58 | self.min_stake_threshold, 59 | self.voting_power_cap, 60 | &assigner, 61 | )? 62 | .to_full_snapshot_info(); 63 | let mut out_writer = self.output.open()?; 64 | let content = self 65 | .output_format 66 | .format_json(serde_json::to_value(initials)?)?; 67 | out_writer.write_all(content.as_bytes())?; 68 | Ok(()) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/stats/archive.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::{ 2 | stats::archive::{load_from_csv, load_from_folder, ArchiveStats}, 3 | utils::csv::dump_to_csv_or_print, 4 | }; 5 | use color_eyre::Report; 6 | use std::collections::BTreeMap; 7 | use std::fmt::Debug; 8 | use std::path::PathBuf; 9 | use structopt::StructOpt; 10 | 11 | #[derive(StructOpt, Debug)] 12 | pub struct ArchiveCommand { 13 | #[structopt(long = "csv", required_unless = "folder")] 14 | pub csv: Option, 15 | 16 | #[structopt(long = "folder", required_unless = "csv")] 17 | pub folder: Option, 18 | 19 | #[structopt(long = "output")] 20 | pub output: Option, 21 | 22 | #[structopt(short = "d", long = "distribution")] 23 | pub calculate_distribution: bool, 24 | 25 | #[structopt(subcommand)] 26 | pub command: Command, 27 | } 28 | 29 | impl ArchiveCommand { 30 | pub fn exec(self) -> Result<(), Report> { 31 | let archiver: ArchiveStats = { 32 | if let Some(csv) = &self.csv { 33 | load_from_csv(&csv)?.into() 34 | } else if let Some(folder) = &self.folder { 35 | load_from_folder(&folder)?.into() 36 | } else { 37 | panic!("no csv nor folder defined"); 38 | } 39 | }; 40 | 41 | match &self.command { 42 | Command::VotesByCaster => { 43 | let result = archiver.number_of_votes_per_caster(); 44 | if self.calculate_distribution { 45 | let dist = ArchiveStats::calculate_distribution(&result); 46 | dump_to_csv_or_print(self.output, dist.values())?; 47 | } else { 48 | dump_to_csv_or_print(self.output, result.values())?; 49 | } 50 | } 51 | Command::VotesBySlot => { 52 | let result = archiver.number_of_tx_per_slot(); 53 | if self.calculate_distribution { 54 | let dist = ArchiveStats::calculate_distribution(&result); 55 | dump_to_csv_or_print(self.output, dist.values())?; 56 | } else { 57 | dump_to_csv_or_print(self.output, result.values())?; 58 | } 59 | } 60 | Command::BatchSizeByCaster(batch_size_by_caster) => { 61 | let result = batch_size_by_caster.exec(archiver)?; 62 | if self.calculate_distribution { 63 | let dist = ArchiveStats::calculate_distribution(&result); 64 | dump_to_csv_or_print(self.output, dist.values())?; 65 | } else { 66 | dump_to_csv_or_print(self.output, result.values())?; 67 | } 68 | } 69 | }; 70 | 71 | Ok(()) 72 | } 73 | } 74 | 75 | #[derive(StructOpt, Debug)] 76 | pub enum Command { 77 | VotesByCaster, 78 | VotesBySlot, 79 | BatchSizeByCaster(BatchSizeByCaster), 80 | } 81 | 82 | #[derive(StructOpt, Debug)] 83 | pub struct BatchSizeByCaster { 84 | #[structopt(short = "s", long = "slots-in-epoch")] 85 | pub slots_in_epoch: u32, 86 | } 87 | 88 | impl BatchSizeByCaster { 89 | pub fn exec(&self, archiver: ArchiveStats) -> Result, Report> { 90 | Ok(archiver.max_batch_size_per_caster(self.slots_in_epoch)?) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/stats/live.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::stats::live::{start, Harvester, Settings}; 2 | use color_eyre::Report; 3 | use jortestkit::console::ProgressBarMode; 4 | use jortestkit::prelude::parse_progress_bar_mode_from_str; 5 | use std::path::PathBuf; 6 | use structopt::StructOpt; 7 | 8 | /// Commands connect to desired backend and query endpoint with some interval. 9 | /// It can dump result to to console in a progress mode/standard printout or file. 10 | #[derive(StructOpt, Debug)] 11 | pub struct LiveStatsCommand { 12 | #[structopt(long = "endpoint")] 13 | pub endpoint: String, 14 | #[structopt(long = "interval")] 15 | pub interval: u64, 16 | #[structopt( 17 | long = "progress-bar-mode", 18 | default_value = "Monitor", 19 | parse(from_str = parse_progress_bar_mode_from_str) 20 | )] 21 | pub console: ProgressBarMode, 22 | #[structopt(long = "logger")] 23 | pub file: Option, 24 | #[structopt(long = "duration")] 25 | pub duration: u64, 26 | } 27 | 28 | impl LiveStatsCommand { 29 | pub fn exec(&self) -> Result<(), Report> { 30 | let settings = Settings { 31 | endpoint: self.endpoint.clone(), 32 | progress: self.console, 33 | interval: self.interval, 34 | logger: self.file.clone(), 35 | duration: self.duration, 36 | }; 37 | 38 | let harvester = Harvester::new( 39 | self.endpoint.clone(), 40 | std::time::Duration::from_secs(self.interval), 41 | ); 42 | 43 | start( 44 | harvester, 45 | settings, 46 | &format!("{} monitoring", self.endpoint), 47 | ) 48 | .map_err(Into::into) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/stats/mod.rs: -------------------------------------------------------------------------------- 1 | mod archive; 2 | mod live; 3 | mod snapshot; 4 | mod voters; 5 | 6 | use archive::ArchiveCommand; 7 | use color_eyre::Report; 8 | use live::LiveStatsCommand; 9 | use snapshot::SnapshotCommand; 10 | use structopt::StructOpt; 11 | use voters::VotersCommand; 12 | 13 | #[derive(StructOpt, Debug)] 14 | pub enum Stats { 15 | Voters(VotersCommand), 16 | Live(LiveStatsCommand), 17 | Archive(ArchiveCommand), 18 | Snapshot(SnapshotCommand), 19 | } 20 | 21 | impl Stats { 22 | pub fn exec(self) -> Result<(), Report> { 23 | match self { 24 | Self::Voters(voters) => voters.exec(), 25 | Self::Live(live) => live.exec(), 26 | Self::Archive(archive) => archive.exec(), 27 | Self::Snapshot(snapshot) => snapshot.exec(), 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/stats/snapshot.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::stats::distribution::Stats; 2 | use catalyst_toolbox::stats::snapshot::read_initials; 3 | use catalyst_toolbox::stats::voters::calculate_wallet_distribution_from_initials; 4 | use color_eyre::Report; 5 | use jormungandr_lib::interfaces::Initial; 6 | use std::path::PathBuf; 7 | use structopt::StructOpt; 8 | #[derive(StructOpt, Debug)] 9 | pub struct SnapshotCommand { 10 | #[structopt(long = "support-lovelace")] 11 | pub support_lovelace: bool, 12 | #[structopt(name = "SNAPSHOT")] 13 | pub snapshot: PathBuf, 14 | #[structopt(long = "threshold")] 15 | pub threshold: u64, 16 | #[structopt(subcommand)] 17 | pub command: Command, 18 | } 19 | 20 | #[derive(StructOpt, Debug)] 21 | pub enum Command { 22 | Count, 23 | Ada, 24 | } 25 | 26 | impl SnapshotCommand { 27 | pub fn exec(&self) -> Result<(), Report> { 28 | let initials: Vec = read_initials(&jortestkit::file::read_file(&self.snapshot)?)?; 29 | 30 | match self.command { 31 | Command::Count => calculate_wallet_distribution_from_initials( 32 | Stats::new(self.threshold)?, 33 | initials, 34 | vec![], 35 | self.support_lovelace, 36 | |stats, _, _| stats.add(1), 37 | )? 38 | .print_count_per_level(), 39 | Command::Ada => calculate_wallet_distribution_from_initials( 40 | Stats::new(self.threshold)?, 41 | initials, 42 | vec![], 43 | self.support_lovelace, 44 | |stats, value, _| stats.add(value), 45 | )? 46 | .print_ada_per_level(), 47 | }; 48 | 49 | Ok(()) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/stats/voters/active.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::stats::distribution::Stats; 2 | use catalyst_toolbox::stats::voters::calculate_active_wallet_distribution; 3 | use color_eyre::Report; 4 | use std::ops::Range; 5 | use std::path::PathBuf; 6 | use structopt::StructOpt; 7 | use thiserror::Error; 8 | 9 | #[derive(StructOpt, Debug)] 10 | pub struct ActiveVotersCommand { 11 | #[structopt(long = "support-lovelace")] 12 | pub support_lovelace: bool, 13 | #[structopt(long = "block0")] 14 | pub block0: String, 15 | #[structopt(long = "threshold")] 16 | pub threshold: u64, 17 | #[structopt(long = "votes-count-file")] 18 | pub votes_count_path: PathBuf, 19 | #[structopt(long = "votes-count-levels")] 20 | pub votes_count_levels: Option, 21 | #[structopt(subcommand)] 22 | pub command: Command, 23 | } 24 | 25 | #[derive(StructOpt, Debug)] 26 | pub enum Command { 27 | Count, 28 | Ada, 29 | Votes, 30 | } 31 | 32 | impl ActiveVotersCommand { 33 | pub fn exec(&self) -> Result<(), Report> { 34 | match self.command { 35 | Command::Count => calculate_active_wallet_distribution( 36 | Stats::new(self.threshold)?, 37 | &self.block0, 38 | &self.votes_count_path, 39 | self.support_lovelace, 40 | |stats, value, _| stats.add(value), 41 | )? 42 | .print_count_per_level(), 43 | Command::Ada => calculate_active_wallet_distribution( 44 | Stats::new(self.threshold)?, 45 | &self.block0, 46 | &self.votes_count_path, 47 | self.support_lovelace, 48 | |stats, value, _| stats.add(value), 49 | )? 50 | .print_ada_per_level(), 51 | Command::Votes => calculate_active_wallet_distribution( 52 | Stats::new_with_levels(get_casted_votes_levels(&self.votes_count_levels)?), 53 | &self.block0, 54 | &self.votes_count_path, 55 | self.support_lovelace, 56 | |stats, _, weight| stats.add_with_weight(1, weight as u32), 57 | )? 58 | .print_count_per_level(), 59 | }; 60 | 61 | Ok(()) 62 | } 63 | } 64 | 65 | fn get_casted_votes_levels(path: &Option) -> Result>, Error> { 66 | if let Some(path) = path { 67 | serde_json::from_reader(jcli_lib::utils::io::open_file_read(&Some(path))?) 68 | .map_err(Into::into) 69 | } else { 70 | Ok(default_casted_votes_levels()) 71 | } 72 | } 73 | 74 | fn default_casted_votes_levels() -> Vec> { 75 | vec![ 76 | (1..5), 77 | (5..10), 78 | (10..20), 79 | (20..50), 80 | (50..100), 81 | (100..200), 82 | (200..400), 83 | (400..800), 84 | (800..5_000), 85 | ] 86 | } 87 | 88 | #[allow(clippy::large_enum_variant)] 89 | #[derive(Error, Debug)] 90 | pub enum Error { 91 | #[error(transparent)] 92 | Io(#[from] std::io::Error), 93 | #[error(transparent)] 94 | Serde(#[from] serde_json::Error), 95 | } 96 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/stats/voters/initials.rs: -------------------------------------------------------------------------------- 1 | use catalyst_toolbox::stats::distribution::Stats; 2 | use catalyst_toolbox::stats::voters::calculate_wallet_distribution; 3 | use color_eyre::Report; 4 | use structopt::StructOpt; 5 | 6 | #[derive(StructOpt, Debug)] 7 | pub struct InitialVotersCommand { 8 | #[structopt(long = "support-lovelace")] 9 | pub support_lovelace: bool, 10 | #[structopt(long = "block0")] 11 | pub block0: String, 12 | #[structopt(long = "threshold")] 13 | pub threshold: u64, 14 | #[structopt(subcommand)] 15 | pub command: Command, 16 | } 17 | 18 | #[derive(StructOpt, Debug)] 19 | pub enum Command { 20 | Count, 21 | Ada, 22 | } 23 | 24 | impl InitialVotersCommand { 25 | pub fn exec(&self) -> Result<(), Report> { 26 | match self.command { 27 | Command::Count => calculate_wallet_distribution( 28 | &self.block0, 29 | Stats::new(self.threshold)?, 30 | self.support_lovelace, 31 | |stats, value, _| stats.add(value), 32 | )? 33 | .print_count_per_level(), 34 | Command::Ada => calculate_wallet_distribution( 35 | &self.block0, 36 | Stats::new(self.threshold)?, 37 | self.support_lovelace, 38 | |stats, value, _| stats.add(value), 39 | )? 40 | .print_ada_per_level(), 41 | }; 42 | 43 | Ok(()) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/stats/voters/mod.rs: -------------------------------------------------------------------------------- 1 | mod active; 2 | mod initials; 3 | 4 | use active::ActiveVotersCommand; 5 | use color_eyre::Report; 6 | use initials::InitialVotersCommand; 7 | use structopt::StructOpt; 8 | 9 | #[derive(StructOpt, Debug)] 10 | pub enum VotersCommand { 11 | Initials(InitialVotersCommand), 12 | Active(ActiveVotersCommand), 13 | } 14 | 15 | impl VotersCommand { 16 | pub fn exec(self) -> Result<(), Report> { 17 | match self { 18 | Self::Initials(initials) => initials.exec(), 19 | Self::Active(active) => active.exec(), 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/bin/cli/vote_check/mod.rs: -------------------------------------------------------------------------------- 1 | use color_eyre::eyre::bail; 2 | use color_eyre::Report; 3 | use structopt::StructOpt; 4 | 5 | use catalyst_toolbox::vote_check::CheckNode; 6 | 7 | use jormungandr_lib::interfaces::VotePlanStatus; 8 | 9 | use std::fs::File; 10 | use std::path::PathBuf; 11 | 12 | /// Verify that your votes were correctly tallied. 13 | /// 14 | /// Requires Jormungandr to be installed in the system 15 | #[derive(Debug, PartialEq, Eq, StructOpt)] 16 | #[structopt(rename_all = "kebab-case")] 17 | pub struct VoteCheck { 18 | /// Path to folder containing the full blockchain history saved in Jormungandr 19 | /// storage format. 20 | #[structopt(short, long, parse(from_os_str))] 21 | blockchain: PathBuf, 22 | /// Genesis block hash 23 | #[structopt(short, long)] 24 | genesis_block_hash: String, 25 | /// Ids of the transactions to check 26 | #[structopt(short, long)] 27 | transactions: Vec, 28 | /// Path to the expected results of the election, in Json format as returned by the /vote/active/plans endpoint 29 | #[structopt(short, long)] 30 | expected_results: PathBuf, 31 | /// Path to the Jormungandr binary. If not provided, will look for 'jormungandr' in PATH 32 | #[structopt(short, long)] 33 | jormungandr_bin: Option, 34 | } 35 | 36 | impl VoteCheck { 37 | /// Vote verification follows this plan: 38 | /// * Start a new node with the storage containing the full blockchain history to validate 39 | /// that all ledger operations. 40 | /// * Check that the election results obtained are the same as provided 41 | /// * Check that the transactions containing your votes were indeed included in a block 42 | /// in the main chain 43 | /// 44 | pub fn exec(self) -> Result<(), Report> { 45 | let node = CheckNode::spawn( 46 | self.blockchain.clone(), 47 | self.genesis_block_hash.clone(), 48 | self.jormungandr_bin, 49 | )?; 50 | 51 | let expected_results: Vec = 52 | serde_json::from_reader(File::open(self.expected_results)?)?; 53 | let actual_results = node.active_vote_plans()?; 54 | 55 | for vote_plan in expected_results { 56 | if !actual_results.contains(&vote_plan) { 57 | let expected = serde_json::to_string_pretty(&vote_plan).unwrap(); 58 | let actual = actual_results 59 | .iter() 60 | .find(|act| act.id == vote_plan.id) 61 | .map(|act| serde_json::to_string_pretty(act).unwrap()) 62 | .unwrap_or_default(); 63 | 64 | bail!("results do not match, expected: {expected:?}, actual: {actual:?}"); 65 | } 66 | } 67 | 68 | node.check_transactions_on_chain(self.transactions)?; 69 | 70 | println!("Vote(s) correctly validated!"); 71 | 72 | Ok(()) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/community_advisors/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod models; 2 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/community_advisors/models/mod.rs: -------------------------------------------------------------------------------- 1 | mod de; 2 | 3 | use serde::{Deserialize, Deserializer}; 4 | 5 | pub use de::{ 6 | AdvisorReviewId, AdvisorReviewRow, ReviewRanking, VeteranAdvisorId, VeteranRankingRow, 7 | }; 8 | 9 | pub enum ProposalStatus { 10 | Approved, 11 | NotApproved, 12 | } 13 | 14 | #[derive(Deserialize)] 15 | pub struct ApprovedProposalRow { 16 | #[serde(rename(deserialize = "internal_id"))] 17 | pub proposal_id: String, 18 | #[serde(rename(deserialize = "meets_approval_threshold"))] 19 | pub status: ProposalStatus, 20 | pub requested_dollars: String, 21 | } 22 | 23 | impl<'de> Deserialize<'de> for ProposalStatus { 24 | fn deserialize(deserializer: D) -> Result 25 | where 26 | D: Deserializer<'de>, 27 | { 28 | let status: String = String::deserialize(deserializer)?; 29 | Ok(match status.to_lowercase().as_ref() { 30 | "yes" => ProposalStatus::Approved, 31 | _ => ProposalStatus::NotApproved, 32 | }) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/http/mock.rs: -------------------------------------------------------------------------------- 1 | use color_eyre::Result; 2 | use reqwest::StatusCode; 3 | 4 | use super::{HttpClient, HttpResponse}; 5 | 6 | pub enum Method { 7 | Get, 8 | Post, 9 | Put, 10 | Delete, 11 | Head, 12 | Connect, 13 | Options, 14 | Trace, 15 | Path, 16 | } 17 | 18 | #[non_exhaustive] 19 | pub struct Spec<'a> { 20 | pub method: Method, 21 | pub path: &'a str, 22 | } 23 | 24 | pub struct MockClient { 25 | handler: fn(&Spec) -> (String, StatusCode), 26 | } 27 | 28 | impl MockClient { 29 | pub fn new(handler: fn(&Spec) -> (String, StatusCode)) -> Self { 30 | Self { handler } 31 | } 32 | } 33 | 34 | impl HttpClient for MockClient { 35 | fn get(&self, path: &str) -> Result> 36 | where 37 | T: for<'a> serde::Deserialize<'a>, 38 | { 39 | let spec = Spec { 40 | method: Method::Get, 41 | path, 42 | }; 43 | 44 | let (body, status) = (self.handler)(&spec); 45 | Ok(HttpResponse { 46 | body, 47 | status, 48 | _marker: std::marker::PhantomData, 49 | }) 50 | } 51 | } 52 | 53 | #[test] 54 | fn example_usage() { 55 | fn function_that_calls_api(client: &T) -> Result { 56 | let response = client.get("/example")?; 57 | response.json() 58 | } 59 | 60 | let mock_client = MockClient::new(|spec| match spec { 61 | Spec { 62 | method: Method::Get, 63 | path: "/example", 64 | } => ("123".to_string(), StatusCode::OK), 65 | _ => ("not found".to_string(), StatusCode::NOT_FOUND), 66 | }); 67 | 68 | let response = function_that_calls_api(&mock_client).unwrap(); 69 | assert_eq!(response, 123); 70 | } 71 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/http/mod.rs: -------------------------------------------------------------------------------- 1 | use std::marker::PhantomData; 2 | 3 | use ::reqwest::StatusCode; 4 | use color_eyre::eyre::Result; 5 | use serde::Deserialize; 6 | use tracing::warn; 7 | 8 | use self::{rate_limit::RateLimitClient, reqwest::ReqwestClient}; 9 | 10 | #[cfg(test)] 11 | pub mod mock; 12 | mod rate_limit; 13 | mod reqwest; 14 | 15 | const RATE_LIMIT_ENV_VAR: &str = "CATALYST_RATE_LIMIT_MS"; 16 | 17 | pub fn default_http_client(api_key: Option<&str>) -> impl HttpClient { 18 | let rate_limit = match std::env::var(RATE_LIMIT_ENV_VAR).map(|s| s.parse::()) { 19 | Ok(Ok(rate_limit)) => rate_limit, 20 | Ok(Err(_)) => { 21 | warn!( 22 | "{} could not be parsed as a u64, defaulting to no rate-limiting", 23 | RATE_LIMIT_ENV_VAR 24 | ); 25 | 0 26 | } 27 | _ => 0, 28 | }; 29 | RateLimitClient::new(ReqwestClient::new(api_key), rate_limit) 30 | } 31 | 32 | #[cfg(test)] 33 | #[allow(unused)] 34 | fn test_default_client_send_sync() { 35 | fn check(_t: T) {} 36 | check(default_http_client(None)); 37 | } 38 | 39 | /// Types which can make HTTP requests 40 | pub trait HttpClient: Send + Sync + 'static { 41 | fn get(&self, path: &str) -> Result> 42 | where 43 | T: for<'a> Deserialize<'a>; 44 | } 45 | 46 | /// A value returned from a HTTP method 47 | pub struct HttpResponse Deserialize<'a>> { 48 | _marker: PhantomData, 49 | body: String, 50 | status: StatusCode, 51 | } 52 | 53 | impl Deserialize<'a>> HttpResponse { 54 | pub fn json(self) -> Result { 55 | Ok(serde_json::from_str(&self.body)?) 56 | } 57 | 58 | pub fn status(&self) -> StatusCode { 59 | self.status 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/http/rate_limit.rs: -------------------------------------------------------------------------------- 1 | use std::time::{Duration, Instant}; 2 | 3 | use governor::{ 4 | clock::DefaultClock, 5 | state::{InMemoryState, NotKeyed}, 6 | Quota, RateLimiter, 7 | }; 8 | 9 | use color_eyre::Report; 10 | use serde::Deserialize; 11 | use tracing::debug; 12 | 13 | use super::{HttpClient, HttpResponse}; 14 | 15 | #[derive(Debug)] 16 | pub struct RateLimitClient { 17 | inner: T, 18 | limiter: Option>, 19 | } 20 | 21 | impl RateLimitClient { 22 | pub fn new(inner: T, request_interval_ms: u64) -> Self { 23 | let limiter = if request_interval_ms == 0 { 24 | None 25 | } else { 26 | let quota = Quota::with_period(Duration::from_millis(request_interval_ms)).unwrap(); 27 | Some(RateLimiter::direct(quota)) 28 | }; 29 | Self { inner, limiter } 30 | } 31 | } 32 | 33 | impl HttpClient for RateLimitClient { 34 | fn get(&self, path: &str) -> Result, Report> 35 | where 36 | S: for<'a> Deserialize<'a>, 37 | { 38 | if let Some(limiter) = &self.limiter { 39 | while let Err(e) = limiter.check() { 40 | let time = e.wait_time_from(Instant::now()); 41 | debug!("waiting for rate limit: {time:?}"); 42 | std::thread::sleep(time); 43 | } 44 | } 45 | self.inner.get(path) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/http/reqwest.rs: -------------------------------------------------------------------------------- 1 | use color_eyre::eyre::Result; 2 | use reqwest::{ 3 | blocking::Client, 4 | header::{HeaderMap, HeaderValue}, 5 | Url, 6 | }; 7 | use serde::Deserialize; 8 | 9 | use super::{HttpClient, HttpResponse}; 10 | 11 | const BASE_IDEASCALE_URL: &str = "https://cardano.ideascale.com/a/rest/v1/"; 12 | 13 | #[derive(Debug)] 14 | pub struct ReqwestClient { 15 | client: Client, 16 | base_url: Url, 17 | } 18 | 19 | impl ReqwestClient { 20 | pub fn new(api_key: Option<&str>) -> Self { 21 | let mut client = Client::builder(); 22 | 23 | if let Some(api_key) = api_key { 24 | let mut headers = HeaderMap::new(); 25 | let mut auth_value = HeaderValue::from_str(api_key).unwrap(); 26 | auth_value.set_sensitive(true); 27 | headers.insert("api_token", auth_value); 28 | client = client.default_headers(headers); 29 | }; 30 | 31 | let client = client.build().unwrap(); 32 | 33 | Self { 34 | client, 35 | base_url: BASE_IDEASCALE_URL.try_into().unwrap(), 36 | } 37 | } 38 | } 39 | 40 | impl HttpClient for ReqwestClient { 41 | fn get(&self, path: &str) -> Result> 42 | where 43 | T: for<'a> Deserialize<'a>, 44 | { 45 | let url = self.base_url.join(path.as_ref())?; 46 | let response = self.client.get(url).send()?; 47 | let status = response.status(); 48 | 49 | Ok(HttpResponse { 50 | _marker: std::marker::PhantomData, 51 | status, 52 | body: response.text()?, 53 | }) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/ideascale/fetch.rs: -------------------------------------------------------------------------------- 1 | use crate::http::HttpClient; 2 | use crate::ideascale::models::de::{Fund, Funnel, Proposal, Stage}; 3 | 4 | use color_eyre::Report; 5 | use reqwest::StatusCode; 6 | use std::collections::HashMap; 7 | 8 | pub type Scores = HashMap; 9 | pub type Sponsors = HashMap; 10 | 11 | pub fn get_funds_data(client: &impl HttpClient) -> Result, Report> { 12 | info!("getting funds"); 13 | client.get("campaigns/groups")?.json() 14 | } 15 | 16 | pub fn get_stages(client: &impl HttpClient) -> Result, Report> { 17 | client.get("stages")?.json() 18 | } 19 | 20 | /// we test token by running lightweight query and observe response code 21 | pub fn is_token_valid(client: &impl HttpClient) -> Result { 22 | Ok(client.get::<()>("profile/avatars")?.status() == StatusCode::OK) 23 | } 24 | 25 | pub fn get_proposals_data( 26 | client: &impl HttpClient, 27 | challenge_id: u32, 28 | ) -> Result, Report> { 29 | info!("getting proposal data"); 30 | let path = &format!("campaigns/{}/ideas/0/100000", challenge_id); 31 | client.get(path)?.json() 32 | } 33 | 34 | pub fn get_funnels_data_for_fund(client: &impl HttpClient) -> Result, Report> { 35 | info!("getting funnels"); 36 | client.get("funnels")?.json() 37 | } 38 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/ideascale/models/custom_fields.rs: -------------------------------------------------------------------------------- 1 | use serde::Deserialize; 2 | 3 | #[derive(Debug, Deserialize)] 4 | pub struct CustomFieldTags { 5 | pub proposer_url: String, 6 | pub proposal_solution: String, 7 | pub proposal_brief: String, 8 | pub proposal_importance: String, 9 | pub proposal_goal: String, 10 | pub proposal_metrics: String, 11 | pub proposal_public_key: String, 12 | pub proposal_funds: Vec, 13 | pub proposal_relevant_experience: String, 14 | pub proposal_why: String, 15 | } 16 | 17 | impl Default for CustomFieldTags { 18 | fn default() -> Self { 19 | Self { 20 | proposer_url: "website_github_repository__not_required_".to_string(), 21 | proposal_solution: "problem_solution".to_string(), 22 | proposal_brief: "challenge_brief".to_string(), 23 | proposal_importance: "importance".to_string(), 24 | proposal_goal: "how_does_success_look_like_".to_string(), 25 | proposal_metrics: "key_metrics_to_measure".to_string(), 26 | proposal_public_key: "ada_payment_address".to_string(), 27 | proposal_funds: vec![ 28 | "requested_funds".to_string(), 29 | "requested_funds_coti".to_string(), 30 | ], 31 | proposal_relevant_experience: "relevant_experience".to_string(), 32 | proposal_why: "importance".to_string(), 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/ideascale/models/de/ada_rewards.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fmt::{Display, Formatter}, 3 | num::ParseIntError, 4 | }; 5 | 6 | use once_cell::sync::Lazy; 7 | use regex::Regex; 8 | use serde::{ 9 | de::{Error, Visitor}, 10 | Deserialize, Deserializer, 11 | }; 12 | 13 | #[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)] 14 | pub struct AdaRewards(pub u64); 15 | 16 | impl<'de> Deserialize<'de> for AdaRewards { 17 | fn deserialize(deserializer: D) -> Result 18 | where 19 | D: Deserializer<'de>, 20 | { 21 | let num = deserializer.deserialize_str(V)?; 22 | Ok(Self(num)) 23 | } 24 | } 25 | 26 | static REGEX: Lazy = Lazy::new(|| Regex::new(r#"\$([0-9]+) in ada"#).unwrap()); 27 | 28 | struct V; 29 | 30 | impl<'a> Visitor<'a> for V { 31 | type Value = u64; 32 | 33 | fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { 34 | formatter.write_str( 35 | "a string of the form: `$N in ada`, where `N` is a u64 (e.g. \"$123 in ada\")", 36 | ) 37 | } 38 | 39 | fn visit_str(self, v: &str) -> Result 40 | where 41 | E: Error, 42 | { 43 | // input is not standarized, hack an early return if it is just 0 ada 44 | if v.starts_with("0 ada") { 45 | return Ok(0); 46 | } 47 | 48 | let bad_pattern = || E::custom("didn't match `$N in ada` pattern"); 49 | let bad_u64 = |e: ParseIntError| E::custom(format!("unvalid u64: {e}")); 50 | 51 | // ignore the first capture, since this is the whole string 52 | let capture = REGEX.captures_iter(v).next().ok_or_else(bad_pattern)?; 53 | let value = capture.get(1).ok_or_else(bad_pattern)?; 54 | value.as_str().parse().map_err(bad_u64) 55 | } 56 | } 57 | 58 | impl From for AdaRewards { 59 | fn from(v: u64) -> Self { 60 | Self(v) 61 | } 62 | } 63 | 64 | impl From for u64 { 65 | fn from(rewards: AdaRewards) -> Self { 66 | rewards.0 67 | } 68 | } 69 | 70 | impl Display for AdaRewards { 71 | fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { 72 | write!(f, "${} in ada", self.0) 73 | } 74 | } 75 | 76 | #[cfg(test)] 77 | mod tests { 78 | use super::*; 79 | 80 | fn parse(s: &str) -> Result { 81 | let s = format!(r#""{s}""#); 82 | serde_json::from_str(&s) 83 | } 84 | 85 | #[test] 86 | fn can_parse_good_values() { 87 | assert_eq!(parse("$123 in ada").unwrap(), AdaRewards(123)); 88 | assert_eq!(parse("0 ada").unwrap(), AdaRewards(0)); 89 | assert_eq!( 90 | parse("0 ada with some stuff at the end").unwrap(), 91 | AdaRewards(0) 92 | ); 93 | } 94 | 95 | #[test] 96 | fn fails_to_parse_bad_values() { 97 | // missing dollar sign 98 | assert!(parse("123 in ada").is_err()); 99 | // negative number 100 | assert!(parse("$-123 in ada").is_err()); 101 | // fraction 102 | assert!(parse("$123.0 in ada").is_err()); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/ideascale/models/de/approval.rs: -------------------------------------------------------------------------------- 1 | use serde::{de::Visitor, Deserialize, Deserializer}; 2 | 3 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] 4 | pub enum Approval { 5 | Approved, 6 | NotApproved, 7 | } 8 | 9 | impl Approval { 10 | pub fn as_bool(&self) -> bool { 11 | (*self).into() 12 | } 13 | } 14 | 15 | impl From for Approval { 16 | fn from(b: bool) -> Self { 17 | match b { 18 | true => Approval::Approved, 19 | false => Approval::NotApproved, 20 | } 21 | } 22 | } 23 | 24 | impl From for bool { 25 | fn from(b: Approval) -> Self { 26 | match b { 27 | Approval::Approved => true, 28 | Approval::NotApproved => false, 29 | } 30 | } 31 | } 32 | 33 | impl<'de> Deserialize<'de> for Approval { 34 | fn deserialize(deserializer: D) -> Result 35 | where 36 | D: Deserializer<'de>, 37 | { 38 | deserializer.deserialize_str(V) 39 | } 40 | } 41 | 42 | struct V; 43 | 44 | impl Visitor<'_> for V { 45 | type Value = Approval; 46 | 47 | fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { 48 | formatter.write_str("a case-insensitive string representing an approval status (with this string \"approved\" meaning `Approved`, and all other strings being `NotApproved`)") 49 | } 50 | 51 | fn visit_str(self, v: &str) -> Result 52 | where 53 | E: serde::de::Error, 54 | { 55 | match v.to_lowercase().as_ref() { 56 | "approved" => Ok(Approval::Approved), 57 | _ => Ok(Approval::NotApproved), 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/ideascale/models/de/challenge_title.rs: -------------------------------------------------------------------------------- 1 | use serde::{de::Visitor, Deserialize}; 2 | 3 | #[derive(Debug, Clone, PartialEq, Eq)] 4 | pub struct ChallengeTitle(String); 5 | 6 | impl<'de> Deserialize<'de> for ChallengeTitle { 7 | fn deserialize(deserializer: D) -> Result 8 | where 9 | D: serde::Deserializer<'de>, 10 | { 11 | deserializer.deserialize_str(V) 12 | } 13 | } 14 | 15 | struct V; 16 | 17 | impl Visitor<'_> for V { 18 | type Value = ChallengeTitle; 19 | 20 | fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { 21 | formatter.write_str( 22 | "a string representing the title of a challenge (ignoring any leading `FX: `)", 23 | ) 24 | } 25 | 26 | fn visit_str(self, v: &str) -> Result 27 | where 28 | E: serde::de::Error, 29 | { 30 | Ok(ChallengeTitle::new(v)) 31 | } 32 | } 33 | 34 | impl ChallengeTitle { 35 | pub fn new(s: &str) -> Self { 36 | let s = s.trim_start_matches("FX: "); 37 | Self(s.to_string()) 38 | } 39 | 40 | pub fn as_str(&self) -> &str { 41 | &self.0 42 | } 43 | } 44 | 45 | impl From for String { 46 | fn from(ChallengeTitle(inner): ChallengeTitle) -> Self { 47 | inner 48 | } 49 | } 50 | 51 | #[cfg(test)] 52 | mod tests { 53 | use serde_json::json; 54 | 55 | use super::*; 56 | 57 | fn parse(s: &str) -> ChallengeTitle { 58 | let json = json!(s); 59 | serde_json::from_value(json).unwrap() 60 | } 61 | 62 | #[test] 63 | fn strips_leading_fx() { 64 | assert_eq!(parse("hello"), ChallengeTitle("hello".into())); 65 | assert_eq!(parse("FX: hello"), ChallengeTitle("hello".into())); 66 | assert_eq!(parse("FX:hello"), ChallengeTitle("FX:hello".into())); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/ideascale/models/de/clean_string.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Formatter; 2 | 3 | use once_cell::sync::Lazy; 4 | use regex::Regex; 5 | use serde::{ 6 | de::{Error, Visitor}, 7 | Deserialize, Deserializer, 8 | }; 9 | 10 | /// A newtype wrapper around `String` 11 | /// 12 | /// When deserialized, the following characters are removed: `-`, `*`, `/` 13 | #[derive(Debug, Clone, PartialEq, Eq)] 14 | pub struct CleanString(pub String); 15 | 16 | impl<'de> Deserialize<'de> for CleanString { 17 | fn deserialize(deserializer: D) -> Result 18 | where 19 | D: Deserializer<'de>, 20 | { 21 | deserializer.deserialize_str(V) 22 | } 23 | } 24 | 25 | struct V; 26 | 27 | impl<'a> Visitor<'a> for V { 28 | type Value = CleanString; 29 | 30 | fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { 31 | formatter.write_str("a string") 32 | } 33 | 34 | fn visit_str(self, v: &str) -> Result 35 | where 36 | E: Error, 37 | { 38 | let s = clean_str(v); 39 | Ok(CleanString(s)) 40 | } 41 | } 42 | 43 | impl From<&str> for CleanString { 44 | fn from(s: &str) -> Self { 45 | CleanString(s.to_string()) 46 | } 47 | } 48 | 49 | impl From for CleanString { 50 | fn from(s: String) -> Self { 51 | CleanString(s) 52 | } 53 | } 54 | 55 | impl ToString for CleanString { 56 | fn to_string(&self) -> String { 57 | self.0.clone() 58 | } 59 | } 60 | 61 | impl AsRef for CleanString { 62 | fn as_ref(&self) -> &str { 63 | &self.0 64 | } 65 | } 66 | 67 | static REGEX: Lazy = Lazy::new(|| Regex::new("[-*/]").unwrap()); 68 | 69 | pub fn clean_str(s: &str) -> String { 70 | REGEX.replace_all(s, "").to_string() 71 | } 72 | 73 | #[cfg(any(test, feature = "property-test-api"))] 74 | mod tests { 75 | use proptest::{ 76 | arbitrary::{Arbitrary, StrategyFor}, 77 | prelude::*, 78 | strategy::Map, 79 | }; 80 | use serde_json::json; 81 | use test_strategy::proptest; 82 | 83 | use super::*; 84 | 85 | impl Arbitrary for CleanString { 86 | type Parameters = (); 87 | type Strategy = Map, fn(String) -> Self>; 88 | 89 | fn arbitrary_with((): Self::Parameters) -> Self::Strategy { 90 | any::().prop_map(|s| CleanString(clean_str(&s))) 91 | } 92 | } 93 | 94 | fn parse(s: &str) -> CleanString { 95 | let s = format!(r#""{s}""#); 96 | serde_json::from_str(&s).unwrap() 97 | } 98 | 99 | #[test] 100 | fn correctly_formats_strings() { 101 | assert_eq!(parse("hello"), CleanString::from("hello")); 102 | assert_eq!(parse("h*e-l/lo"), CleanString::from("hello")); 103 | } 104 | 105 | #[proptest] 106 | fn any_string_deserializes_to_clean_string(s: String) { 107 | let json = json!(s); 108 | let _: CleanString = serde_json::from_value(json).unwrap(); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/ideascale/models/de/mod.rs: -------------------------------------------------------------------------------- 1 | use serde::Deserialize; 2 | 3 | mod ada_rewards; 4 | pub use ada_rewards::AdaRewards; 5 | 6 | mod clean_string; 7 | pub use clean_string::{clean_str, CleanString}; 8 | 9 | mod approval; 10 | pub use approval::Approval; 11 | 12 | mod challenge_title; 13 | pub use challenge_title::ChallengeTitle; 14 | 15 | #[derive(Debug, Deserialize, Clone)] 16 | pub struct Challenge { 17 | pub id: u32, 18 | #[serde(alias = "name")] 19 | pub title: ChallengeTitle, 20 | #[serde(alias = "tagline")] 21 | pub rewards: AdaRewards, 22 | pub description: CleanString, 23 | #[serde(alias = "groupId")] 24 | pub fund_id: u32, 25 | #[serde(alias = "funnelId")] 26 | pub funnel_id: u32, 27 | #[serde(alias = "campaignUrl")] 28 | pub challenge_url: String, 29 | } 30 | 31 | #[derive(Debug, Deserialize, Clone)] 32 | pub struct Funnel { 33 | pub id: u32, 34 | #[serde(alias = "name")] 35 | pub title: CleanString, 36 | pub description: CleanString, 37 | } 38 | 39 | #[derive(Debug, Deserialize, Clone)] 40 | pub struct Fund { 41 | pub id: u32, 42 | pub name: CleanString, 43 | #[serde(alias = "campaigns")] 44 | pub challenges: Vec, 45 | } 46 | 47 | #[derive(Debug, Deserialize, Clone)] 48 | pub struct Proposal { 49 | #[serde(alias = "id")] 50 | pub proposal_id: u32, 51 | pub proposal_category: Option, 52 | #[serde(alias = "title")] 53 | pub proposal_title: CleanString, 54 | #[serde(alias = "text")] 55 | pub proposal_summary: CleanString, 56 | 57 | #[serde(alias = "url")] 58 | pub proposal_url: String, 59 | #[serde(default)] 60 | pub proposal_files_url: String, 61 | 62 | #[serde(alias = "customFieldsByKey")] 63 | pub custom_fields: ProposalCustomFieldsByKey, 64 | 65 | #[serde(alias = "authorInfo")] 66 | pub proposer: Proposer, 67 | 68 | #[serde(alias = "stageId")] 69 | pub stage_id: u32, 70 | 71 | #[serde(alias = "stageLabel")] 72 | pub stage_type: String, 73 | 74 | #[serde(alias = "campaignId")] 75 | pub challenge_id: u32, 76 | 77 | #[serde(alias = "flag")] 78 | pub approved: Approval, 79 | } 80 | 81 | #[derive(Debug, Deserialize, Clone)] 82 | pub struct Proposer { 83 | pub name: String, 84 | #[serde(alias = "email")] 85 | pub contact: String, 86 | } 87 | 88 | #[derive(Debug, Deserialize, Clone)] 89 | pub struct ProposalCustomFieldsByKey { 90 | #[serde(flatten)] 91 | pub fields: serde_json::Value, 92 | } 93 | 94 | #[derive(Debug, Deserialize, Clone)] 95 | pub struct Stage { 96 | #[serde(default)] 97 | pub label: String, 98 | #[serde(alias = "funnelId", default)] 99 | pub funnel_id: u32, 100 | #[serde(alias = "assessmentId", default)] 101 | pub assessment_id: u32, 102 | } 103 | 104 | impl Funnel { 105 | pub fn is_community(&self) -> bool { 106 | self.title.0.contains("Community Setting") 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/ideascale/models/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod custom_fields; 2 | pub mod de; 3 | pub mod se; 4 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/ideascale/models/se.rs: -------------------------------------------------------------------------------- 1 | use serde::Serialize; 2 | 3 | #[derive(Debug, Serialize)] 4 | pub struct Challenge { 5 | pub challenge_type: String, 6 | pub challenge_url: String, 7 | pub description: String, 8 | pub fund_id: String, 9 | pub id: String, 10 | pub rewards_total: String, 11 | pub proposers_rewards: String, 12 | pub title: String, 13 | #[serde(skip_serializing_if = "Option::is_none")] 14 | pub highlight: Option, 15 | } 16 | 17 | #[derive(Debug, Serialize)] 18 | pub struct Highlight { 19 | pub sponsor: String, 20 | } 21 | 22 | #[derive(Debug, Serialize)] 23 | pub struct Fund { 24 | pub id: i32, 25 | pub goal: String, 26 | pub threshold: i64, 27 | pub rewards_info: String, 28 | } 29 | 30 | #[derive(Debug, Serialize)] 31 | pub struct Proposal { 32 | pub category_name: String, 33 | #[serde(default = "default_vote_options")] 34 | pub chain_vote_options: String, 35 | pub challenge_id: String, 36 | pub challenge_type: String, 37 | pub chain_vote_type: String, 38 | pub internal_id: String, 39 | pub proposal_funds: String, 40 | pub proposal_id: String, 41 | pub proposal_impact_score: String, 42 | pub proposal_summary: String, 43 | pub proposal_title: String, 44 | pub proposal_url: String, 45 | pub proposer_email: String, 46 | pub proposer_name: String, 47 | pub proposer_relevant_experience: String, 48 | #[serde(default)] 49 | pub proposer_url: String, 50 | #[serde(skip_serializing_if = "Option::is_none")] 51 | pub proposal_solution: Option, 52 | #[serde(skip_serializing_if = "Option::is_none")] 53 | pub proposal_brief: Option, 54 | #[serde(skip_serializing_if = "Option::is_none")] 55 | pub proposal_importance: Option, 56 | #[serde(skip_serializing_if = "Option::is_none")] 57 | pub proposal_goal: Option, 58 | #[serde(skip_serializing_if = "Option::is_none")] 59 | pub proposal_metrics: Option, 60 | } 61 | 62 | #[allow(dead_code)] 63 | fn default_vote_options() -> &'static str { 64 | "blank,yes,no" 65 | } 66 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/kedqr/img.rs: -------------------------------------------------------------------------------- 1 | use super::payload; 2 | use chain_crypto::{Ed25519Extended, SecretKey, SecretKeyError}; 3 | use image::{DynamicImage, ImageBuffer, ImageError, Luma}; 4 | use qrcode::{ 5 | render::{svg, unicode}, 6 | EcLevel, QrCode, 7 | }; 8 | use std::fmt; 9 | use std::fs::File; 10 | use std::io::{self, prelude::*}; 11 | use std::path::Path; 12 | use symmetric_cipher::Error as SymmetricCipherError; 13 | use thiserror::Error; 14 | 15 | pub struct KeyQrCode { 16 | inner: QrCode, 17 | } 18 | 19 | #[derive(Error, Debug)] 20 | pub enum KeyQrCodeError { 21 | #[error("encryption-decryption protocol error")] 22 | SymmetricCipher(#[from] SymmetricCipherError), 23 | #[error("io error")] 24 | Io(#[from] io::Error), 25 | #[error("invalid secret key")] 26 | SecretKey(#[from] SecretKeyError), 27 | #[error("couldn't decode QR code")] 28 | QrDecodeError(#[from] QrDecodeError), 29 | #[error("failed to decode hex")] 30 | HexDecodeError(#[from] hex::FromHexError), 31 | #[error("failed to decode hex")] 32 | QrCodeHashError(#[from] super::payload::Error), 33 | #[error(transparent)] 34 | Image(#[from] ImageError), 35 | } 36 | 37 | #[derive(Error, Debug)] 38 | pub enum QrDecodeError { 39 | #[error("couldn't decode QR code")] 40 | DecodeError(#[from] quircs::DecodeError), 41 | #[error("couldn't extract QR code")] 42 | ExtractError(#[from] quircs::ExtractError), 43 | #[error("QR code payload is not valid uf8")] 44 | NonUtf8Payload, 45 | } 46 | 47 | impl KeyQrCode { 48 | pub fn generate(key: SecretKey, password: &[u8]) -> Self { 49 | let enc_hex = payload::generate(key, password); 50 | let inner = QrCode::with_error_correction_level(&enc_hex, EcLevel::H).unwrap(); 51 | 52 | KeyQrCode { inner } 53 | } 54 | 55 | pub fn write_svg(&self, path: impl AsRef) -> Result<(), KeyQrCodeError> { 56 | let mut out = File::create(path)?; 57 | let svg_file = self 58 | .inner 59 | .render() 60 | .quiet_zone(true) 61 | .dark_color(svg::Color("#000000")) 62 | .light_color(svg::Color("#ffffff")) 63 | .build(); 64 | out.write_all(svg_file.as_bytes())?; 65 | out.flush()?; 66 | Ok(()) 67 | } 68 | 69 | pub fn to_img(&self) -> ImageBuffer, Vec> { 70 | let qr = &self.inner; 71 | let img = qr.render::>().build(); 72 | img 73 | } 74 | 75 | pub fn decode( 76 | img: DynamicImage, 77 | password: &[u8], 78 | ) -> Result>, KeyQrCodeError> { 79 | let mut decoder = quircs::Quirc::default(); 80 | 81 | let img = img.into_luma8(); 82 | 83 | let codes = decoder.identify(img.width() as usize, img.height() as usize, &img); 84 | 85 | codes 86 | .map(|code| -> Result<_, KeyQrCodeError> { 87 | let decoded = code 88 | .map_err(QrDecodeError::ExtractError) 89 | .and_then(|c| c.decode().map_err(QrDecodeError::DecodeError))?; 90 | 91 | // TODO: I actually don't know if this can fail 92 | let h = std::str::from_utf8(&decoded.payload) 93 | .map_err(|_| QrDecodeError::NonUtf8Payload)?; 94 | payload::decode(h, password).map_err(Into::into) 95 | }) 96 | .collect() 97 | } 98 | } 99 | 100 | impl fmt::Display for KeyQrCode { 101 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 102 | let qr_img = self 103 | .inner 104 | .render::() 105 | .quiet_zone(true) 106 | .dark_color(unicode::Dense1x2::Light) 107 | .light_color(unicode::Dense1x2::Dark) 108 | .build(); 109 | write!(f, "{}", qr_img) 110 | } 111 | } 112 | 113 | #[cfg(test)] 114 | mod tests { 115 | use super::*; 116 | 117 | // TODO: Improve into an integration test using a temporary directory. 118 | // Leaving here as an example. 119 | #[test] 120 | #[ignore] 121 | fn generate_svg() { 122 | const PASSWORD: &[u8] = &[1, 2, 3, 4]; 123 | let sk = SecretKey::generate(rand::thread_rng()); 124 | let qr = KeyQrCode::generate(sk, PASSWORD); 125 | qr.write_svg("qr-code.svg").unwrap(); 126 | } 127 | 128 | #[test] 129 | #[ignore] 130 | fn encode_decode() { 131 | const PASSWORD: &[u8] = &[1, 2, 3, 4]; 132 | let sk = SecretKey::generate(rand::thread_rng()); 133 | let qr = KeyQrCode::generate(sk.clone(), PASSWORD); 134 | let img = qr.to_img(); 135 | // img.save("qr.png").unwrap(); 136 | assert_eq!( 137 | sk.leak_secret().as_ref(), 138 | KeyQrCode::decode(DynamicImage::ImageLuma8(img), PASSWORD).unwrap()[0] 139 | .clone() 140 | .leak_secret() 141 | .as_ref() 142 | ); 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/kedqr/mod.rs: -------------------------------------------------------------------------------- 1 | mod img; 2 | mod payload; 3 | 4 | pub use img::{KeyQrCode, KeyQrCodeError}; 5 | pub use payload::{decode, generate, Error as KeyQrCodePayloadError}; 6 | use std::path::PathBuf; 7 | use std::str::FromStr; 8 | use thiserror::Error; 9 | 10 | pub const PIN_LENGTH: usize = 4; 11 | 12 | #[derive(Debug, PartialEq, Eq)] 13 | pub struct QrPin { 14 | pub password: [u8; 4], 15 | } 16 | 17 | #[derive(Error, Debug)] 18 | pub enum Error { 19 | #[error(transparent)] 20 | BadPin(#[from] BadPinError), 21 | #[error(transparent)] 22 | KeyQrCodeHash(#[from] KeyQrCodePayloadError), 23 | #[error(transparent)] 24 | KeyQrCode(#[from] KeyQrCodeError), 25 | } 26 | 27 | #[derive(Error, Debug)] 28 | pub enum BadPinError { 29 | #[error("The PIN must consist of {PIN_LENGTH} digits, found {0}")] 30 | InvalidLength(usize), 31 | #[error("Invalid digit {0}")] 32 | InvalidDigit(char), 33 | #[error("cannot detect file name from path {0:?} in order to read qr pin from it")] 34 | UnableToDetectFileName(PathBuf), 35 | } 36 | 37 | impl FromStr for QrPin { 38 | type Err = BadPinError; 39 | 40 | fn from_str(s: &str) -> Result { 41 | if s.chars().count() != PIN_LENGTH { 42 | return Err(BadPinError::InvalidLength(s.len())); 43 | } 44 | 45 | let mut pwd = [0u8; 4]; 46 | for (i, digit) in s.chars().enumerate() { 47 | pwd[i] = digit.to_digit(10).ok_or(BadPinError::InvalidDigit(digit))? as u8; 48 | } 49 | Ok(QrPin { password: pwd }) 50 | } 51 | } 52 | 53 | #[derive(Clone, Debug)] 54 | pub enum PinReadMode { 55 | Global(String), 56 | FromFileName(PathBuf), 57 | } 58 | 59 | /// supported format is *1234.png 60 | impl PinReadMode { 61 | pub fn into_qr_pin(&self) -> Result { 62 | match self { 63 | PinReadMode::Global(ref global) => QrPin::from_str(global), 64 | PinReadMode::FromFileName(qr) => { 65 | let file_name = qr 66 | .file_stem() 67 | .ok_or_else(|| BadPinError::UnableToDetectFileName(qr.to_path_buf()))?; 68 | QrPin::from_str( 69 | &file_name 70 | .to_str() 71 | .unwrap() 72 | .chars() 73 | .rev() 74 | .take(4) 75 | .collect::>() 76 | .iter() 77 | .rev() 78 | .collect::(), 79 | ) 80 | } 81 | } 82 | } 83 | } 84 | 85 | #[cfg(test)] 86 | mod tests { 87 | use super::*; 88 | use chain_crypto::SecretKey; 89 | use image::DynamicImage; 90 | 91 | #[test] 92 | fn parse_pin_successfully() { 93 | for (pin, pwd) in &[ 94 | ("0000", [0, 0, 0, 0]), 95 | ("1123", [1, 1, 2, 3]), 96 | ("0002", [0, 0, 0, 2]), 97 | ] { 98 | let qr_pin = QrPin::from_str(pin).unwrap(); 99 | assert_eq!(qr_pin, QrPin { password: *pwd }) 100 | } 101 | } 102 | #[test] 103 | fn pins_that_do_not_satisfy_length_reqs_return_error() { 104 | for bad_pin in &["", "1", "11", "111", "11111"] { 105 | let qr_pin = QrPin::from_str(bad_pin); 106 | assert!(qr_pin.is_err(),) 107 | } 108 | } 109 | 110 | #[test] 111 | fn pins_that_do_not_satisfy_content_reqs_return_error() { 112 | for bad_pin in &[" ", " 111", "llll", "000u"] { 113 | let qr_pin = QrPin::from_str(bad_pin); 114 | assert!(qr_pin.is_err(),) 115 | } 116 | } 117 | 118 | // TODO: Improve into an integration test using a temporary directory. 119 | // Leaving here as an example. 120 | #[test] 121 | fn generate_svg() { 122 | const PASSWORD: &[u8] = &[1, 2, 3, 4]; 123 | let sk = SecretKey::generate(rand::thread_rng()); 124 | let qr = KeyQrCode::generate(sk, PASSWORD); 125 | qr.write_svg("qr-code.svg").unwrap(); 126 | } 127 | 128 | #[test] 129 | fn encode_decode() { 130 | const PASSWORD: &[u8] = &[1, 2, 3, 4]; 131 | let sk = SecretKey::generate(rand::thread_rng()); 132 | let qr = KeyQrCode::generate(sk.clone(), PASSWORD); 133 | let img = qr.to_img(); 134 | // img.save("qr.png").unwrap(); 135 | assert_eq!( 136 | sk.leak_secret().as_ref(), 137 | KeyQrCode::decode(DynamicImage::ImageLuma8(img), PASSWORD).unwrap()[0] 138 | .clone() 139 | .leak_secret() 140 | .as_ref() 141 | ); 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/kedqr/payload.rs: -------------------------------------------------------------------------------- 1 | use chain_crypto::{Ed25519Extended, SecretKey, SecretKeyError}; 2 | use std::io; 3 | use symmetric_cipher::{decrypt, encrypt, Error as SymmetricCipherError}; 4 | use thiserror::Error; 5 | 6 | #[derive(Error, Debug)] 7 | pub enum Error { 8 | #[error("encryption-decryption protocol error")] 9 | SymmetricCipher(#[from] SymmetricCipherError), 10 | #[error("io error")] 11 | Io(#[from] io::Error), 12 | #[error("invalid secret key")] 13 | SecretKey(#[from] SecretKeyError), 14 | #[error("failed to decode hex")] 15 | HexDecode(#[from] hex::FromHexError), 16 | } 17 | 18 | pub fn generate(key: SecretKey, password: &[u8]) -> String { 19 | let secret = key.leak_secret(); 20 | let rng = rand::thread_rng(); 21 | // this won't fail because we already know it's an ed25519extended key, 22 | // so it is safe to unwrap 23 | let enc = encrypt(password, secret.as_ref(), rng).unwrap(); 24 | // Using binary would make the QR codes more compact and probably less 25 | // prone to scanning errors. 26 | hex::encode(enc) 27 | } 28 | 29 | pub fn decode>( 30 | payload: S, 31 | password: &[u8], 32 | ) -> Result, Error> { 33 | let encrypted_bytes = hex::decode(payload.into())?; 34 | let key = decrypt(password, &encrypted_bytes)?; 35 | Ok(SecretKey::from_binary(&key)?) 36 | } 37 | 38 | #[cfg(test)] 39 | mod tests { 40 | use super::*; 41 | 42 | #[test] 43 | fn encode_decode() { 44 | const PASSWORD: &[u8] = &[1, 2, 3, 4]; 45 | let sk = SecretKey::generate(rand::thread_rng()); 46 | let hash = generate(sk.clone(), PASSWORD); 47 | assert_eq!( 48 | sk.leak_secret().as_ref(), 49 | decode(hash, PASSWORD).unwrap().leak_secret().as_ref() 50 | ); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod archive; 2 | pub mod community_advisors; 3 | pub mod http; 4 | pub mod ideascale; 5 | pub mod kedqr; 6 | pub mod logs; 7 | pub mod notifications; 8 | pub mod recovery; 9 | pub mod rewards; 10 | pub mod stats; 11 | pub mod utils; 12 | pub mod vca_reviews; 13 | pub mod vote_check; 14 | 15 | #[macro_use] 16 | extern crate tracing; 17 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/logs/compare.rs: -------------------------------------------------------------------------------- 1 | use crate::logs::sentry::{Error, SentryFragmentLog}; 2 | use crate::recovery::tally::{deconstruct_account_transaction, ValidationError}; 3 | 4 | use chain_core::property::Fragment as _; 5 | use chain_impl_mockchain::fragment::Fragment; 6 | use chain_impl_mockchain::vote::Payload; 7 | use jormungandr_lib::interfaces::PersistentFragmentLog; 8 | 9 | use std::collections::HashSet; 10 | 11 | #[derive(Debug, Eq, PartialEq)] 12 | pub struct LogCmpFields { 13 | pub public_key: String, 14 | pub chain_proposal_index: u8, 15 | pub voteplan_id: String, 16 | pub choice: u8, 17 | pub fragment_id: String, 18 | } 19 | 20 | pub fn persistent_fragment_log_to_log_cmp_fields( 21 | fragment: &PersistentFragmentLog, 22 | ) -> Result { 23 | if let Fragment::VoteCast(ref transaction) = fragment.fragment.clone() { 24 | let (vote_cast, identifier, choice) = deconstruct_account_transaction( 25 | &transaction.as_slice(), 26 | ) 27 | .and_then(|(vote_cast, identifier, _)| { 28 | if let Payload::Public { choice } = vote_cast.payload().clone() { 29 | Ok((vote_cast, identifier, choice)) 30 | } else { 31 | Err(ValidationError::UnsupportedPrivateVotes) 32 | } 33 | })?; 34 | Ok(LogCmpFields { 35 | fragment_id: fragment.fragment.id().to_string(), 36 | public_key: identifier.to_string(), 37 | chain_proposal_index: vote_cast.proposal_index(), 38 | choice: choice.as_byte(), 39 | voteplan_id: vote_cast.vote_plan().to_string(), 40 | }) 41 | } else { 42 | Err(Error::NotVoteCastTransaction { 43 | fragment_id: fragment.fragment.id().to_string(), 44 | }) 45 | } 46 | } 47 | 48 | pub struct LogCmpStats { 49 | pub sentry_logs_size: usize, 50 | pub fragment_logs_size: usize, 51 | pub duplicated_sentry_logs: usize, 52 | pub duplicated_fragment_logs: usize, 53 | pub fragment_ids_differ: HashSet, 54 | pub unhandled_fragment_logs: Vec<(Fragment, Error)>, 55 | } 56 | 57 | pub fn compare_logs( 58 | sentry_logs: &[SentryFragmentLog], 59 | fragment_logs: &[PersistentFragmentLog], 60 | ) -> LogCmpStats { 61 | let sentry_logs_size = sentry_logs.len(); 62 | let fragment_logs_size = fragment_logs.len(); 63 | let sentry_cmp: Vec = sentry_logs.iter().cloned().map(Into::into).collect(); 64 | 65 | let (fragments_cmp, unhandled_fragment_logs): (Vec, Vec<(Fragment, Error)>) = 66 | fragment_logs.iter().fold( 67 | (Vec::new(), Vec::new()), 68 | |(mut success, mut errored), log| { 69 | match persistent_fragment_log_to_log_cmp_fields(log) { 70 | Ok(log) => { 71 | success.push(log); 72 | } 73 | Err(e) => errored.push((log.fragment.clone(), e)), 74 | }; 75 | (success, errored) 76 | }, 77 | ); 78 | 79 | let sentry_fragments_ids: HashSet = sentry_cmp 80 | .iter() 81 | .map(|e| e.fragment_id.to_string()) 82 | .collect(); 83 | let fragment_logs_ids: HashSet = fragments_cmp 84 | .iter() 85 | .map(|e| e.fragment_id.to_string()) 86 | .collect(); 87 | let fragment_ids_differ: HashSet = sentry_fragments_ids 88 | .difference(&fragment_logs_ids) 89 | .cloned() 90 | .collect(); 91 | let duplicated_sentry_logs = sentry_logs_size - sentry_fragments_ids.len(); 92 | let duplicated_fragment_logs = fragment_logs_size - fragment_logs_ids.len(); 93 | 94 | LogCmpStats { 95 | sentry_logs_size, 96 | fragment_logs_size, 97 | duplicated_sentry_logs, 98 | duplicated_fragment_logs, 99 | fragment_ids_differ, 100 | unhandled_fragment_logs, 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/logs/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod compare; 2 | pub mod sentry; 3 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/notifications/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod requests; 2 | pub mod responses; 3 | pub mod send; 4 | 5 | use thiserror::Error; 6 | 7 | #[allow(clippy::large_enum_variant)] 8 | #[derive(Debug, Error)] 9 | pub enum Error { 10 | #[error(transparent)] 11 | CreateMessageError(#[from] requests::create_message::Error), 12 | 13 | #[error(transparent)] 14 | SerdeError(#[from] serde_json::Error), 15 | 16 | #[error(transparent)] 17 | RequestError(#[from] reqwest::Error), 18 | 19 | #[error("error reading file, source: {0}")] 20 | FileError(#[from] std::io::Error), 21 | 22 | #[error("sent data is invalid:\n {request}")] 23 | BadDataSent { request: String }, 24 | 25 | #[error("request was unsuccessful, feedback:\n {response}")] 26 | UnsuccessfulRequest { response: String }, 27 | } 28 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/notifications/requests/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod create_message; 2 | use serde::{Deserialize, Serialize}; 3 | 4 | #[derive(Serialize, Deserialize)] 5 | #[serde(untagged)] 6 | pub enum RequestData { 7 | CreateMessageRequest(create_message::CreateMessage), 8 | } 9 | 10 | #[derive(Serialize)] 11 | pub struct Request { 12 | request: RequestData, 13 | } 14 | 15 | impl Request { 16 | pub fn new(data: RequestData) -> Self { 17 | Self { request: data } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/notifications/responses/create_message.rs: -------------------------------------------------------------------------------- 1 | use reqwest::StatusCode; 2 | use serde::de::Error; 3 | use serde::{Deserialize, Deserializer, Serialize, Serializer}; 4 | 5 | #[derive(Serialize, Deserialize, Debug)] 6 | pub struct InnerResponse { 7 | #[serde(alias = "Messages")] 8 | pub messages: Vec, 9 | } 10 | 11 | #[derive(Serialize, Deserialize, Debug)] 12 | pub struct CreateMessageResponse { 13 | #[serde( 14 | deserialize_with = "deserialize_status_code", 15 | serialize_with = "serialize_status_code" 16 | )] 17 | pub status_code: StatusCode, 18 | pub status_message: String, 19 | pub response: InnerResponse, 20 | } 21 | 22 | fn deserialize_status_code<'de, D>(deserializer: D) -> Result 23 | where 24 | D: Deserializer<'de>, 25 | { 26 | StatusCode::from_u16(u16::deserialize(deserializer)?) 27 | .map_err(|_| D::Error::custom("Invalid StatusCode")) 28 | } 29 | 30 | fn serialize_status_code(status_code: &StatusCode, serializer: S) -> Result 31 | where 32 | S: Serializer, 33 | { 34 | serializer.serialize_u16(status_code.as_u16()) 35 | } 36 | 37 | #[cfg(test)] 38 | mod test { 39 | use super::CreateMessageResponse; 40 | use reqwest::StatusCode; 41 | 42 | #[test] 43 | fn test_deserialize() { 44 | let msg = r#"{ 45 | "status_code": 200, 46 | "status_message": "OK", 47 | "response": { 48 | "Messages": [ 49 | "C3F8-C3863ED4-334AD4F1" 50 | ] 51 | } 52 | }"#; 53 | let message_code = "C3F8-C3863ED4-334AD4F1"; 54 | let response: CreateMessageResponse = serde_json::from_str(msg).expect("valid json data"); 55 | assert_eq!(response.status_code, StatusCode::OK); 56 | assert_eq!(response.status_message, "OK"); 57 | assert_eq!(response.response.messages.len(), 1); 58 | assert_eq!(response.response.messages[0], message_code); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/notifications/responses/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod create_message; 2 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/notifications/send.rs: -------------------------------------------------------------------------------- 1 | use reqwest::{blocking::Client, StatusCode, Url}; 2 | 3 | use crate::notifications::{ 4 | requests::Request, responses::create_message::CreateMessageResponse, Error, 5 | }; 6 | 7 | pub fn send_create_message( 8 | url: Url, 9 | notification: &Request, 10 | ) -> Result { 11 | let client = Client::new(); 12 | let response = client 13 | .post(url) 14 | .body(serde_json::to_string(¬ification)?) 15 | .send()?; 16 | match response.status() { 17 | StatusCode::OK => {} 18 | StatusCode::BAD_REQUEST => { 19 | return Err(Error::BadDataSent { 20 | request: serde_json::to_string_pretty(¬ification)?, 21 | }) 22 | } 23 | _ => { 24 | return Err(Error::UnsuccessfulRequest { 25 | response: response.text()?, 26 | }) 27 | } 28 | }; 29 | let response_message = response.json()?; 30 | Ok(response_message) 31 | } 32 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/recovery/mod.rs: -------------------------------------------------------------------------------- 1 | mod replay; 2 | pub mod tally; 3 | 4 | pub use replay::{Error as ReplayError, Replay}; 5 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/recovery/replay.rs: -------------------------------------------------------------------------------- 1 | use crate::recovery::tally::recover_ledger_from_logs; 2 | use chain_core::property::Fragment; 3 | use chain_impl_mockchain::block::Block; 4 | pub use jcli_lib::utils::{ 5 | output_file::{Error as OutputFileError, OutputFile}, 6 | output_format::{Error as OutputFormatError, OutputFormat}, 7 | }; 8 | use jormungandr_lib::interfaces::{ 9 | load_persistent_fragments_logs_from_folder_path, VotePlanStatus, 10 | }; 11 | use std::io::Write; 12 | use std::path::PathBuf; 13 | use tracing::warn; 14 | 15 | /// Recover the tally from fragment log files and the initial preloaded block0 binary file. 16 | pub struct Replay { 17 | block0: Block, 18 | /// Path to the folder containing the log files used for the tally reconstruction 19 | logs_path: PathBuf, 20 | output: OutputFile, 21 | output_format: OutputFormat, 22 | } 23 | 24 | impl Replay { 25 | pub fn new( 26 | block0: Block, 27 | logs_path: PathBuf, 28 | output: OutputFile, 29 | output_format: OutputFormat, 30 | ) -> Self { 31 | Self { 32 | block0, 33 | logs_path, 34 | output, 35 | output_format, 36 | } 37 | } 38 | 39 | pub fn exec(self) -> Result<(), Error> { 40 | let fragments = load_persistent_fragments_logs_from_folder_path(&self.logs_path) 41 | .map_err(Error::PersistenLogsLoading)?; 42 | 43 | let (ledger, failed) = recover_ledger_from_logs(&self.block0, fragments)?; 44 | if !failed.is_empty() { 45 | warn!("{} fragments couldn't be properly processed", failed.len()); 46 | for failed_fragment in failed { 47 | warn!("{}", failed_fragment.id()); 48 | } 49 | } 50 | let voteplans = ledger.active_vote_plans(); 51 | let voteplan_status: Vec = 52 | voteplans.into_iter().map(VotePlanStatus::from).collect(); 53 | let mut out_writer = self.output.open()?; 54 | let content = self 55 | .output_format 56 | .format_json(serde_json::to_value(&voteplan_status)?)?; 57 | out_writer.write_all(content.as_bytes())?; 58 | Ok(()) 59 | } 60 | } 61 | 62 | #[allow(clippy::large_enum_variant)] 63 | #[derive(thiserror::Error, Debug)] 64 | pub enum Error { 65 | #[error(transparent)] 66 | Io(#[from] std::io::Error), 67 | 68 | #[error(transparent)] 69 | Serialization(#[from] serde_json::Error), 70 | 71 | #[error(transparent)] 72 | Recovery(#[from] crate::recovery::tally::Error), 73 | 74 | #[error(transparent)] 75 | OutputFile(#[from] OutputFileError), 76 | 77 | #[error(transparent)] 78 | OutputFormat(#[from] OutputFormatError), 79 | 80 | #[error("Could not load persistent logs from path")] 81 | PersistenLogsLoading(#[source] std::io::Error), 82 | } 83 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/rewards/community_advisors/funding.rs: -------------------------------------------------------------------------------- 1 | use crate::rewards::Funds; 2 | use serde::Deserialize; 3 | 4 | #[derive(Deserialize)] 5 | pub struct FundSetting { 6 | pub proposal_ratio: u8, 7 | pub bonus_ratio: u8, 8 | pub total: Funds, 9 | } 10 | 11 | impl FundSetting { 12 | #[inline] 13 | pub fn proposal_funds(&self) -> Funds { 14 | self.total * (Funds::from(self.proposal_ratio) / Funds::from(100)) 15 | } 16 | 17 | #[inline] 18 | pub fn bonus_funds(&self) -> Funds { 19 | self.total * (Funds::from(self.bonus_ratio) / Funds::from(100)) 20 | } 21 | 22 | #[inline] 23 | pub fn total_funds(&self) -> Funds { 24 | self.total 25 | } 26 | } 27 | 28 | #[derive(Deserialize)] 29 | pub struct ProposalRewardSlots { 30 | pub excellent_slots: u64, 31 | pub good_slots: u64, 32 | pub max_good_reviews: u64, 33 | pub max_excellent_reviews: u64, 34 | } 35 | 36 | impl Default for ProposalRewardSlots { 37 | fn default() -> Self { 38 | Self { 39 | excellent_slots: 12, 40 | good_slots: 4, 41 | max_good_reviews: 3, 42 | max_excellent_reviews: 2, 43 | } 44 | } 45 | } 46 | 47 | impl ProposalRewardSlots { 48 | pub fn max_winning_tickets(&self) -> u64 { 49 | self.max_excellent_reviews * self.excellent_slots + self.max_good_reviews * self.good_slots 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/rewards/community_advisors/lottery.rs: -------------------------------------------------------------------------------- 1 | use super::CommunityAdvisor; 2 | use rand::Rng; 3 | use std::collections::BTreeMap; 4 | 5 | pub type TotalTickets = u64; 6 | pub type TicketsDistribution = BTreeMap; 7 | pub type CasWinnings = BTreeMap; 8 | 9 | pub fn lottery_distribution( 10 | mut distribution: TicketsDistribution, 11 | tickets_to_distribute: TotalTickets, 12 | rng: &mut R, 13 | ) -> (CasWinnings, TicketsDistribution) { 14 | let total_tickets = distribution.values().sum::() as usize; 15 | 16 | // Virtually create all tickets and choose the winning tickets using their index. 17 | let mut indexes = 18 | rand::seq::index::sample(rng, total_tickets, tickets_to_distribute as usize).into_vec(); 19 | indexes.sort_unstable(); 20 | let mut indexes = indexes.into_iter().peekable(); 21 | 22 | // To avoid using too much memory, tickets are not actually created, and we iterate 23 | // the CAs to reconstruct the owner of each ticket. 24 | let mut winnings = CasWinnings::new(); 25 | let mut cumulative_ticket_index = 0; 26 | 27 | // Consistent iteration is needed to get reproducible results. In this case, 28 | // it's ensured by the use of BTreeMap::iter() 29 | for (ca, n_tickets) in distribution.iter_mut() { 30 | let tickets_won = std::iter::from_fn(|| { 31 | indexes.next_if(|tkt| *tkt < (cumulative_ticket_index + *n_tickets) as usize) 32 | }) 33 | .count(); 34 | cumulative_ticket_index += *n_tickets; 35 | if tickets_won > 0 { 36 | winnings.insert(ca.clone(), tickets_won as u64); 37 | } 38 | *n_tickets -= tickets_won as u64; 39 | } 40 | (winnings, distribution) 41 | } 42 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/rewards/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod community_advisors; 2 | pub mod dreps; 3 | pub mod proposers; 4 | pub mod veterans; 5 | pub mod voters; 6 | 7 | use rust_decimal::Decimal; 8 | pub type Funds = Decimal; 9 | // Lets match to the same type as the funds, but naming it funds would be confusing 10 | pub type Rewards = Decimal; 11 | pub type VoteCount = HashMap>; 12 | 13 | use jormungandr_lib::crypto::{account::Identifier, hash::Hash}; 14 | use std::collections::{HashMap, HashSet}; 15 | use thiserror::Error; 16 | use vit_servicing_station_lib::db::models::proposals::FullProposalInfo; 17 | 18 | #[derive(Debug, Error)] 19 | pub enum Error { 20 | #[error("hash is not a valid blake2b256 hash")] 21 | InvalidHash(Vec), 22 | } 23 | 24 | pub struct Threshold { 25 | total: usize, 26 | per_challenge: HashMap, 27 | proposals_per_challenge: HashMap>, 28 | } 29 | 30 | impl Threshold { 31 | pub fn new( 32 | total_threshold: usize, 33 | per_challenge: HashMap, 34 | proposals: Vec, 35 | ) -> Result { 36 | let proposals = proposals 37 | .into_iter() 38 | .map(|p| { 39 | <[u8; 32]>::try_from(p.proposal.chain_proposal_id) 40 | .map_err(Error::InvalidHash) 41 | .map(|hash| (p.proposal.challenge_id, Hash::from(hash))) 42 | }) 43 | .collect::, Error>>()?; 44 | Ok(Self { 45 | total: total_threshold, 46 | per_challenge, 47 | proposals_per_challenge: proposals.into_iter().fold( 48 | HashMap::new(), 49 | |mut acc, (challenge_id, hash)| { 50 | acc.entry(challenge_id).or_default().insert(hash); 51 | acc 52 | }, 53 | ), 54 | }) 55 | } 56 | 57 | fn filter(&self, votes: &HashSet) -> bool { 58 | if votes.len() < self.total { 59 | return false; 60 | } 61 | 62 | for (challenge, threshold) in &self.per_challenge { 63 | let votes_in_challengs = self 64 | .proposals_per_challenge 65 | .get(challenge) 66 | .map(|props| votes.intersection(props).count()) 67 | .unwrap_or_default(); 68 | if votes_in_challengs < *threshold { 69 | return false; 70 | } 71 | } 72 | 73 | true 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/rewards/proposers/types.rs: -------------------------------------------------------------------------------- 1 | use jormungandr_lib::crypto::hash::Hash; 2 | use serde::{Deserialize, Serialize}; 3 | use std::{path::PathBuf, str::FromStr}; 4 | use structopt::StructOpt; 5 | 6 | use color_eyre::{eyre::eyre, Report}; 7 | 8 | macro_rules! bool_enum { 9 | ($enum_name:ident, $true_case:ident, $false_case:ident) => { 10 | #[derive(Debug, Serialize, Deserialize)] 11 | #[serde(rename_all = "UPPERCASE")] 12 | pub enum $enum_name { 13 | $true_case, 14 | $false_case, 15 | } 16 | 17 | impl From for $enum_name { 18 | fn from(b: bool) -> Self { 19 | match b { 20 | true => $enum_name::$true_case, 21 | false => $enum_name::$false_case, 22 | } 23 | } 24 | } 25 | }; 26 | } 27 | 28 | bool_enum!(YesNo, Yes, No); 29 | bool_enum!(FundedStatus, Funded, NotFunded); 30 | 31 | #[derive(Debug, Serialize, Deserialize)] 32 | pub enum NotFundedReason { 33 | #[serde(rename = "Not Funded - Over Budget")] 34 | OverBudget, 35 | #[serde(rename = "Not Funded - Approval Threshold")] 36 | ApprovalThreshold, 37 | } 38 | 39 | #[derive(StructOpt)] 40 | #[structopt(rename_all = "kebab-case")] 41 | pub struct ProposerRewards { 42 | #[structopt(long = "output-file")] 43 | pub output: PathBuf, 44 | 45 | #[structopt(long = "block0-path")] 46 | pub block0: PathBuf, 47 | 48 | #[structopt(default_value = "0.01")] 49 | #[structopt(long)] 50 | pub total_stake_threshold: f64, 51 | 52 | #[structopt(default_value = "1.15")] 53 | #[structopt(long)] 54 | pub approval_threshold: f64, 55 | 56 | #[structopt(default_value = "csv")] 57 | #[structopt(long)] 58 | pub output_format: OutputFormat, 59 | 60 | #[structopt(long = "proposals-path")] 61 | pub proposals: Option, 62 | #[structopt(long = "excluded-proposals-path")] 63 | pub excluded_proposals: Option, 64 | #[structopt(long = "active-voteplan-path")] 65 | pub active_voteplans: Option, 66 | #[structopt(long = "challenges-path")] 67 | pub challenges: Option, 68 | 69 | #[structopt(default_value = "https://servicing-station.vit.iohk.io")] 70 | pub vit_station_url: String, 71 | 72 | #[structopt(long = "committee-keys-path")] 73 | pub committee_keys: Option, 74 | } 75 | 76 | #[derive(Debug, Serialize, Deserialize)] 77 | pub struct Calculation { 78 | pub internal_id: String, 79 | pub proposal_id: Hash, 80 | pub proposal: String, 81 | pub overall_score: i64, 82 | pub yes: u64, 83 | pub no: u64, 84 | pub result: i64, 85 | pub meets_approval_threshold: YesNo, 86 | pub requested_dollars: i64, 87 | pub status: FundedStatus, 88 | pub fund_depletion: f64, 89 | pub not_funded_reason: Option, 90 | pub link_to_ideascale: String, 91 | } 92 | 93 | #[cfg(test)] 94 | impl Default for Calculation { 95 | fn default() -> Self { 96 | Self { 97 | internal_id: Default::default(), 98 | proposal_id: <[u8; 32]>::default().into(), 99 | proposal: Default::default(), 100 | overall_score: Default::default(), 101 | yes: Default::default(), 102 | no: Default::default(), 103 | result: Default::default(), 104 | meets_approval_threshold: YesNo::Yes, 105 | requested_dollars: Default::default(), 106 | status: FundedStatus::NotFunded, 107 | fund_depletion: Default::default(), 108 | not_funded_reason: None, 109 | link_to_ideascale: Default::default(), 110 | } 111 | } 112 | } 113 | 114 | #[derive(Debug, Clone, Copy)] 115 | pub enum OutputFormat { 116 | Json, 117 | Csv, 118 | } 119 | 120 | impl FromStr for OutputFormat { 121 | type Err = Report; 122 | 123 | fn from_str(s: &str) -> Result { 124 | match s { 125 | "json" => Ok(OutputFormat::Json), 126 | "csv" => Ok(OutputFormat::Csv), 127 | s => Err(eyre!("expected one of `csv` or `json`, found {s}")), 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/rewards/proposers/util.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | borrow::Cow, 3 | path::{Path, PathBuf}, 4 | }; 5 | 6 | use once_cell::sync::Lazy; 7 | use regex::Regex; 8 | 9 | pub fn build_path_for_challenge(path: &Path, challenge_name: &str) -> PathBuf { 10 | let challenge_name = sanitize_name(challenge_name); 11 | let ext = path.extension(); 12 | 13 | let mut path = path.with_extension("").as_os_str().to_owned(); 14 | path.push("_"); 15 | path.push(&*challenge_name); 16 | let path = PathBuf::from(path); 17 | 18 | match ext { 19 | Some(ext) => path.with_extension(ext), 20 | None => path, 21 | } 22 | } 23 | 24 | fn sanitize_name(name: &str) -> Cow<'_, str> { 25 | static REMOVE_REGEX: Lazy = Lazy::new(|| Regex::new(r#"[^-\w.]"#).unwrap()); 26 | static REPLACE_UNDERSCORE_REGEX: Lazy = Lazy::new(|| Regex::new(r#" |:"#).unwrap()); // space or colon 27 | // 28 | let name = REPLACE_UNDERSCORE_REGEX.replace_all(name, "_"); 29 | match name { 30 | Cow::Borrowed(borrow) => REMOVE_REGEX.replace_all(borrow, ""), 31 | Cow::Owned(owned) => { 32 | let result = REMOVE_REGEX.replace_all(&owned, ""); 33 | Cow::Owned(result.to_string()) 34 | } 35 | } 36 | } 37 | 38 | #[cfg(test)] 39 | mod tests { 40 | use super::*; 41 | 42 | #[test] 43 | fn sanitize_replaces_correctly() { 44 | assert_eq!(sanitize_name("asdf"), "asdf"); 45 | // colons and spaces replaced with underscores 46 | assert_eq!(sanitize_name("a b:c"), "a_b_c"); 47 | // other symbols removed 48 | assert_eq!(sanitize_name("a£$%^&*()bc"), "abc"); 49 | // . and - are allowed 50 | assert_eq!(sanitize_name("a.b-c"), "a.b-c"); 51 | // all together 52 | assert_eq!(sanitize_name("foo$%. bar:baz"), "foo._bar_baz"); 53 | } 54 | 55 | #[test] 56 | fn test_build_path() { 57 | let path = "/some/path.ext"; 58 | let challenge = "challenge"; 59 | let built_path = build_path_for_challenge(Path::new(path), challenge); 60 | assert_eq!(built_path, PathBuf::from("/some/path_challenge.ext")); 61 | } 62 | 63 | #[test] 64 | fn test_build_path_hidden_file() { 65 | let path = "/some/.path.ext"; 66 | let challenge = "challenge"; 67 | let built_path = build_path_for_challenge(Path::new(path), challenge); 68 | assert_eq!(built_path, PathBuf::from("/some/.path_challenge.ext")); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/stats/archive/calculator.rs: -------------------------------------------------------------------------------- 1 | use super::loader::ArchiverRecord; 2 | use chain_impl_mockchain::block::BlockDateParseError; 3 | use chain_time::TimeEra; 4 | use itertools::Itertools; 5 | use jormungandr_lib::interfaces::BlockDate; 6 | use std::collections::BTreeMap; 7 | use std::collections::BTreeSet; 8 | use thiserror::Error; 9 | 10 | pub struct ArchiveStats { 11 | records: Vec, 12 | } 13 | 14 | impl From> for ArchiveStats { 15 | fn from(records: Vec) -> Self { 16 | Self { records } 17 | } 18 | } 19 | 20 | impl ArchiveStats { 21 | pub fn calculate_distribution(result: &BTreeMap) -> BTreeMap { 22 | let mut distribution: BTreeMap = BTreeMap::new(); 23 | 24 | for value in result.values() { 25 | *distribution.entry(*value).or_default() += 1; 26 | } 27 | 28 | distribution 29 | } 30 | 31 | pub fn number_of_tx_per_slot(&self) -> BTreeMap { 32 | self.records 33 | .iter() 34 | .group_by(|item| item.time.to_string()) 35 | .into_iter() 36 | .map(|(key, group)| (key, group.count())) 37 | .collect() 38 | } 39 | 40 | pub fn distinct_casters(&self) -> BTreeSet { 41 | self.records 42 | .iter() 43 | .map(|item| item.caster.to_string()) 44 | .collect() 45 | } 46 | 47 | pub fn number_of_votes_per_caster(&self) -> BTreeMap { 48 | self.records 49 | .iter() 50 | .group_by(|item| item.caster.to_string()) 51 | .into_iter() 52 | .map(|(key, group)| (key, group.count())) 53 | .collect() 54 | } 55 | 56 | /// Method return max batch size calculated as biggest consecutive chain of transactions 57 | /// sent to blockchain. 58 | /// WARNING: single transaction in each slot would be counted as a batch also. 59 | pub fn max_batch_size_per_caster( 60 | &self, 61 | slots_in_epoch: u32, 62 | ) -> Result, ArchiveCalculatorError> { 63 | let time_era = self.records[0].time.time_era(slots_in_epoch); 64 | 65 | Ok(self 66 | .records 67 | .iter() 68 | .group_by(|item| item.caster.to_string()) 69 | .into_iter() 70 | .map(|(key, group)| { 71 | let mut sorted_group: Vec<&ArchiverRecord> = group.collect(); 72 | 73 | sorted_group.sort_by_key(|a| a.time); 74 | 75 | let mut max_batch_size = 1; 76 | let mut current_batch_size = 1; 77 | let mut last_slot: BlockDate = sorted_group[0].time; 78 | 79 | for item in sorted_group.iter().skip(1) { 80 | let current: BlockDate = item.time; 81 | if are_equal_or_adjacent(&last_slot, ¤t, &time_era) { 82 | current_batch_size += 1; 83 | } else { 84 | max_batch_size = std::cmp::max(max_batch_size, current_batch_size); 85 | current_batch_size = 1; 86 | } 87 | last_slot = current; 88 | } 89 | (key, std::cmp::max(max_batch_size, current_batch_size)) 90 | }) 91 | .collect()) 92 | } 93 | } 94 | 95 | fn are_equal_or_adjacent(left: &BlockDate, right: &BlockDate, time_era: &TimeEra) -> bool { 96 | left == right || left.clone().shift_slot(1, time_era) == *right 97 | } 98 | 99 | #[derive(Debug, Error)] 100 | pub enum ArchiveCalculatorError { 101 | #[error("general error")] 102 | General(#[from] std::io::Error), 103 | #[error("cannot calculate distribution: cannot calculate max element result is empty")] 104 | EmptyResult, 105 | #[error("csv error")] 106 | Csv(#[from] csv::Error), 107 | #[error("block date error")] 108 | BlockDateParse(#[from] BlockDateParseError), 109 | } 110 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/stats/archive/loader.rs: -------------------------------------------------------------------------------- 1 | use csv; 2 | use jormungandr_lib::interfaces::BlockDate; 3 | use serde::Deserialize; 4 | use std::{ffi::OsStr, fmt, path::Path}; 5 | use thiserror::Error; 6 | 7 | #[derive(Debug, Deserialize)] 8 | pub struct ArchiverRecord { 9 | pub fragment_id: String, 10 | pub caster: String, 11 | pub proposal: u32, 12 | #[serde(deserialize_with = "deserialize_block_date_from_float")] 13 | pub time: BlockDate, 14 | pub choice: u8, 15 | pub raw_fragment: String, 16 | } 17 | 18 | use serde::de::Visitor; 19 | use serde::Deserializer; 20 | 21 | pub fn deserialize_block_date_from_float<'de, D>(deserializer: D) -> Result 22 | where 23 | D: Deserializer<'de>, 24 | { 25 | struct VoteOptionsDeserializer(); 26 | 27 | impl<'de> Visitor<'de> for VoteOptionsDeserializer { 28 | type Value = BlockDate; 29 | 30 | fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { 31 | formatter.write_str("float in format {epoch}.{slod_id}") 32 | } 33 | 34 | fn visit_f64(self, value: f64) -> Result 35 | where 36 | E: serde::de::Error, 37 | { 38 | Ok(value.to_string().parse().unwrap()) 39 | } 40 | } 41 | 42 | deserializer.deserialize_f64(VoteOptionsDeserializer()) 43 | } 44 | 45 | pub fn load_from_csv>( 46 | csv_path: P, 47 | ) -> Result, ArchiveReaderError> { 48 | let csv_path = csv_path.as_ref(); 49 | 50 | let mut reader = csv::ReaderBuilder::new() 51 | .flexible(true) 52 | .has_headers(true) 53 | .quoting(true) 54 | .quote(b'"') 55 | .trim(csv::Trim::All) 56 | .from_path(csv_path)?; 57 | 58 | let mut results = Vec::new(); 59 | for record in reader.deserialize() { 60 | match record { 61 | Ok(data) => { 62 | results.push(data); 63 | } 64 | Err(e) => return Err(ArchiveReaderError::Csv(e)), 65 | } 66 | } 67 | Ok(results) 68 | } 69 | 70 | pub fn load_from_folder>( 71 | folder_path: P, 72 | ) -> Result, ArchiveReaderError> { 73 | let mut records = Vec::new(); 74 | 75 | for entry in std::fs::read_dir(folder_path)? { 76 | let entry = entry?; 77 | let path = entry.path(); 78 | if let Some(extension) = path.extension().and_then(OsStr::to_str) { 79 | if extension == "csv" { 80 | records.extend(load_from_csv(path)?); 81 | } 82 | } 83 | } 84 | Ok(records) 85 | } 86 | 87 | #[derive(Debug, Error)] 88 | pub enum ArchiveReaderError { 89 | #[error("general error")] 90 | General(#[from] std::io::Error), 91 | #[error("csv error")] 92 | Csv(#[from] csv::Error), 93 | } 94 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/stats/archive/mod.rs: -------------------------------------------------------------------------------- 1 | mod calculator; 2 | mod loader; 3 | 4 | pub use calculator::{ArchiveCalculatorError, ArchiveStats}; 5 | pub use loader::{load_from_csv, load_from_folder, ArchiveReaderError}; 6 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/stats/distribution.rs: -------------------------------------------------------------------------------- 1 | use core::ops::Range; 2 | use std::collections::HashMap; 3 | use thiserror::Error; 4 | 5 | fn levels(threshold: u64) -> Result>, Error> { 6 | if !(450..=1_000).contains(&threshold) { 7 | return Err(Error::InvalidThreshold(threshold)); 8 | } 9 | 10 | Ok(vec![ 11 | (0..450), 12 | (450..threshold), 13 | (threshold..1_000), 14 | (1_000..2_000), 15 | (2_000..5_000), 16 | (5_000..10_000), 17 | (10_000..20_000), 18 | (20_000..50_000), 19 | (50_000..100_000), 20 | (100_000..250_000), 21 | (250_000..500_000), 22 | (500_000..1_000_000), 23 | (1_000_000..5_000_000), 24 | (5_000_000..10_000_000), 25 | (10_000_000..25_000_000), 26 | (25_000_000..50_000_000), 27 | (50_000_000..32_000_000_000), 28 | ]) 29 | } 30 | 31 | #[derive(Default)] 32 | pub struct Record { 33 | pub count: u32, 34 | pub total: u64, 35 | } 36 | 37 | pub struct Stats { 38 | pub content: HashMap, Record>, 39 | } 40 | 41 | impl Stats { 42 | pub fn new(threshold: u64) -> Result { 43 | Ok(Self::new_with_levels(levels(threshold)?)) 44 | } 45 | 46 | pub fn new_with_levels(levels: Vec>) -> Self { 47 | Self { 48 | content: levels 49 | .into_iter() 50 | .map(|range| (range, Default::default())) 51 | .collect(), 52 | } 53 | } 54 | 55 | pub fn add_with_weight(&mut self, value: u64, weight: u32) { 56 | for (range, record) in self.content.iter_mut() { 57 | if range.contains(&value) { 58 | record.count += weight; 59 | record.total += value; 60 | return; 61 | } 62 | } 63 | } 64 | 65 | pub fn add(&mut self, value: u64) { 66 | self.add_with_weight(value, 1); 67 | } 68 | 69 | pub fn print_count_per_level(&self) { 70 | let mut keys = self.content.keys().cloned().collect::>>(); 71 | keys.sort_by_key(|x| x.start); 72 | 73 | for key in keys { 74 | let start = format_big_number(key.start); 75 | let end = format_big_number(key.end); 76 | println!("{} .. {} -> {} ", start, end, self.content[&key].count); 77 | } 78 | println!( 79 | "Total -> {} ", 80 | self.content.values().map(|x| x.count).sum::() 81 | ); 82 | } 83 | 84 | pub fn print_ada_per_level(&self) { 85 | let mut keys = self.content.keys().cloned().collect::>>(); 86 | keys.sort_by_key(|x| x.start); 87 | 88 | for key in keys { 89 | let start = format_big_number(key.start); 90 | let end = format_big_number(key.end); 91 | println!( 92 | "{} .. {} -> {} ", 93 | start, 94 | end, 95 | format_big_number(self.content[&key].total) 96 | ); 97 | } 98 | println!( 99 | "Total -> {} ", 100 | self.content.values().map(|x| x.total).sum::() 101 | ); 102 | } 103 | } 104 | 105 | fn format_big_number(n: u64) -> String { 106 | if n == 0 { 107 | n.to_string() 108 | } else if n % 1_000_000_000 == 0 { 109 | format!("{} MLD", n / 1_000_000) 110 | } else if n % 1_00000 == 0 { 111 | format!("{} M", n / 1_000_000) 112 | } else if n % 1_000 == 0 { 113 | format!("{} k", n / 1_000) 114 | } else { 115 | n.to_string() 116 | } 117 | } 118 | 119 | #[allow(clippy::large_enum_variant)] 120 | #[derive(Error, Debug)] 121 | pub enum Error { 122 | #[error("invalid threshold for distribution levels ({0}). It should be more than 450 and less that 1000")] 123 | InvalidThreshold(u64), 124 | } 125 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/stats/live/harvester.rs: -------------------------------------------------------------------------------- 1 | use jormungandr_automation::jormungandr::JormungandrRest; 2 | use jormungandr_lib::interfaces::FragmentLog; 3 | use serde_json; 4 | use time::OffsetDateTime; 5 | 6 | pub struct Harvester { 7 | rest: JormungandrRest, 8 | endpoint: String, 9 | timeout: std::time::Duration, 10 | } 11 | 12 | impl Harvester { 13 | pub fn new>(endpoint: S, timeout: std::time::Duration) -> Self { 14 | let endpoint = endpoint.into(); 15 | 16 | Self { 17 | rest: JormungandrRest::new(endpoint.clone()), 18 | timeout, 19 | endpoint, 20 | } 21 | } 22 | 23 | pub fn harvest(&self) -> Result { 24 | let mut votes_count: usize = 0; 25 | 26 | for vote_status in self.rest.vote_plan_statuses()? { 27 | votes_count += vote_status 28 | .proposals 29 | .iter() 30 | .map(|x| x.votes_cast) 31 | .sum::(); 32 | } 33 | 34 | let fragment_logs = self.fragment_logs()?; 35 | 36 | Ok(Snapshot { 37 | timestamp: OffsetDateTime::now_utc(), 38 | pending: fragment_logs.iter().filter(|x| x.is_pending()).count(), 39 | total_tx: fragment_logs.len(), 40 | votes_count, 41 | }) 42 | } 43 | 44 | pub fn fragment_logs(&self) -> Result, super::Error> { 45 | let client = reqwest::blocking::Client::builder() 46 | .timeout(self.timeout) 47 | .build()?; 48 | 49 | let res = client 50 | .get(&format!("{}/v0/fragment/logs", self.endpoint)) 51 | .send()?; 52 | serde_json::from_str(&res.text()?).map_err(Into::into) 53 | } 54 | } 55 | 56 | pub struct Snapshot { 57 | pub timestamp: OffsetDateTime, 58 | pub votes_count: usize, 59 | pub pending: usize, 60 | pub total_tx: usize, 61 | } 62 | 63 | impl Snapshot { 64 | pub fn header(&self) -> String { 65 | "date,\tvotes-count,\tpending,\ttotal-tx".to_string() 66 | } 67 | 68 | pub fn entry(&self) -> String { 69 | format!( 70 | "{},\t{},\t{},\t{}", 71 | self.timestamp, self.votes_count, self.pending, self.total_tx 72 | ) 73 | } 74 | 75 | pub fn to_console_output(&self) -> String { 76 | format!( 77 | "date: {},\tvotes-count: {},\tpending: {},\ttotal-tx: {}", 78 | self.timestamp, self.votes_count, self.pending, self.total_tx 79 | ) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/stats/live/mod.rs: -------------------------------------------------------------------------------- 1 | mod harvester; 2 | mod monitor; 3 | mod settings; 4 | 5 | pub use harvester::Harvester; 6 | pub use monitor::start; 7 | pub use settings::Settings; 8 | 9 | use thiserror::Error; 10 | 11 | #[derive(Debug, Error)] 12 | pub enum Error { 13 | #[error(transparent)] 14 | Rest(#[from] jormungandr_automation::jormungandr::RestError), 15 | #[error(transparent)] 16 | Reqwest(#[from] reqwest::Error), 17 | #[error(transparent)] 18 | Serde(#[from] serde_json::Error), 19 | #[error(transparent)] 20 | Io(#[from] std::io::Error), 21 | } 22 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/stats/live/monitor.rs: -------------------------------------------------------------------------------- 1 | use std::time::Instant; 2 | 3 | use super::Error; 4 | use crate::stats::live::Harvester; 5 | use crate::stats::live::Settings; 6 | use jortestkit::console::ProgressBarMode; 7 | use jortestkit::load::ProgressBar; 8 | use jortestkit::prelude::append; 9 | 10 | pub fn start(harvester: Harvester, settings: Settings, title: &str) -> Result<(), Error> { 11 | let mut progress_bar = ProgressBar::new(1); 12 | 13 | println!("{}", title); 14 | jortestkit::load::use_as_monitor_progress_bar(&settings.monitor(), title, &mut progress_bar); 15 | let start = Instant::now(); 16 | 17 | loop { 18 | if settings.duration > start.elapsed().as_secs() { 19 | break; 20 | } 21 | 22 | let stats = harvester.harvest()?; 23 | 24 | match settings.progress_bar_mode() { 25 | ProgressBarMode::Standard => { 26 | println!("{}", stats.to_console_output()); 27 | } 28 | ProgressBarMode::Monitor => { 29 | progress_bar.set_message(&stats.to_console_output()); 30 | } 31 | _ => (), 32 | } 33 | 34 | if let Some(logger) = &settings.logger { 35 | if !logger.exists() { 36 | std::fs::File::create(logger)?; 37 | } 38 | append(logger, stats.header())?; 39 | } 40 | std::thread::sleep(std::time::Duration::from_secs(settings.interval)); 41 | } 42 | 43 | progress_bar.finish_and_clear(); 44 | Ok(()) 45 | } 46 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/stats/live/settings.rs: -------------------------------------------------------------------------------- 1 | use jortestkit::console::ProgressBarMode; 2 | use jortestkit::load::Monitor; 3 | use std::path::PathBuf; 4 | 5 | pub struct Settings { 6 | pub endpoint: String, 7 | pub interval: u64, 8 | pub logger: Option, 9 | pub progress: ProgressBarMode, 10 | pub duration: u64, 11 | } 12 | 13 | impl Settings { 14 | pub fn monitor(&self) -> Monitor { 15 | match self.progress { 16 | ProgressBarMode::Monitor => Monitor::Progress(self.interval), 17 | ProgressBarMode::Standard => Monitor::Standard(self.interval), 18 | ProgressBarMode::None => Monitor::Disabled(self.interval), 19 | } 20 | } 21 | 22 | pub fn progress_bar_mode(&self) -> ProgressBarMode { 23 | self.progress 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/stats/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod archive; 2 | pub mod distribution; 3 | pub mod live; 4 | pub mod snapshot; 5 | pub mod voters; 6 | 7 | use crate::stats::archive::{ArchiveCalculatorError, ArchiveReaderError}; 8 | use jormungandr_automation::testing::block0::Block0Error; 9 | use jormungandr_lib::interfaces::Block0ConfigurationError; 10 | use thiserror::Error; 11 | 12 | #[allow(clippy::large_enum_variant)] 13 | #[derive(Error, Debug)] 14 | pub enum Error { 15 | #[error("get block0")] 16 | GetBlock0(#[from] Block0Error), 17 | #[error("reqwest error")] 18 | Reqwest(#[from] reqwest::Error), 19 | #[error("block0 parse error")] 20 | Block0Parse(#[from] Block0ConfigurationError), 21 | #[error("io error")] 22 | Io(#[from] std::io::Error), 23 | #[error("read error")] 24 | Read(#[from] chain_ser::deser::ReadError), 25 | #[error("bech32 error")] 26 | Bech32(#[from] bech32::Error), 27 | #[error("csv error")] 28 | Csv(#[from] csv::Error), 29 | #[error("archive reader error")] 30 | ArchiveReader(#[from] ArchiveReaderError), 31 | #[error("archive calculator error")] 32 | ArchiveCalculator(#[from] ArchiveCalculatorError), 33 | #[error(transparent)] 34 | Serde(#[from] serde_json::Error), 35 | #[error(transparent)] 36 | Live(#[from] live::Error), 37 | } 38 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/stats/snapshot.rs: -------------------------------------------------------------------------------- 1 | use jormungandr_lib::interfaces::Initial; 2 | use std::path::Path; 3 | 4 | pub fn read_initials>(snapshot: S) -> Result, crate::stats::Error> { 5 | let snapshot = snapshot.into(); 6 | let value: serde_json::Value = serde_json::from_str(&snapshot)?; 7 | let initial = serde_json::to_string(&value["initial"])?; 8 | serde_json::from_str(&initial).map_err(Into::into) 9 | } 10 | 11 | pub fn read_initials_from_file>( 12 | initials: P, 13 | ) -> Result, crate::stats::Error> { 14 | let contents = std::fs::read_to_string(&initials)?; 15 | read_initials(contents) 16 | } 17 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/utils/csv.rs: -------------------------------------------------------------------------------- 1 | use serde::{de::DeserializeOwned, Serialize}; 2 | use std::fmt::Debug; 3 | use std::path::{Path, PathBuf}; 4 | 5 | pub fn load_data_from_csv( 6 | file_path: &Path, 7 | ) -> Result, csv::Error> { 8 | let mut csv_reader = csv::ReaderBuilder::new() 9 | .has_headers(true) 10 | .delimiter(DELIMITER) 11 | .from_path(file_path)?; 12 | csv_reader.deserialize().collect::, _>>() 13 | } 14 | 15 | pub fn dump_data_to_csv<'a, T: 'a + Serialize>( 16 | data: impl IntoIterator, 17 | file_path: &Path, 18 | ) -> Result<(), csv::Error> { 19 | let mut writer = csv::WriterBuilder::new() 20 | .has_headers(true) 21 | .from_path(file_path)?; 22 | for entry in data { 23 | writer.serialize(entry)?; 24 | } 25 | Ok(()) 26 | } 27 | 28 | pub fn dump_to_csv_or_print<'a, T: 'a + Serialize + Debug>( 29 | output: Option, 30 | result: impl Iterator + Debug, 31 | ) -> Result<(), std::io::Error> { 32 | if let Some(output) = &output { 33 | dump_data_to_csv(result, output)?; 34 | } else { 35 | let mut writer = csv::WriterBuilder::new() 36 | .has_headers(true) 37 | .from_writer(std::io::stdout()); 38 | for entry in result { 39 | writer.serialize(entry)?; 40 | } 41 | } 42 | Ok(()) 43 | } 44 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod csv; 2 | pub mod serde; 3 | 4 | use rust_decimal::Decimal; 5 | 6 | const DEFAULT_DECIMAL_PRECISION: u32 = 10; 7 | 8 | #[track_caller] 9 | pub fn assert_are_close(a: Decimal, b: Decimal) { 10 | assert_eq!( 11 | a.round_dp(DEFAULT_DECIMAL_PRECISION), 12 | b.round_dp(DEFAULT_DECIMAL_PRECISION) 13 | ); 14 | } 15 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/utils/serde.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Deserializer}; 2 | 3 | pub fn deserialize_truthy_falsy<'de, D>(deserializer: D) -> Result 4 | where 5 | D: Deserializer<'de>, 6 | { 7 | let truthy_value: String = String::deserialize(deserializer)?; 8 | Ok(matches!( 9 | truthy_value.to_lowercase().as_ref(), 10 | "x" | "1" | "true" 11 | )) 12 | } 13 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/vca_reviews/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::community_advisors::models::AdvisorReviewRow; 2 | use crate::utils; 3 | use vit_servicing_station_lib::db::models::community_advisors_reviews::AdvisorReview; 4 | 5 | use std::path::Path; 6 | 7 | #[derive(thiserror::Error, Debug)] 8 | pub enum Error { 9 | #[error(transparent)] 10 | CouldNotReadCsv(#[from] csv::Error), 11 | 12 | #[error("Couldn't parse advisor review tag for question: {0}")] 13 | CouldntParseTag(String), 14 | } 15 | 16 | impl AdvisorReviewRow { 17 | fn as_advisor_review(&self) -> AdvisorReview { 18 | AdvisorReview { 19 | id: 0, 20 | proposal_id: self.proposal_id.parse().unwrap(), 21 | assessor: self.assessor.clone(), 22 | impact_alignment_rating_given: self.impact_alignment_rating as i32, 23 | impact_alignment_note: self.impact_alignment_note.clone(), 24 | feasibility_rating_given: self.feasibility_rating as i32, 25 | feasibility_note: self.feasibility_note.clone(), 26 | auditability_rating_given: self.auditability_rating as i32, 27 | auditability_note: self.auditability_note.clone(), 28 | ranking: self.score().into(), 29 | } 30 | } 31 | } 32 | 33 | pub fn read_vca_reviews_aggregated_file(filepath: &Path) -> Result, Error> { 34 | Ok( 35 | utils::csv::load_data_from_csv::(filepath)? 36 | .into_iter() 37 | .map(|review| review.as_advisor_review()) 38 | .collect(), 39 | ) 40 | } 41 | -------------------------------------------------------------------------------- /catalyst-toolbox/src/vote_check/explorer.rs: -------------------------------------------------------------------------------- 1 | use graphql_client::GraphQLQuery; 2 | 3 | #[derive(GraphQLQuery)] 4 | #[graphql( 5 | query_path = "resources/explorer/transaction_by_id.graphql", 6 | schema_path = "resources/explorer/schema.graphql", 7 | response_derives = "Debug" 8 | )] 9 | pub struct TransactionById; 10 | -------------------------------------------------------------------------------- /catalyst-toolbox/tests/notifications/main.rs: -------------------------------------------------------------------------------- 1 | mod verifier; 2 | 3 | use crate::verifier::NotificationsVerifier; 4 | use assert_cmd::assert::OutputAssertExt; 5 | use assert_cmd::cargo::CommandCargoExt; 6 | use assert_fs::fixture::PathChild; 7 | use assert_fs::TempDir; 8 | use catalyst_toolbox::notifications::responses::create_message::CreateMessageResponse; 9 | use std::fs::File; 10 | use std::io::Write; 11 | use std::path::Path; 12 | use std::process::Command; 13 | 14 | #[test] 15 | pub fn sanity_notification() { 16 | let access_token = get_env("NOTIFICATION_ACCESS_TOKEN"); 17 | let app_token = get_env("NOTIFICATION_APP_CODE"); 18 | let message = "hello"; 19 | 20 | let temp_dir = TempDir::new().unwrap(); 21 | let file_path = temp_dir.child("notification.msg"); 22 | 23 | create_message_file(file_path.path(), message); 24 | 25 | let result = Command::cargo_bin("catalyst-toolbox") 26 | .unwrap() 27 | .arg("push") 28 | .arg("send") 29 | .arg("from-args") 30 | .arg("--access-token") 31 | .arg(&access_token) 32 | .arg("--application") 33 | .arg(&app_token) 34 | .arg(file_path.path()) 35 | .assert() 36 | .success(); 37 | 38 | let output = std::str::from_utf8(&result.get_output().stdout).unwrap(); 39 | let response: CreateMessageResponse = serde_json::from_str(output).unwrap(); 40 | println!("{:?}", response); 41 | let id = response.response.messages.get(0).unwrap(); 42 | NotificationsVerifier::new(&access_token) 43 | .verify_message_done_with_text(id, &message.to_string()); 44 | } 45 | 46 | fn get_env>(env_name: S) -> String { 47 | let env_name = env_name.into(); 48 | std::env::var(&env_name).unwrap_or_else(|_| panic!("{} not defined", env_name)) 49 | } 50 | 51 | fn create_message_file, S: Into>(path: P, message: S) { 52 | let mut file = File::create(path.as_ref()).unwrap(); 53 | let message = format!("\"{}\"", message.into()); 54 | file.write_all(message.as_bytes()).unwrap(); 55 | } 56 | -------------------------------------------------------------------------------- /catalyst-toolbox/tests/notifications/verifier.rs: -------------------------------------------------------------------------------- 1 | use reqwest::StatusCode; 2 | use serde::{Deserialize, Serialize}; 3 | use thiserror::Error; 4 | 5 | #[derive(Clone, Debug)] 6 | pub struct NotificationsVerifier { 7 | auth: String, 8 | } 9 | 10 | impl NotificationsVerifier { 11 | pub fn new>(auth: S) -> Self { 12 | Self { auth: auth.into() } 13 | } 14 | 15 | pub fn get_message_details>( 16 | &self, 17 | message_id: S, 18 | ) -> Result { 19 | let request = NotificationsRequest::new(&self.auth, &message_id.into()); 20 | println!("{:?}", request); 21 | let client = reqwest::blocking::Client::new(); 22 | let response = client 23 | .post("https://cp.pushwoosh.com/json/1.3/getMessageDetails") 24 | .json(&request) 25 | .send()?; 26 | 27 | assert_eq!(response.status(), StatusCode::OK, "post was unsuccesful"); 28 | let text = response.text()?; 29 | println!("{:#?}", &text); 30 | serde_json::from_str(&text).map_err(Into::into) 31 | } 32 | 33 | pub fn verify_message_done_with_text>(&self, message_id: S, text: S) { 34 | let message_id = message_id.into(); 35 | let response = self.get_message_details(&message_id).unwrap(); 36 | assert_eq!(response.status_code(), 200); 37 | assert!(response.has_response()); 38 | 39 | let message = response.get_message_unsafe(); 40 | 41 | assert_eq!(message.code, message_id); 42 | assert_eq!(message.content.default.as_ref().unwrap(), &text.into()); 43 | } 44 | } 45 | 46 | #[derive(Error, Debug)] 47 | pub enum NotificationsVerifierError { 48 | #[error("serizalization error")] 49 | SerdeError(#[from] serde_json::Error), 50 | #[error("send error")] 51 | ReqwestError(#[from] reqwest::Error), 52 | } 53 | 54 | #[derive(Serialize, Deserialize, Debug)] 55 | pub struct NotificationsRequest { 56 | request: Request, 57 | } 58 | 59 | impl NotificationsRequest { 60 | pub fn new>(auth: S, message: S) -> Self { 61 | Self { 62 | request: Request { 63 | auth: auth.into(), 64 | message: message.into(), 65 | }, 66 | } 67 | } 68 | } 69 | 70 | #[derive(Serialize, Deserialize, Debug)] 71 | pub struct Request { 72 | auth: String, 73 | message: String, 74 | } 75 | 76 | #[derive(Serialize, Deserialize, Debug)] 77 | pub struct MessageDetailsResponse { 78 | status_code: u32, 79 | status_message: String, 80 | response: Option, 81 | } 82 | 83 | impl MessageDetailsResponse { 84 | pub fn status_code(&self) -> u32 { 85 | self.status_code 86 | } 87 | 88 | pub fn has_response(&self) -> bool { 89 | self.response.is_some() 90 | } 91 | 92 | pub fn get_message_unsafe(&self) -> &Message { 93 | &self.response.as_ref().unwrap().message 94 | } 95 | } 96 | 97 | #[derive(Serialize, Deserialize, Debug)] 98 | pub struct Response { 99 | message: Message, 100 | } 101 | 102 | #[derive(Serialize, Deserialize, Debug)] 103 | pub struct Message { 104 | id: u64, 105 | created: String, 106 | send_date: String, 107 | status: String, 108 | content: Content, 109 | platforms: String, 110 | ignore_user_timezone: String, 111 | code: String, 112 | data: Option, 113 | tracking_code: Option, 114 | ios_title: Option, 115 | ios_subtitle: Option, 116 | ios_root_params: Option, 117 | android_header: Option, 118 | android_root_params: Option, 119 | conditions: Option, 120 | conditions_operator: String, 121 | } 122 | 123 | #[derive(Serialize, Deserialize, Debug)] 124 | pub struct Content { 125 | default: Option, 126 | } 127 | -------------------------------------------------------------------------------- /ci/release-info.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import re 4 | import sys 5 | from datetime import date 6 | from subprocess import Popen, PIPE 7 | 8 | 9 | def read_version(manifest_path, ref=None): 10 | """ 11 | Reads the package version from the manifest file, 12 | and optionally validates it against the given tag reference. 13 | """ 14 | p = Popen( 15 | ['cargo', 'read-manifest', '--manifest-path', manifest_path], 16 | stdout=PIPE 17 | ) 18 | d = json.load(p.stdout) 19 | version = d['version'] 20 | if ref is not None and ref != 'refs/tags/v' + version: 21 | print( 22 | '::error file={path}::version {0} does not match release tag {1}' 23 | .format(version, ref, path=manifest_path) 24 | ) 25 | sys.exit(1) 26 | return version 27 | 28 | 29 | event_name = sys.argv[1] 30 | 31 | date = date.today().strftime('%Y%m%d') 32 | 33 | ref = None 34 | if event_name == 'push': 35 | ref = os.getenv('GITHUB_REF') 36 | if ref.startswith('refs/tags/'): 37 | release_type = 'tagged' 38 | elif ref == 'refs/heads/ci/test/nightly': 39 | # emulate the nightly workflow 40 | release_type = 'nightly' 41 | ref = None 42 | else: 43 | raise ValueError('unexpected ref ' + ref) 44 | elif event_name == 'schedule': 45 | release_type = 'nightly' 46 | else: 47 | raise ValueError('unexpected event name ' + event_name) 48 | 49 | 50 | cargo_toml = './catalyst-toolbox/Cargo.toml' 51 | version = read_version(cargo_toml, ref) 52 | release_flags = '' 53 | if release_type == 'tagged': 54 | read_version(cargo_toml, ref) 55 | tag = 'v' + version 56 | elif release_type == 'nightly': 57 | version = re.sub( 58 | r'^(\d+\.\d+\.\d+)(-.*)?$', 59 | r'\1-nightly.' + date, 60 | version, 61 | ) 62 | tag = 'nightly.' + date 63 | release_flags = '--prerelease' 64 | 65 | for name in ('version', 'date', 'tag', 'release_type', 'release_flags'): 66 | print('::set-output name={0}::{1}'.format(name, globals()[name])) 67 | -------------------------------------------------------------------------------- /ci/strip-own-version-from-cargo-lock.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -p 2 | 3 | BEGIN { 4 | $ln = 0; $ours = 0; 5 | } 6 | 7 | if (/^\[\[package\]\]/ .. ($ln == 2)) { 8 | if (/^name = "vit-servicing-station-.*"/) { 9 | $ours = 1; 10 | } else { 11 | s/^version =.*// if $ours; 12 | } 13 | ++$ln; 14 | } else { 15 | $ln = 0; $ours = 0; 16 | } 17 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | ( 2 | import 3 | ( 4 | let 5 | lock = builtins.fromJSON (builtins.readFile ./flake.lock); 6 | in 7 | fetchTarball { 8 | url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; 9 | sha256 = lock.nodes.flake-compat.locked.narHash; 10 | } 11 | ) 12 | {src = ./.;} 13 | ) 14 | .defaultNix 15 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "flake-compat": { 4 | "flake": false, 5 | "locked": { 6 | "lastModified": 1650374568, 7 | "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=", 8 | "owner": "edolstra", 9 | "repo": "flake-compat", 10 | "rev": "b4a34015c698c7793d592d66adbab377907a2be8", 11 | "type": "github" 12 | }, 13 | "original": { 14 | "owner": "edolstra", 15 | "repo": "flake-compat", 16 | "type": "github" 17 | } 18 | }, 19 | "flake-utils": { 20 | "locked": { 21 | "lastModified": 1659877975, 22 | "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", 23 | "owner": "numtide", 24 | "repo": "flake-utils", 25 | "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", 26 | "type": "github" 27 | }, 28 | "original": { 29 | "owner": "numtide", 30 | "repo": "flake-utils", 31 | "type": "github" 32 | } 33 | }, 34 | "naersk": { 35 | "inputs": { 36 | "nixpkgs": [ 37 | "nixpkgs" 38 | ] 39 | }, 40 | "locked": { 41 | "lastModified": 1662220400, 42 | "narHash": "sha256-9o2OGQqu4xyLZP9K6kNe1pTHnyPz0Wr3raGYnr9AIgY=", 43 | "owner": "nix-community", 44 | "repo": "naersk", 45 | "rev": "6944160c19cb591eb85bbf9b2f2768a935623ed3", 46 | "type": "github" 47 | }, 48 | "original": { 49 | "owner": "nix-community", 50 | "repo": "naersk", 51 | "type": "github" 52 | } 53 | }, 54 | "nixpkgs": { 55 | "locked": { 56 | "lastModified": 1664281702, 57 | "narHash": "sha256-haixZ4TJLu1Dciow54wrHrHvlGDVr5sW6MTeAV/ZLuI=", 58 | "owner": "NixOS", 59 | "repo": "nixpkgs", 60 | "rev": "7e52b35fe98481a279d89f9c145f8076d049d2b9", 61 | "type": "github" 62 | }, 63 | "original": { 64 | "owner": "NixOS", 65 | "ref": "nixos-unstable", 66 | "repo": "nixpkgs", 67 | "type": "github" 68 | } 69 | }, 70 | "pre-commit-hooks": { 71 | "inputs": { 72 | "flake-utils": [ 73 | "flake-utils" 74 | ], 75 | "nixpkgs": [ 76 | "nixpkgs" 77 | ] 78 | }, 79 | "locked": { 80 | "lastModified": 1663082609, 81 | "narHash": "sha256-lmCCIu4dj59qbzkGKHQtolhpIEQMeAd2XUbXVPqgPYo=", 82 | "owner": "cachix", 83 | "repo": "pre-commit-hooks.nix", 84 | "rev": "60cad1a326df17a8c6cf2bb23436609fdd83024e", 85 | "type": "github" 86 | }, 87 | "original": { 88 | "owner": "cachix", 89 | "repo": "pre-commit-hooks.nix", 90 | "type": "github" 91 | } 92 | }, 93 | "root": { 94 | "inputs": { 95 | "flake-compat": "flake-compat", 96 | "flake-utils": "flake-utils", 97 | "naersk": "naersk", 98 | "nixpkgs": "nixpkgs", 99 | "pre-commit-hooks": "pre-commit-hooks", 100 | "rust-overlay": "rust-overlay" 101 | } 102 | }, 103 | "rust-overlay": { 104 | "inputs": { 105 | "flake-utils": [ 106 | "flake-utils" 107 | ], 108 | "nixpkgs": [ 109 | "nixpkgs" 110 | ] 111 | }, 112 | "locked": { 113 | "lastModified": 1664420529, 114 | "narHash": "sha256-hCSvJeoWZZbBTCR/QyazP+VzGHpuKfcgaPx2hQ90w7s=", 115 | "owner": "oxalica", 116 | "repo": "rust-overlay", 117 | "rev": "1601b5a28c50fd9d40bd61b8878f3499e09bce7a", 118 | "type": "github" 119 | }, 120 | "original": { 121 | "owner": "oxalica", 122 | "repo": "rust-overlay", 123 | "type": "github" 124 | } 125 | } 126 | }, 127 | "root": "root", 128 | "version": 7 129 | } 130 | -------------------------------------------------------------------------------- /shell.nix: -------------------------------------------------------------------------------- 1 | ( 2 | import 3 | ( 4 | let 5 | lock = builtins.fromJSON (builtins.readFile ./flake.lock); 6 | in 7 | fetchTarball { 8 | url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; 9 | sha256 = lock.nodes.flake-compat.locked.narHash; 10 | } 11 | ) 12 | {src = ./.;} 13 | ) 14 | .shellNix 15 | -------------------------------------------------------------------------------- /snapshot-lib/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "snapshot-lib" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | authors = ["Giacomo Pasini "] 7 | license = "MIT OR Apache-2.0" 8 | 9 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 10 | 11 | [dependencies] 12 | jormungandr-lib = { git = "https://github.com/input-output-hk/jormungandr.git", branch = "master" } 13 | serde = { version = "1", features = ["derive"] } 14 | proptest = { git = "https://github.com/input-output-hk/proptest.git", branch = "master", optional = true } 15 | chain-addr = { git = "https://github.com/input-output-hk/chain-libs.git", branch = "master", optional = true } 16 | test-strategy = { version = "0.2", optional = true } 17 | serde_test = { version = "1", optional = true } 18 | hex = { version = "0.4" } 19 | thiserror = "1.0" 20 | fraction = { version = "0.10", features = ["with-serde-support"] } 21 | reqwest = { version = "0.11", features = ["blocking", "json"] } 22 | bech32 = "0.8.1" 23 | graphql_client = { version = "0.10" } 24 | chain-crypto = { git = "https://github.com/input-output-hk/chain-libs.git", branch = "master" } 25 | rust_decimal = "1.16" 26 | rust_decimal_macros = "1" 27 | 28 | [dev-dependencies] 29 | serde_test = "1" 30 | test-strategy = "0.2" 31 | serde_json = "1.0" 32 | serde_yaml = "0.8.17" 33 | proptest = { git = "https://github.com/input-output-hk/proptest.git", branch = "master" } 34 | chain-addr = { git = "https://github.com/input-output-hk/chain-libs.git", branch = "master" } 35 | 36 | [features] 37 | proptest = ["dep:proptest", "dep:chain-addr", "dep:test-strategy", "dep:serde_test"] 38 | test-api = [] 39 | -------------------------------------------------------------------------------- /snapshot-lib/resources/repsdb/all_representatives.graphql: -------------------------------------------------------------------------------- 1 | query AllReps { 2 | representatives { 3 | data { 4 | attributes { 5 | address 6 | } 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /snapshot-lib/src/voter_hir.rs: -------------------------------------------------------------------------------- 1 | use ::serde::{Deserialize, Serialize}; 2 | use jormungandr_lib::{crypto::account::Identifier, interfaces::Value}; 3 | 4 | pub type VotingGroup = String; 5 | 6 | /// Define High Level Intermediate Representation (HIR) for voting 7 | /// entities in the Catalyst ecosystem. 8 | /// 9 | /// This is intended as a high level description of the setup, which is not 10 | /// enough on its own to spin a blockchain, but it's slimmer, easier to understand 11 | /// and free from implementation constraints. 12 | /// 13 | /// You can roughly read this as 14 | /// "voting_key will participate in this voting round with role voting_group and will have voting_power influence" 15 | #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Hash, Eq)] 16 | pub struct VoterHIR { 17 | // Keep hex encoding as in CIP-36 18 | #[serde(with = "serde")] 19 | pub voting_key: Identifier, 20 | /// Voting group this key belongs to. 21 | /// If this key belong to multiple voting groups, multiple records for the same 22 | /// key will be used. 23 | pub voting_group: VotingGroup, 24 | /// Voting power as processed by the snapshot 25 | pub voting_power: Value, 26 | } 27 | 28 | mod serde { 29 | use super::*; 30 | use ::serde::{de::Error, Deserializer, Serializer}; 31 | 32 | pub fn serialize(voting_key: &Identifier, serializer: S) -> Result 33 | where 34 | S: Serializer, 35 | { 36 | serializer.serialize_str(&voting_key.to_hex()) 37 | } 38 | 39 | pub fn deserialize<'de, D>(deserializer: D) -> Result 40 | where 41 | D: Deserializer<'de>, 42 | { 43 | let hex = String::deserialize(deserializer)?; 44 | Identifier::from_hex(hex.trim_start_matches("0x")) 45 | .map_err(|e| D::Error::custom(format!("invalid public key: {}", e))) 46 | } 47 | } 48 | 49 | #[cfg(any(test, feature = "proptest"))] 50 | pub mod tests { 51 | use super::*; 52 | use ::proptest::{prelude::*, strategy::BoxedStrategy}; 53 | use jormungandr_lib::crypto::account::Identifier; 54 | use std::ops::Range; 55 | 56 | impl Arbitrary for VoterHIR { 57 | type Parameters = (String, VpRange); 58 | type Strategy = BoxedStrategy; 59 | fn arbitrary_with(args: Self::Parameters) -> Self::Strategy { 60 | (any::<([u8; 32])>(), args.1 .0) 61 | .prop_map(move |(key, voting_power)| VoterHIR { 62 | voting_key: Identifier::from_hex(&hex::encode(key)).unwrap(), 63 | voting_power: voting_power.into(), 64 | voting_group: args.0.clone(), 65 | }) 66 | .boxed() 67 | } 68 | } 69 | 70 | pub struct VpRange(Range); 71 | 72 | impl VpRange { 73 | pub const fn ada_distribution() -> Self { 74 | Self(1..45_000_000_000) 75 | } 76 | } 77 | 78 | impl Default for VpRange { 79 | fn default() -> Self { 80 | Self(0..u64::MAX) 81 | } 82 | } 83 | 84 | impl From> for VpRange { 85 | fn from(range: Range) -> Self { 86 | Self(range) 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /snapshot-lib/src/voting_group.rs: -------------------------------------------------------------------------------- 1 | use crate::VotingGroup; 2 | use graphql_client::GraphQLQuery; 3 | use jormungandr_lib::crypto::account::Identifier; 4 | use std::collections::HashSet; 5 | use thiserror::Error; 6 | 7 | pub const DEFAULT_DIRECT_VOTER_GROUP: &str = "direct"; 8 | pub const DEFAULT_REPRESENTATIVE_GROUP: &str = "rep"; 9 | 10 | pub trait VotingGroupAssigner { 11 | fn assign(&self, vk: &Identifier) -> VotingGroup; 12 | } 13 | 14 | pub struct RepsVotersAssigner { 15 | direct_voters: VotingGroup, 16 | reps: VotingGroup, 17 | repsdb: HashSet, 18 | } 19 | 20 | #[derive(Debug, Error)] 21 | pub enum Error { 22 | #[error(transparent)] 23 | Io(#[from] reqwest::Error), 24 | } 25 | 26 | #[derive(GraphQLQuery)] 27 | #[graphql( 28 | query_path = "resources/repsdb/all_representatives.graphql", 29 | schema_path = "resources/repsdb/schema.graphql", 30 | response_derives = "Debug" 31 | )] 32 | pub struct AllReps; 33 | 34 | #[allow(dead_code)] 35 | fn get_all_reps(_url: impl reqwest::IntoUrl) -> Result, Error> { 36 | // let response: all_reps::ResponseData = reqwest::blocking::Client::new() 37 | // .post(url) 38 | // .json(&AllReps::build_query(all_reps::Variables)) 39 | // .send()? 40 | // .json()?; 41 | 42 | // Ok(response 43 | // .representatives 44 | // .iter() 45 | // .flat_map(|reps| reps.data.iter()) 46 | // .flat_map(|rep| rep.attributes.as_ref()) 47 | // .flat_map(|attributes| attributes.address.as_ref()) 48 | // .flat_map(|addr| Identifier::from_hex(addr)) 49 | // .collect()) 50 | todo!() 51 | } 52 | 53 | impl RepsVotersAssigner { 54 | pub fn new(direct_voters: VotingGroup, reps: VotingGroup) -> Self { 55 | Self { 56 | direct_voters, 57 | reps, 58 | repsdb: HashSet::new(), 59 | } 60 | } 61 | 62 | #[cfg(feature = "test-api")] 63 | pub fn new_from_repsdb( 64 | direct_voters: VotingGroup, 65 | reps: VotingGroup, 66 | repsdb: HashSet, 67 | ) -> Result { 68 | Ok(Self { 69 | direct_voters, 70 | reps, 71 | repsdb, 72 | }) 73 | } 74 | } 75 | 76 | impl VotingGroupAssigner for RepsVotersAssigner { 77 | fn assign(&self, vk: &Identifier) -> VotingGroup { 78 | if self.repsdb.contains(vk) { 79 | self.reps.clone() 80 | } else { 81 | self.direct_voters.clone() 82 | } 83 | } 84 | } 85 | 86 | #[cfg(any(test, feature = "test-api", feature = "proptest"))] 87 | impl VotingGroupAssigner for F 88 | where 89 | F: Fn(&Identifier) -> VotingGroup, 90 | { 91 | fn assign(&self, vk: &Identifier) -> VotingGroup { 92 | self(vk) 93 | } 94 | } 95 | --------------------------------------------------------------------------------