├── .dockerignore ├── .editorconfig ├── .git-blame-ignore-revs ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── actions │ └── nimbus-build-system │ │ └── action.yml └── workflows │ ├── Readme.md │ ├── ci-reusable.yml │ ├── ci.yml │ ├── docker-dist-tests.yml │ ├── docker-reusable.yml │ ├── docker.yml │ ├── docs.yml │ ├── nim-matrix.yml │ └── release.yml ├── .gitignore ├── .gitmodules ├── Jenkinsfile ├── LICENSE-APACHEv2 ├── LICENSE-MIT ├── Makefile ├── README.md ├── benchmarks ├── .gitignore ├── README.md ├── config.nims ├── create_circuits.nim ├── run_benchmarks.nim └── utils.nim ├── build.nims ├── codecov.yml ├── codex.nim ├── codex.nimble ├── codex ├── blockexchange.nim ├── blockexchange │ ├── engine.nim │ ├── engine │ │ ├── advertiser.nim │ │ ├── discovery.nim │ │ ├── engine.nim │ │ ├── payments.nim │ │ └── pendingblocks.nim │ ├── network.nim │ ├── network │ │ ├── network.nim │ │ └── networkpeer.nim │ ├── peers.nim │ ├── peers │ │ ├── peercontext.nim │ │ └── peerctxstore.nim │ └── protobuf │ │ ├── blockexc.nim │ │ ├── message.nim │ │ ├── message.proto │ │ ├── message.proto.license │ │ ├── payments.nim │ │ └── presence.nim ├── blocktype.nim ├── chunker.nim ├── clock.nim ├── codex.nim ├── codextypes.nim ├── conf.nim ├── contracts.nim ├── contracts │ ├── Readme.md │ ├── clock.nim │ ├── config.nim │ ├── deployment.nim │ ├── interactions.nim │ ├── interactions │ │ ├── clientinteractions.nim │ │ ├── hostinteractions.nim │ │ ├── interactions.nim │ │ └── validatorinteractions.nim │ ├── market.nim │ ├── marketplace.nim │ ├── proofs.nim │ ├── provider.nim │ └── requests.nim ├── discovery.nim ├── erasure.nim ├── erasure │ ├── backend.nim │ ├── backends │ │ └── leopard.nim │ └── erasure.nim ├── errors.nim ├── indexingstrategy.nim ├── logutils.nim ├── manifest.nim ├── manifest │ ├── coders.nim │ └── manifest.nim ├── market.nim ├── merkletree.nim ├── merkletree │ ├── codex.nim │ ├── codex │ │ ├── coders.nim │ │ └── codex.nim │ ├── merkletree.nim │ └── poseidon2.nim ├── namespaces.nim ├── nat.nim ├── node.nim ├── periods.nim ├── purchasing.nim ├── purchasing │ ├── purchase.nim │ ├── purchaseid.nim │ ├── statemachine.nim │ └── states │ │ ├── cancelled.nim │ │ ├── error.nim │ │ ├── failed.nim │ │ ├── finished.nim │ │ ├── pending.nim │ │ ├── started.nim │ │ ├── submitted.nim │ │ └── unknown.nim ├── rest │ ├── api.nim │ ├── coders.nim │ └── json.nim ├── rng.nim ├── sales.nim ├── sales │ ├── reservations.nim │ ├── salesagent.nim │ ├── salescontext.nim │ ├── salesdata.nim │ ├── slotqueue.nim │ ├── statemachine.nim │ └── states │ │ ├── cancelled.nim │ │ ├── downloading.nim │ │ ├── errored.nim │ │ ├── failed.nim │ │ ├── filled.nim │ │ ├── filling.nim │ │ ├── finished.nim │ │ ├── ignored.nim │ │ ├── initialproving.nim │ │ ├── payout.nim │ │ ├── preparing.nim │ │ ├── proving.nim │ │ ├── provingsimulated.nim │ │ ├── slotreserving.nim │ │ └── unknown.nim ├── slots.nim ├── slots │ ├── builder.nim │ ├── builder │ │ └── builder.nim │ ├── converters.nim │ ├── proofs.nim │ ├── proofs │ │ ├── backendfactory.nim │ │ ├── backends.nim │ │ ├── backends │ │ │ ├── circomcompat.nim │ │ │ └── converters.nim │ │ ├── backendutils.nim │ │ └── prover.nim │ ├── sampler.nim │ ├── sampler │ │ ├── sampler.nim │ │ └── utils.nim │ └── types.nim ├── stores.nim ├── stores │ ├── blockstore.nim │ ├── cachestore.nim │ ├── keyutils.nim │ ├── maintenance.nim │ ├── networkstore.nim │ ├── queryiterhelper.nim │ ├── repostore.nim │ ├── repostore │ │ ├── coders.nim │ │ ├── operations.nim │ │ ├── store.nim │ │ └── types.nim │ └── treehelper.nim ├── streams.nim ├── streams │ ├── asyncstreamwrapper.nim │ ├── seekablestream.nim │ └── storestream.nim ├── systemclock.nim ├── units.nim ├── utils.nim ├── utils │ ├── addrutils.nim │ ├── arrayutils.nim │ ├── asyncheapqueue.nim │ ├── asynciter.nim │ ├── asyncstatemachine.nim │ ├── digest.nim │ ├── exceptions.nim │ ├── fileutils.nim │ ├── iter.nim │ ├── json.nim │ ├── keyutils.nim │ ├── natutils.nim │ ├── options.nim │ ├── poseidon2digest.nim │ ├── safeasynciter.nim │ ├── stintutils.nim │ ├── timer.nim │ └── trackedfutures.nim ├── validation.nim └── validationconfig.nim ├── config.nims ├── docker ├── README.md ├── codex.Dockerfile ├── docker-compose.yaml └── docker-entrypoint.sh ├── env.sh ├── flake.lock ├── flake.nix ├── metrics ├── README.md ├── assets │ ├── import.png │ ├── imported.png │ └── main.png ├── codex-grafana-dashboard.json └── prometheus.yml ├── nix ├── README.md ├── checksums.nix ├── csources.nix ├── default.nix ├── nimble.nix ├── service.nix └── tools.nix ├── openapi.yaml ├── redocly.yaml ├── tests ├── asynctest.nim ├── checktest.nim ├── circuits │ └── fixtures │ │ ├── input.json │ │ ├── proof_main.r1cs │ │ ├── proof_main.wasm │ │ └── proof_main.zkey ├── codex │ ├── blockexchange │ │ ├── discovery │ │ │ ├── testdiscovery.nim │ │ │ └── testdiscoveryengine.nim │ │ ├── engine │ │ │ ├── testadvertiser.nim │ │ │ ├── testblockexc.nim │ │ │ ├── testengine.nim │ │ │ └── testpayments.nim │ │ ├── protobuf │ │ │ ├── testpayments.nim │ │ │ └── testpresence.nim │ │ ├── testdiscovery.nim │ │ ├── testengine.nim │ │ ├── testnetwork.nim │ │ ├── testpeerctxstore.nim │ │ ├── testpendingblocks.nim │ │ └── testprotobuf.nim │ ├── examples.nim │ ├── helpers.nim │ ├── helpers │ │ ├── always.nim │ │ ├── mockchunker.nim │ │ ├── mockclock.nim │ │ ├── mockdiscovery.nim │ │ ├── mockmarket.nim │ │ ├── mockrepostore.nim │ │ ├── mockreservations.nim │ │ ├── mocksalesagent.nim │ │ ├── mockslotqueueitem.nim │ │ ├── mocktimer.nim │ │ ├── nodeutils.nim │ │ └── randomchunker.nim │ ├── merkletree │ │ ├── generictreetests.nim │ │ ├── helpers.nim │ │ ├── testcodexcoders.nim │ │ ├── testcodextree.nim │ │ ├── testmerkledigest.nim │ │ └── testposeidon2tree.nim │ ├── node │ │ ├── helpers.nim │ │ ├── testcontracts.nim │ │ └── testnode.nim │ ├── sales │ │ ├── helpers │ │ │ └── periods.nim │ │ ├── states │ │ │ ├── testcancelled.nim │ │ │ ├── testdownloading.nim │ │ │ ├── testerrored.nim │ │ │ ├── testfilled.nim │ │ │ ├── testfilling.nim │ │ │ ├── testfinished.nim │ │ │ ├── testignored.nim │ │ │ ├── testinitialproving.nim │ │ │ ├── testpayout.nim │ │ │ ├── testpreparing.nim │ │ │ ├── testproving.nim │ │ │ ├── testsimulatedproving.nim │ │ │ ├── testslotreserving.nim │ │ │ └── testunknown.nim │ │ ├── testreservations.nim │ │ ├── testsales.nim │ │ ├── testsalesagent.nim │ │ ├── testslotqueue.nim │ │ └── teststates.nim │ ├── slots │ │ ├── backends │ │ │ ├── helpers.nim │ │ │ └── testcircomcompat.nim │ │ ├── helpers.nim │ │ ├── sampler │ │ │ ├── testsampler.nim │ │ │ └── testutils.nim │ │ ├── testbackendfactory.nim │ │ ├── testbackends.nim │ │ ├── testconverters.nim │ │ ├── testprover.nim │ │ ├── testsampler.nim │ │ └── testslotbuilder.nim │ ├── stores │ │ ├── commonstoretests.nim │ │ ├── repostore │ │ │ └── testcoders.nim │ │ ├── testcachestore.nim │ │ ├── testkeyutils.nim │ │ ├── testmaintenance.nim │ │ ├── testqueryiterhelper.nim │ │ └── testrepostore.nim │ ├── testasyncheapqueue.nim │ ├── testasyncstreamwrapper.nim │ ├── testblockexchange.nim │ ├── testchunking.nim │ ├── testclock.nim │ ├── testerasure.nim │ ├── testindexingstrategy.nim │ ├── testlogutils.nim │ ├── testmanifest.nim │ ├── testmerkletree.nim │ ├── testnat.nim │ ├── testnode.nim │ ├── testpurchasing.nim │ ├── testsales.nim │ ├── testslots.nim │ ├── teststores.nim │ ├── teststorestream.nim │ ├── testsystemclock.nim │ ├── testutils.nim │ ├── testvalidation.nim │ └── utils │ │ ├── testasynciter.nim │ │ ├── testasyncstatemachine.nim │ │ ├── testiter.nim │ │ ├── testkeyutils.nim │ │ ├── testoptions.nim │ │ ├── testsafeasynciter.nim │ │ ├── testtimer.nim │ │ ├── testtrackedfutures.nim │ │ └── testutils.nim ├── config.nims ├── contracts │ ├── deployment.nim │ ├── examples.nim │ ├── helpers │ │ └── mockprovider.nim │ ├── nim.cfg │ ├── testClock.nim │ ├── testContracts.nim │ ├── testDeployment.nim │ ├── testMarket.nim │ ├── testProvider.nim │ └── time.nim ├── coverage.nim ├── coverage.nims ├── ethertest.nim ├── examples.nim ├── fixtures │ └── test.jpg ├── helpers.nim ├── helpers │ ├── multisetup.nim │ ├── templeveldb.nim │ └── trackers.nim ├── integration │ ├── clioption.nim │ ├── codexclient.nim │ ├── codexconfig.nim │ ├── codexprocess.nim │ ├── hardhatconfig.nim │ ├── hardhatprocess.nim │ ├── marketplacesuite.nim │ ├── multinodes.nim │ ├── nodeconfig.nim │ ├── nodeconfigs.nim │ ├── nodeprocess.nim │ ├── testblockexpiration.nim │ ├── testcli.nim │ ├── testecbug.nim │ ├── testmarketplace.nim │ ├── testproofs.nim │ ├── testpurchasing.nim │ ├── testrestapi.nim │ ├── testrestapivalidation.nim │ ├── testsales.nim │ ├── testupdownload.nim │ ├── testvalidator.nim │ └── twonodes.nim ├── logging.nim ├── nimlldb.py ├── testCodex.nim ├── testContracts.nim ├── testIntegration.nim ├── testTaiko.nim ├── testTools.nim └── tools │ └── cirdl │ └── testcirdl.nim ├── tools ├── cirdl │ └── cirdl.nim └── scripts │ └── git_pre_commit_format.sh └── vendor └── urls.rules /.dockerignore: -------------------------------------------------------------------------------- 1 | .github 2 | build 3 | docs 4 | metrics 5 | nimcache 6 | tests 7 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | indent_style = space 3 | insert_final_newline = true 4 | indent_size = 2 5 | trim_trailing_whitespace = true -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Formatted with nph v0.6.1-0-g0d8000e 2 | e5df8c50d3b6e70e6eec1ff031657d2b7bb6fe63 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG]" 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Environment:** 27 | - OS: [e.g. MacOs 12.2.1] 28 | - Version: [e.g. v1.0.0] 29 | - Tag/Commit [e.g. 824296e695842b2]: 30 | - Arch [e.g. amd64]: 31 | 32 | **Additional context** 33 | Add any other context about the problem here. 34 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/docker-dist-tests.yml: -------------------------------------------------------------------------------- 1 | name: Docker - Dist-Tests 2 | 3 | 4 | on: 5 | push: 6 | branches: 7 | - master 8 | tags: 9 | - 'v*.*.*' 10 | paths-ignore: 11 | - '**/*.md' 12 | - '.gitignore' 13 | - '.github/**' 14 | - '!.github/workflows/docker-dist-tests.yml' 15 | - '!.github/workflows/docker-reusable.yml' 16 | - 'docker/**' 17 | - '!docker/codex.Dockerfile' 18 | - '!docker/docker-entrypoint.sh' 19 | workflow_dispatch: 20 | inputs: 21 | run_release_tests: 22 | description: Run Release tests 23 | required: false 24 | type: boolean 25 | default: false 26 | 27 | 28 | jobs: 29 | get-contracts-hash: 30 | runs-on: ubuntu-latest 31 | outputs: 32 | hash: ${{ steps.get-hash.outputs.hash }} 33 | steps: 34 | - uses: actions/checkout@v4 35 | with: 36 | submodules: true 37 | 38 | - name: Get submodule short hash 39 | id: get-hash 40 | run: | 41 | hash=$(git rev-parse --short HEAD:vendor/codex-contracts-eth) 42 | echo "hash=$hash" >> $GITHUB_OUTPUT 43 | build-and-push: 44 | name: Build and Push 45 | uses: ./.github/workflows/docker-reusable.yml 46 | needs: get-contracts-hash 47 | with: 48 | nimflags: '-d:disableMarchNative -d:codex_enable_api_debug_peers=true -d:codex_enable_proof_failures=true -d:codex_enable_log_counter=true -d:verify_circuit=true' 49 | nat_ip_auto: true 50 | tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }} 51 | tag_suffix: dist-tests 52 | contract_image: "codexstorage/codex-contracts-eth:sha-${{ needs.get-contracts-hash.outputs.hash }}-dist-tests" 53 | run_release_tests: ${{ inputs.run_release_tests }} 54 | secrets: inherit 55 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Docker 2 | 3 | 4 | on: 5 | push: 6 | branches: 7 | - master 8 | tags: 9 | - 'v*.*.*' 10 | paths-ignore: 11 | - '**/*.md' 12 | - '.gitignore' 13 | - '.github/**' 14 | - '!.github/workflows/docker.yml' 15 | - '!.github/workflows/docker-reusable.yml' 16 | - 'docker/**' 17 | - '!docker/codex.Dockerfile' 18 | - '!docker/docker-entrypoint.sh' 19 | workflow_dispatch: 20 | 21 | 22 | jobs: 23 | get-contracts-hash: 24 | runs-on: ubuntu-latest 25 | outputs: 26 | hash: ${{ steps.get-hash.outputs.hash }} 27 | steps: 28 | - uses: actions/checkout@v4 29 | with: 30 | submodules: true 31 | 32 | - name: Get submodule short hash 33 | id: get-hash 34 | run: | 35 | hash=$(git rev-parse --short HEAD:vendor/codex-contracts-eth) 36 | echo "hash=$hash" >> $GITHUB_OUTPUT 37 | build-and-push: 38 | name: Build and Push 39 | uses: ./.github/workflows/docker-reusable.yml 40 | needs: get-contracts-hash 41 | with: 42 | tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }} 43 | contract_image: "codexstorage/codex-contracts-eth:sha-${{ needs.get-contracts-hash.outputs.hash }}" 44 | secrets: inherit -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: OpenAPI 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*.*.*" 7 | paths: 8 | - "openapi.yaml" 9 | - ".github/workflows/docs.yml" 10 | pull_request: 11 | branches: 12 | - "**" 13 | paths: 14 | - "openapi.yaml" 15 | - ".github/workflows/docs.yml" 16 | 17 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 18 | permissions: 19 | contents: read 20 | pages: write 21 | id-token: write 22 | 23 | jobs: 24 | lint: 25 | name: Lint 26 | runs-on: ubuntu-latest 27 | steps: 28 | - name: Checkout 29 | uses: actions/checkout@v4 30 | with: 31 | fetch-depth: 0 32 | 33 | - uses: actions/setup-node@v4 34 | with: 35 | node-version: 18 36 | 37 | - name: Lint OpenAPI 38 | run: npx @redocly/cli lint openapi.yaml 39 | 40 | deploy: 41 | name: Deploy 42 | runs-on: ubuntu-latest 43 | if: startsWith(github.ref, 'refs/tags/') 44 | steps: 45 | - name: Checkout 46 | uses: actions/checkout@v4 47 | with: 48 | fetch-depth: 0 49 | 50 | - uses: actions/setup-node@v4 51 | with: 52 | node-version: 18 53 | 54 | - name: Build OpenAPI 55 | run: npx @redocly/cli build-docs openapi.yaml --output openapi/index.html --title "Codex API" 56 | 57 | - name: Build Postman Collection 58 | run: npx -y openapi-to-postmanv2 -s openapi.yaml -o openapi/postman.json -p -O folderStrategy=Tags,includeAuthInfoInExample=false 59 | 60 | - name: Upload artifact 61 | uses: actions/upload-pages-artifact@v3 62 | with: 63 | path: openapi 64 | 65 | - name: Deploy to GitHub Pages 66 | uses: actions/deploy-pages@v4 67 | -------------------------------------------------------------------------------- /.github/workflows/nim-matrix.yml: -------------------------------------------------------------------------------- 1 | name: Nim matrix 2 | 3 | on: 4 | merge_group: 5 | workflow_dispatch: 6 | 7 | env: 8 | cache_nonce: 0 # Allows for easily busting actions/cache caches 9 | nim_version: pinned 10 | 11 | jobs: 12 | matrix: 13 | runs-on: ubuntu-latest 14 | outputs: 15 | matrix: ${{ steps.matrix.outputs.matrix }} 16 | cache_nonce: ${{ env.cache_nonce }} 17 | steps: 18 | - name: Compute matrix 19 | id: matrix 20 | uses: fabiocaccamo/create-matrix-action@v5 21 | with: 22 | matrix: | 23 | os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} 24 | os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} 25 | os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} 26 | os {linux}, cpu {amd64}, builder {ubuntu-latest}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} 27 | 28 | build: 29 | needs: matrix 30 | uses: ./.github/workflows/ci-reusable.yml 31 | with: 32 | matrix: ${{ needs.matrix.outputs.matrix }} 33 | cache_nonce: ${{ needs.matrix.outputs.cache_nonce }} 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !*/ 3 | !*.* 4 | *.exe 5 | 6 | !LICENSE* 7 | !Makefile 8 | !Jenkinsfile 9 | 10 | nimcache/ 11 | 12 | # Executables when using nix will be stored in result/ directory 13 | result/ 14 | 15 | # Executables shall be put in an ignored build/ directory 16 | build/ 17 | 18 | # Coverage data shall be put in an ignored coverage/ directory 19 | coverage/ 20 | 21 | # Nimble packages 22 | /vendor/.nimble 23 | /vendor/packages/ 24 | # /vendor/*/ 25 | 26 | # Nimble user files 27 | nimble.develop 28 | nimble.paths 29 | 30 | # vscode 31 | .vscode 32 | 33 | # JetBrain's IDEs 34 | .idea 35 | 36 | # Each developer can create a personal .env file with 37 | # local settings overrides (e.g. WEB3_URL) 38 | .env 39 | 40 | .update.timestamp 41 | codex.nims 42 | nimbus-build-system.paths 43 | docker/hostdatadir 44 | docker/prometheus-data 45 | .DS_Store 46 | nim.cfg 47 | tests/integration/logs 48 | 49 | data/ 50 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env groovy 2 | library 'status-jenkins-lib@v1.9.13' 3 | 4 | pipeline { 5 | agent { label 'linux && x86_64 && nix-2.24' } 6 | 7 | options { 8 | disableConcurrentBuilds() 9 | /* manage how many builds we keep */ 10 | buildDiscarder(logRotator( 11 | numToKeepStr: '20', 12 | daysToKeepStr: '30', 13 | )) 14 | } 15 | 16 | stages { 17 | stage('Build') { 18 | steps { 19 | script { 20 | nix.flake("default") 21 | } 22 | } 23 | } 24 | 25 | stage('Check') { 26 | steps { 27 | script { 28 | sh './result/bin/codex --version' 29 | } 30 | } 31 | } 32 | } 33 | 34 | post { 35 | cleanup { cleanWs() } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /benchmarks/.gitignore: -------------------------------------------------------------------------------- 1 | ceremony 2 | circuit_bench_* 3 | -------------------------------------------------------------------------------- /benchmarks/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## Benchmark Runner 3 | 4 | Modify `runAllBenchmarks` proc in `run_benchmarks.nim` to the desired parameters and variations. 5 | 6 | Then run it: 7 | 8 | ```sh 9 | nim c -r run_benchmarks 10 | ``` 11 | 12 | By default all circuit files for each combinations of circuit args will be generated in a unique folder named like: 13 | nim-codex/benchmarks/circuit_bench_depth32_maxslots256_cellsize2048_blocksize65536_nsamples9_entropy1234567_seed12345_nslots11_ncells512_index3 14 | 15 | Generating the circuit files often takes longer than running benchmarks, so caching the results allows re-running the benchmark as needed. 16 | 17 | You can modify the `CircuitArgs` and `CircuitEnv` objects in `runAllBenchMarks` to suite your needs. See `create_circuits.nim` for their definition. 18 | 19 | The runner executes all commands relative to the `nim-codex` repo. This simplifies finding the correct circuit includes paths, etc. `CircuitEnv` sets all of this. 20 | 21 | ## Codex Ark Circom CLI 22 | 23 | Runs Codex's prover setup with Ark / Circom. 24 | 25 | Compile: 26 | ```sh 27 | nim c codex_ark_prover_cli.nim 28 | ``` 29 | 30 | Run to see usage: 31 | ```sh 32 | ./codex_ark_prover_cli.nim -h 33 | ``` 34 | -------------------------------------------------------------------------------- /benchmarks/config.nims: -------------------------------------------------------------------------------- 1 | --path: 2 | ".." 3 | --path: 4 | "../tests" 5 | --threads: 6 | on 7 | --tlsEmulation: 8 | off 9 | --d: 10 | release 11 | 12 | # when not defined(chronicles_log_level): 13 | # --define:"chronicles_log_level:NONE" # compile all log statements 14 | # --define:"chronicles_sinks:textlines[dynamic]" # allow logs to be filtered at runtime 15 | # --"import":"logging" # ensure that logging is ignored at runtime 16 | -------------------------------------------------------------------------------- /benchmarks/utils.nim: -------------------------------------------------------------------------------- 1 | import std/tables 2 | 3 | template withDir*(dir: string, blk: untyped) = 4 | ## set working dir for duration of blk 5 | let prev = getCurrentDir() 6 | try: 7 | setCurrentDir(dir) 8 | `blk` 9 | finally: 10 | setCurrentDir(prev) 11 | 12 | template runit*(cmd: string) = 13 | ## run shell commands and verify it runs without an error code 14 | echo "RUNNING: ", cmd 15 | let cmdRes = execShellCmd(cmd) 16 | echo "STATUS: ", cmdRes 17 | assert cmdRes == 0 18 | 19 | var benchRuns* = newTable[string, tuple[avgTimeSec: float, count: int]]() 20 | 21 | func avg(vals: openArray[float]): float = 22 | for v in vals: 23 | result += v / vals.len().toFloat() 24 | 25 | template benchmark*(name: untyped, count: int, blk: untyped) = 26 | let benchmarkName: string = name 27 | ## simple benchmarking of a block of code 28 | var runs = newSeqOfCap[float](count) 29 | for i in 1 .. count: 30 | block: 31 | let t0 = epochTime() 32 | `blk` 33 | let elapsed = epochTime() - t0 34 | runs.add elapsed 35 | 36 | var elapsedStr = "" 37 | for v in runs: 38 | elapsedStr &= ", " & v.formatFloat(format = ffDecimal, precision = 3) 39 | stdout.styledWriteLine( 40 | fgGreen, "CPU Time [", benchmarkName, "] ", "avg(", $count, "): ", elapsedStr, " s" 41 | ) 42 | benchRuns[benchmarkName] = (runs.avg(), count) 43 | 44 | template printBenchMarkSummaries*(printRegular=true, printTsv=true) = 45 | if printRegular: 46 | echo "" 47 | for k, v in benchRuns: 48 | echo "Benchmark average run ", v.avgTimeSec, " for ", v.count, " runs ", "for ", k 49 | 50 | if printTsv: 51 | echo "" 52 | echo "name", "\t", "avgTimeSec", "\t", "count" 53 | for k, v in benchRuns: 54 | echo k, "\t", v.avgTimeSec, "\t", v.count 55 | 56 | 57 | import std/math 58 | 59 | func floorLog2*(x: int): int = 60 | var k = -1 61 | var y = x 62 | while (y > 0): 63 | k += 1 64 | y = y shr 1 65 | return k 66 | 67 | func ceilingLog2*(x: int): int = 68 | if (x == 0): 69 | return -1 70 | else: 71 | return (floorLog2(x - 1) + 1) 72 | 73 | func checkPowerOfTwo*(x: int, what: string): int = 74 | let k = ceilingLog2(x) 75 | assert(x == 2 ^ k, ("`" & what & "` is expected to be a power of 2")) 76 | return x 77 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | # advanced settings 6 | 7 | # Prevents PR from being blocked with a reduction in coverage. 8 | # Note, if we want to re-enable this, a `threshold` value can be used 9 | # allow coverage to drop by x% while still posting a success status. 10 | # `informational`: https://docs.codecov.com/docs/commit-status#informational 11 | # `threshold`: https://docs.codecov.com/docs/commit-status#threshold 12 | informational: true 13 | patch: 14 | default: 15 | # advanced settings 16 | 17 | # Prevents PR from being blocked with a reduction in coverage. 18 | # Note, if we want to re-enable this, a `threshold` value can be used 19 | # allow coverage to drop by x% while still posting a success status. 20 | # `informational`: https://docs.codecov.com/docs/commit-status#informational 21 | # `threshold`: https://docs.codecov.com/docs/commit-status#threshold 22 | informational: true 23 | comment: false 24 | -------------------------------------------------------------------------------- /codex.nimble: -------------------------------------------------------------------------------- 1 | version = "0.1.0" 2 | author = "Codex Team" 3 | description = "p2p data durability engine" 4 | license = "MIT" 5 | binDir = "build" 6 | srcDir = "." 7 | installFiles = @["build.nims"] 8 | 9 | include "build.nims" 10 | -------------------------------------------------------------------------------- /codex/blockexchange.nim: -------------------------------------------------------------------------------- 1 | import ./blockexchange/[network, engine, peers] 2 | 3 | import ./blockexchange/protobuf/[blockexc, presence] 4 | 5 | export network, engine, blockexc, presence, peers 6 | -------------------------------------------------------------------------------- /codex/blockexchange/engine.nim: -------------------------------------------------------------------------------- 1 | import ./engine/discovery 2 | import ./engine/advertiser 3 | import ./engine/engine 4 | import ./engine/payments 5 | 6 | export discovery, advertiser, engine, payments 7 | -------------------------------------------------------------------------------- /codex/blockexchange/engine/payments.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2021 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | {.push raises: [].} 11 | 12 | import std/math 13 | import pkg/nitro 14 | import pkg/questionable/results 15 | import ../peers 16 | 17 | export nitro 18 | export results 19 | 20 | const ChainId* = 0.u256 # invalid chain id for now 21 | const Asset* = EthAddress.zero # invalid ERC20 asset address for now 22 | const AmountPerChannel = (10'u64 ^ 18).u256 # 1 asset, ERC20 default is 18 decimals 23 | 24 | func openLedgerChannel*( 25 | wallet: WalletRef, hub: EthAddress, asset: EthAddress 26 | ): ?!ChannelId = 27 | wallet.openLedgerChannel(hub, ChainId, asset, AmountPerChannel) 28 | 29 | func getOrOpenChannel(wallet: WalletRef, peer: BlockExcPeerCtx): ?!ChannelId = 30 | if channel =? peer.paymentChannel: 31 | success channel 32 | elif account =? peer.account: 33 | let channel = ?wallet.openLedgerChannel(account.address, Asset) 34 | peer.paymentChannel = channel.some 35 | success channel 36 | else: 37 | failure "no account set for peer" 38 | 39 | func pay*(wallet: WalletRef, peer: BlockExcPeerCtx, amount: UInt256): ?!SignedState = 40 | if account =? peer.account: 41 | let asset = Asset 42 | let receiver = account.address 43 | let channel = ?wallet.getOrOpenChannel(peer) 44 | wallet.pay(channel, asset, receiver, amount) 45 | else: 46 | failure "no account set for peer" 47 | -------------------------------------------------------------------------------- /codex/blockexchange/network.nim: -------------------------------------------------------------------------------- 1 | import ./network/network 2 | import ./network/networkpeer 3 | 4 | export network, networkpeer 5 | -------------------------------------------------------------------------------- /codex/blockexchange/peers.nim: -------------------------------------------------------------------------------- 1 | import ./peers/peerctxstore 2 | import ./peers/peercontext 3 | 4 | export peerctxstore, peercontext 5 | -------------------------------------------------------------------------------- /codex/blockexchange/peers/peercontext.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2021 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import std/sequtils 11 | import std/tables 12 | import std/sets 13 | 14 | import pkg/libp2p 15 | import pkg/chronos 16 | import pkg/nitro 17 | import pkg/questionable 18 | 19 | import ../protobuf/blockexc 20 | import ../protobuf/payments 21 | import ../protobuf/presence 22 | 23 | import ../../blocktype 24 | import ../../logutils 25 | 26 | export payments, nitro 27 | 28 | type BlockExcPeerCtx* = ref object of RootObj 29 | id*: PeerId 30 | blocks*: Table[BlockAddress, Presence] # remote peer have list including price 31 | peerWants*: seq[WantListEntry] # remote peers want lists 32 | exchanged*: int # times peer has exchanged with us 33 | lastExchange*: Moment # last time peer has exchanged with us 34 | account*: ?Account # ethereum account of this peer 35 | paymentChannel*: ?ChannelId # payment channel id 36 | 37 | proc peerHave*(self: BlockExcPeerCtx): seq[BlockAddress] = 38 | toSeq(self.blocks.keys) 39 | 40 | proc peerHaveCids*(self: BlockExcPeerCtx): HashSet[Cid] = 41 | self.blocks.keys.toSeq.mapIt(it.cidOrTreeCid).toHashSet 42 | 43 | proc peerWantsCids*(self: BlockExcPeerCtx): HashSet[Cid] = 44 | self.peerWants.mapIt(it.address.cidOrTreeCid).toHashSet 45 | 46 | proc contains*(self: BlockExcPeerCtx, address: BlockAddress): bool = 47 | address in self.blocks 48 | 49 | func setPresence*(self: BlockExcPeerCtx, presence: Presence) = 50 | self.blocks[presence.address] = presence 51 | 52 | func cleanPresence*(self: BlockExcPeerCtx, addresses: seq[BlockAddress]) = 53 | for a in addresses: 54 | self.blocks.del(a) 55 | 56 | func cleanPresence*(self: BlockExcPeerCtx, address: BlockAddress) = 57 | self.cleanPresence(@[address]) 58 | 59 | func price*(self: BlockExcPeerCtx, addresses: seq[BlockAddress]): UInt256 = 60 | var price = 0.u256 61 | for a in addresses: 62 | self.blocks.withValue(a, precense): 63 | price += precense[].price 64 | 65 | price 66 | -------------------------------------------------------------------------------- /codex/blockexchange/protobuf/blockexc.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2021 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import std/hashes 11 | import std/sequtils 12 | import pkg/stew/endians2 13 | 14 | import message 15 | 16 | import ../../blocktype 17 | 18 | export Message, protobufEncode, protobufDecode 19 | export Wantlist, WantType, WantListEntry 20 | export BlockDelivery, BlockPresenceType, BlockPresence 21 | export AccountMessage, StateChannelUpdate 22 | 23 | proc hash*(a: BlockAddress): Hash = 24 | if a.leaf: 25 | let data = a.treeCid.data.buffer & @(a.index.uint64.toBytesBE) 26 | hash(data) 27 | else: 28 | hash(a.cid.data.buffer) 29 | 30 | proc hash*(e: WantListEntry): Hash = 31 | hash(e.address) 32 | 33 | proc contains*(a: openArray[WantListEntry], b: BlockAddress): bool = 34 | ## Convenience method to check for peer precense 35 | ## 36 | 37 | a.anyIt(it.address == b) 38 | 39 | proc `==`*(a: WantListEntry, b: BlockAddress): bool = 40 | return a.address == b 41 | 42 | proc `<`*(a, b: WantListEntry): bool = 43 | a.priority < b.priority 44 | 45 | proc `==`*(a: BlockPresence, b: BlockAddress): bool = 46 | return a.address == b 47 | 48 | proc contains*(a: openArray[BlockPresence], b: BlockAddress): bool = 49 | ## Convenience method to check for peer precense 50 | ## 51 | 52 | a.anyIt(it.address == b) 53 | -------------------------------------------------------------------------------- /codex/blockexchange/protobuf/message.proto: -------------------------------------------------------------------------------- 1 | // Protocol of data exchange between Codex nodes. 2 | // Extended version of https://github.com/ipfs/specs/blob/main/BITSWAP.md 3 | 4 | syntax = "proto3"; 5 | 6 | package blockexc.message.pb; 7 | 8 | message Message { 9 | 10 | message Wantlist { 11 | enum WantType { 12 | wantBlock = 0; 13 | wantHave = 1; 14 | } 15 | 16 | message Entry { 17 | bytes block = 1; // the block cid 18 | int32 priority = 2; // the priority (normalized). default to 1 19 | bool cancel = 3; // whether this revokes an entry 20 | WantType wantType = 4; // Note: defaults to enum 0, ie Block 21 | bool sendDontHave = 5; // Note: defaults to false 22 | } 23 | 24 | repeated Entry entries = 1; // a list of wantlist entries 25 | bool full = 2; // whether this is the full wantlist. default to false 26 | } 27 | 28 | message Block { 29 | bytes prefix = 1; // CID prefix (cid version, multicodec and multihash prefix (type + length) 30 | bytes data = 2; 31 | } 32 | 33 | enum BlockPresenceType { 34 | presenceHave = 0; 35 | presenceDontHave = 1; 36 | } 37 | 38 | message BlockPresence { 39 | bytes cid = 1; 40 | BlockPresenceType type = 2; 41 | bytes price = 3; // Amount of assets to pay for the block (UInt256) 42 | } 43 | 44 | message AccountMessage { 45 | bytes address = 1; // Ethereum address to which payments should be made 46 | } 47 | 48 | message StateChannelUpdate { 49 | bytes update = 1; // Signed Nitro state, serialized as JSON 50 | } 51 | 52 | Wantlist wantlist = 1; 53 | repeated Block payload = 3; 54 | repeated BlockPresence blockPresences = 4; 55 | int32 pendingBytes = 5; 56 | AccountMessage account = 6; 57 | StateChannelUpdate payment = 7; 58 | } 59 | -------------------------------------------------------------------------------- /codex/blockexchange/protobuf/message.proto.license: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014-2018 Juan Batiz-Benet 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /codex/blockexchange/protobuf/payments.nim: -------------------------------------------------------------------------------- 1 | {.push raises: [].} 2 | 3 | import pkg/stew/byteutils 4 | import pkg/stint 5 | import pkg/nitro 6 | import pkg/questionable 7 | import ./blockexc 8 | 9 | export AccountMessage 10 | export StateChannelUpdate 11 | 12 | export stint 13 | export nitro 14 | 15 | type Account* = object 16 | address*: EthAddress 17 | 18 | func init*(_: type AccountMessage, account: Account): AccountMessage = 19 | AccountMessage(address: @(account.address.toArray)) 20 | 21 | func parse(_: type EthAddress, bytes: seq[byte]): ?EthAddress = 22 | var address: array[20, byte] 23 | if bytes.len != address.len: 24 | return EthAddress.none 25 | for i in 0 ..< address.len: 26 | address[i] = bytes[i] 27 | EthAddress(address).some 28 | 29 | func init*(_: type Account, message: AccountMessage): ?Account = 30 | without address =? EthAddress.parse(message.address): 31 | return none Account 32 | some Account(address: address) 33 | 34 | func init*(_: type StateChannelUpdate, state: SignedState): StateChannelUpdate = 35 | StateChannelUpdate(update: state.toJson.toBytes) 36 | 37 | proc init*(_: type SignedState, update: StateChannelUpdate): ?SignedState = 38 | SignedState.fromJson(string.fromBytes(update.update)) 39 | -------------------------------------------------------------------------------- /codex/blockexchange/protobuf/presence.nim: -------------------------------------------------------------------------------- 1 | {.push raises: [].} 2 | 3 | import libp2p 4 | import pkg/stint 5 | import pkg/questionable 6 | import pkg/questionable/results 7 | import ./blockexc 8 | 9 | import ../../blocktype 10 | 11 | export questionable 12 | export stint 13 | export BlockPresenceType 14 | 15 | type 16 | PresenceMessage* = blockexc.BlockPresence 17 | Presence* = object 18 | address*: BlockAddress 19 | have*: bool 20 | price*: UInt256 21 | 22 | func parse(_: type UInt256, bytes: seq[byte]): ?UInt256 = 23 | if bytes.len > 32: 24 | return UInt256.none 25 | UInt256.fromBytesBE(bytes).some 26 | 27 | func init*(_: type Presence, message: PresenceMessage): ?Presence = 28 | without price =? UInt256.parse(message.price): 29 | return none Presence 30 | 31 | some Presence( 32 | address: message.address, 33 | have: message.`type` == BlockPresenceType.Have, 34 | price: price, 35 | ) 36 | 37 | func init*(_: type PresenceMessage, presence: Presence): PresenceMessage = 38 | PresenceMessage( 39 | address: presence.address, 40 | `type`: if presence.have: BlockPresenceType.Have else: BlockPresenceType.DontHave, 41 | price: @(presence.price.toBytesBE), 42 | ) 43 | -------------------------------------------------------------------------------- /codex/clock.nim: -------------------------------------------------------------------------------- 1 | import pkg/chronos 2 | import pkg/stew/endians2 3 | import pkg/upraises 4 | import pkg/stint 5 | 6 | type 7 | Clock* = ref object of RootObj 8 | SecondsSince1970* = int64 9 | Timeout* = object of CatchableError 10 | 11 | method now*(clock: Clock): SecondsSince1970 {.base, gcsafe, upraises: [].} = 12 | raiseAssert "not implemented" 13 | 14 | method waitUntil*(clock: Clock, time: SecondsSince1970) {.base, async.} = 15 | raiseAssert "not implemented" 16 | 17 | method start*(clock: Clock) {.base, async.} = 18 | discard 19 | 20 | method stop*(clock: Clock) {.base, async.} = 21 | discard 22 | 23 | proc withTimeout*( 24 | future: Future[void], clock: Clock, expiry: SecondsSince1970 25 | ) {.async.} = 26 | let timeout = clock.waitUntil(expiry) 27 | try: 28 | await future or timeout 29 | finally: 30 | await timeout.cancelAndWait() 31 | if not future.completed: 32 | await future.cancelAndWait() 33 | raise newException(Timeout, "Timed out") 34 | 35 | proc toBytes*(i: SecondsSince1970): seq[byte] = 36 | let asUint = cast[uint64](i) 37 | @(asUint.toBytes) 38 | 39 | proc toSecondsSince1970*(bytes: seq[byte]): SecondsSince1970 = 40 | let asUint = uint64.fromBytes(bytes) 41 | cast[int64](asUint) 42 | 43 | proc toSecondsSince1970*(num: uint64): SecondsSince1970 = 44 | cast[int64](num) 45 | 46 | proc toSecondsSince1970*(bigint: UInt256): SecondsSince1970 = 47 | bigint.truncate(int64) 48 | -------------------------------------------------------------------------------- /codex/contracts.nim: -------------------------------------------------------------------------------- 1 | import contracts/requests 2 | import contracts/marketplace 3 | import contracts/market 4 | import contracts/interactions 5 | import contracts/provider 6 | 7 | export requests 8 | export marketplace 9 | export market 10 | export interactions 11 | export provider 12 | -------------------------------------------------------------------------------- /codex/contracts/deployment.nim: -------------------------------------------------------------------------------- 1 | import std/os 2 | import std/tables 3 | import pkg/ethers 4 | import pkg/questionable 5 | 6 | import ../conf 7 | import ../logutils 8 | import ./marketplace 9 | 10 | type Deployment* = ref object 11 | provider: Provider 12 | config: CodexConf 13 | 14 | const knownAddresses = { 15 | # Hardhat localhost network 16 | "31337": 17 | {"Marketplace": Address.init("0x322813Fd9A801c5507c9de605d63CEA4f2CE6c44")}.toTable, 18 | # Taiko Alpha-3 Testnet 19 | "167005": 20 | {"Marketplace": Address.init("0x948CF9291b77Bd7ad84781b9047129Addf1b894F")}.toTable, 21 | # Codex Testnet - May 30 2025 07:33:06 AM (+00:00 UTC) 22 | "789987": 23 | {"Marketplace": Address.init("0x7c7a749DE7156305E55775e7Ab3931abd6f7300E")}.toTable, 24 | }.toTable 25 | 26 | proc getKnownAddress(T: type, chainId: UInt256): ?Address = 27 | let id = chainId.toString(10) 28 | notice "Looking for well-known contract address with ChainID ", chainId = id 29 | 30 | if not (id in knownAddresses): 31 | return none Address 32 | 33 | return knownAddresses[id].getOrDefault($T, Address.none) 34 | 35 | proc new*(_: type Deployment, provider: Provider, config: CodexConf): Deployment = 36 | Deployment(provider: provider, config: config) 37 | 38 | proc address*(deployment: Deployment, contract: type): Future[?Address] {.async.} = 39 | when contract is Marketplace: 40 | if address =? deployment.config.marketplaceAddress: 41 | return some address 42 | 43 | let chainId = await deployment.provider.getChainId() 44 | return contract.getKnownAddress(chainId) 45 | -------------------------------------------------------------------------------- /codex/contracts/interactions.nim: -------------------------------------------------------------------------------- 1 | import ./interactions/interactions 2 | import ./interactions/hostinteractions 3 | import ./interactions/clientinteractions 4 | import ./interactions/validatorinteractions 5 | 6 | export interactions 7 | export hostinteractions 8 | export clientinteractions 9 | export validatorinteractions 10 | -------------------------------------------------------------------------------- /codex/contracts/interactions/clientinteractions.nim: -------------------------------------------------------------------------------- 1 | import pkg/ethers 2 | 3 | import ../../purchasing 4 | import ../../logutils 5 | import ../market 6 | import ../clock 7 | import ./interactions 8 | 9 | export purchasing 10 | export logutils 11 | 12 | type ClientInteractions* = ref object of ContractInteractions 13 | purchasing*: Purchasing 14 | 15 | proc new*( 16 | _: type ClientInteractions, clock: OnChainClock, purchasing: Purchasing 17 | ): ClientInteractions = 18 | ClientInteractions(clock: clock, purchasing: purchasing) 19 | 20 | proc start*(self: ClientInteractions) {.async.} = 21 | await procCall ContractInteractions(self).start() 22 | await self.purchasing.start() 23 | 24 | proc stop*(self: ClientInteractions) {.async.} = 25 | await self.purchasing.stop() 26 | await procCall ContractInteractions(self).stop() 27 | -------------------------------------------------------------------------------- /codex/contracts/interactions/hostinteractions.nim: -------------------------------------------------------------------------------- 1 | import pkg/chronos 2 | 3 | import ../../logutils 4 | import ../../sales 5 | import ./interactions 6 | 7 | export sales 8 | export logutils 9 | 10 | type HostInteractions* = ref object of ContractInteractions 11 | sales*: Sales 12 | 13 | proc new*(_: type HostInteractions, clock: Clock, sales: Sales): HostInteractions = 14 | ## Create a new HostInteractions instance 15 | ## 16 | HostInteractions(clock: clock, sales: sales) 17 | 18 | method start*(self: HostInteractions) {.async.} = 19 | await procCall ContractInteractions(self).start() 20 | await self.sales.start() 21 | 22 | method stop*(self: HostInteractions) {.async.} = 23 | await self.sales.stop() 24 | await procCall ContractInteractions(self).start() 25 | -------------------------------------------------------------------------------- /codex/contracts/interactions/interactions.nim: -------------------------------------------------------------------------------- 1 | import pkg/ethers 2 | import ../clock 3 | import ../marketplace 4 | import ../market 5 | 6 | export clock 7 | 8 | type ContractInteractions* = ref object of RootObj 9 | clock*: Clock 10 | 11 | method start*(self: ContractInteractions) {.async, base.} = 12 | discard 13 | 14 | method stop*(self: ContractInteractions) {.async, base.} = 15 | discard 16 | -------------------------------------------------------------------------------- /codex/contracts/interactions/validatorinteractions.nim: -------------------------------------------------------------------------------- 1 | import ./interactions 2 | import ../../validation 3 | 4 | export validation 5 | 6 | type ValidatorInteractions* = ref object of ContractInteractions 7 | validation: Validation 8 | 9 | proc new*( 10 | _: type ValidatorInteractions, clock: OnChainClock, validation: Validation 11 | ): ValidatorInteractions = 12 | ValidatorInteractions(clock: clock, validation: validation) 13 | 14 | proc start*(self: ValidatorInteractions) {.async.} = 15 | await procCall ContractInteractions(self).start() 16 | await self.validation.start() 17 | 18 | proc stop*(self: ValidatorInteractions) {.async.} = 19 | await self.validation.stop() 20 | await procCall ContractInteractions(self).stop() 21 | -------------------------------------------------------------------------------- /codex/contracts/proofs.nim: -------------------------------------------------------------------------------- 1 | import pkg/stint 2 | import pkg/contractabi 3 | import pkg/ethers/contracts/fields 4 | 5 | type 6 | Groth16Proof* = object 7 | a*: G1Point 8 | b*: G2Point 9 | c*: G1Point 10 | 11 | G1Point* = object 12 | x*: UInt256 13 | y*: UInt256 14 | 15 | # A field element F_{p^2} encoded as `real + i * imag` 16 | Fp2Element* = object 17 | real*: UInt256 18 | imag*: UInt256 19 | 20 | G2Point* = object 21 | x*: Fp2Element 22 | y*: Fp2Element 23 | 24 | func solidityType*(_: type G1Point): string = 25 | solidityType(G1Point.fieldTypes) 26 | 27 | func solidityType*(_: type Fp2Element): string = 28 | solidityType(Fp2Element.fieldTypes) 29 | 30 | func solidityType*(_: type G2Point): string = 31 | solidityType(G2Point.fieldTypes) 32 | 33 | func solidityType*(_: type Groth16Proof): string = 34 | solidityType(Groth16Proof.fieldTypes) 35 | 36 | func encode*(encoder: var AbiEncoder, point: G1Point) = 37 | encoder.write(point.fieldValues) 38 | 39 | func encode*(encoder: var AbiEncoder, element: Fp2Element) = 40 | encoder.write(element.fieldValues) 41 | 42 | func encode*(encoder: var AbiEncoder, point: G2Point) = 43 | encoder.write(point.fieldValues) 44 | 45 | func encode*(encoder: var AbiEncoder, proof: Groth16Proof) = 46 | encoder.write(proof.fieldValues) 47 | -------------------------------------------------------------------------------- /codex/erasure.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2022 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import ./erasure/erasure 11 | import ./erasure/backends/leopard 12 | 13 | export erasure 14 | 15 | func leoEncoderProvider*( 16 | size, buffers, parity: int 17 | ): EncoderBackend {.raises: [Defect].} = 18 | ## create new Leo Encoder 19 | LeoEncoderBackend.new(size, buffers, parity) 20 | 21 | func leoDecoderProvider*( 22 | size, buffers, parity: int 23 | ): DecoderBackend {.raises: [Defect].} = 24 | ## create new Leo Decoder 25 | LeoDecoderBackend.new(size, buffers, parity) 26 | -------------------------------------------------------------------------------- /codex/erasure/backend.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2022 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import pkg/upraises 11 | 12 | push: 13 | {.upraises: [].} 14 | 15 | import ../stores 16 | 17 | type 18 | ErasureBackend* = ref object of RootObj 19 | blockSize*: int # block size in bytes 20 | buffers*: int # number of original pieces 21 | parity*: int # number of redundancy pieces 22 | 23 | EncoderBackend* = ref object of ErasureBackend 24 | DecoderBackend* = ref object of ErasureBackend 25 | 26 | method release*(self: ErasureBackend) {.base, gcsafe.} = 27 | ## release the backend 28 | ## 29 | raiseAssert("not implemented!") 30 | 31 | method encode*( 32 | self: EncoderBackend, 33 | buffers, parity: ptr UncheckedArray[ptr UncheckedArray[byte]], 34 | dataLen, parityLen: int, 35 | ): Result[void, cstring] {.base, gcsafe.} = 36 | ## encode buffers using a backend 37 | ## 38 | raiseAssert("not implemented!") 39 | 40 | method decode*( 41 | self: DecoderBackend, 42 | buffers, parity, recovered: ptr UncheckedArray[ptr UncheckedArray[byte]], 43 | dataLen, parityLen, recoveredLen: int, 44 | ): Result[void, cstring] {.base, gcsafe.} = 45 | ## decode buffers using a backend 46 | ## 47 | raiseAssert("not implemented!") 48 | -------------------------------------------------------------------------------- /codex/manifest.nim: -------------------------------------------------------------------------------- 1 | import ./manifest/coders 2 | import ./manifest/manifest 3 | 4 | export manifest, coders 5 | -------------------------------------------------------------------------------- /codex/merkletree.nim: -------------------------------------------------------------------------------- 1 | import ./merkletree/merkletree 2 | import ./merkletree/codex 3 | import ./merkletree/poseidon2 4 | 5 | export codex, poseidon2, merkletree 6 | 7 | type 8 | SomeMerkleTree* = ByteTree | CodexTree | Poseidon2Tree 9 | SomeMerkleProof* = ByteProof | CodexProof | Poseidon2Proof 10 | SomeMerkleHash* = ByteHash | Poseidon2Hash 11 | -------------------------------------------------------------------------------- /codex/merkletree/codex.nim: -------------------------------------------------------------------------------- 1 | import ./codex/codex 2 | import ./codex/coders 3 | 4 | export codex, coders 5 | -------------------------------------------------------------------------------- /codex/namespaces.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2022 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | const 11 | # Namespaces 12 | CodexMetaNamespace* = "meta" # meta info stored here 13 | CodexRepoNamespace* = "repo" # repository namespace, blocks and manifests are subkeys 14 | CodexBlockTotalNamespace* = CodexMetaNamespace & "/total" 15 | # number of blocks in the repo 16 | CodexBlocksNamespace* = CodexRepoNamespace & "/blocks" # blocks namespace 17 | CodexManifestNamespace* = CodexRepoNamespace & "/manifests" # manifest namespace 18 | CodexBlocksTtlNamespace* = # Cid TTL 19 | CodexMetaNamespace & "/ttl" 20 | CodexBlockProofNamespace* = # Cid and Proof 21 | CodexMetaNamespace & "/proof" 22 | CodexDhtNamespace* = "dht" # Dht namespace 23 | CodexDhtProvidersNamespace* = # Dht providers namespace 24 | CodexDhtNamespace & "/providers" 25 | CodexQuotaNamespace* = CodexMetaNamespace & "/quota" # quota's namespace 26 | -------------------------------------------------------------------------------- /codex/periods.nim: -------------------------------------------------------------------------------- 1 | import pkg/stint 2 | 3 | type 4 | Periodicity* = object 5 | seconds*: uint64 6 | 7 | Period* = uint64 8 | Timestamp* = uint64 9 | 10 | func periodOf*(periodicity: Periodicity, timestamp: Timestamp): Period = 11 | timestamp div periodicity.seconds 12 | 13 | func periodStart*(periodicity: Periodicity, period: Period): Timestamp = 14 | period * periodicity.seconds 15 | 16 | func periodEnd*(periodicity: Periodicity, period: Period): Timestamp = 17 | periodicity.periodStart(period + 1) 18 | -------------------------------------------------------------------------------- /codex/purchasing.nim: -------------------------------------------------------------------------------- 1 | import std/tables 2 | import pkg/stint 3 | import pkg/chronos 4 | import pkg/questionable 5 | import pkg/nimcrypto 6 | import ./market 7 | import ./clock 8 | import ./purchasing/purchase 9 | 10 | export questionable 11 | export chronos 12 | export market 13 | export purchase 14 | 15 | type 16 | Purchasing* = ref object 17 | market*: Market 18 | clock: Clock 19 | purchases: Table[PurchaseId, Purchase] 20 | proofProbability*: UInt256 21 | 22 | PurchaseTimeout* = Timeout 23 | 24 | const DefaultProofProbability = 100.u256 25 | 26 | proc new*(_: type Purchasing, market: Market, clock: Clock): Purchasing = 27 | Purchasing(market: market, clock: clock, proofProbability: DefaultProofProbability) 28 | 29 | proc load*(purchasing: Purchasing) {.async.} = 30 | let market = purchasing.market 31 | let requestIds = await market.myRequests() 32 | for requestId in requestIds: 33 | let purchase = Purchase.new(requestId, purchasing.market, purchasing.clock) 34 | purchase.load() 35 | purchasing.purchases[purchase.id] = purchase 36 | 37 | proc start*(purchasing: Purchasing) {.async.} = 38 | await purchasing.load() 39 | 40 | proc stop*(purchasing: Purchasing) {.async.} = 41 | discard 42 | 43 | proc populate*( 44 | purchasing: Purchasing, request: StorageRequest 45 | ): Future[StorageRequest] {.async.} = 46 | result = request 47 | if result.ask.proofProbability == 0.u256: 48 | result.ask.proofProbability = purchasing.proofProbability 49 | if result.nonce == Nonce.default: 50 | var id = result.nonce.toArray 51 | doAssert randomBytes(id) == 32 52 | result.nonce = Nonce(id) 53 | result.client = await purchasing.market.getSigner() 54 | 55 | proc purchase*( 56 | purchasing: Purchasing, request: StorageRequest 57 | ): Future[Purchase] {.async.} = 58 | let request = await purchasing.populate(request) 59 | let purchase = Purchase.new(request, purchasing.market, purchasing.clock) 60 | purchase.start() 61 | purchasing.purchases[purchase.id] = purchase 62 | return purchase 63 | 64 | func getPurchase*(purchasing: Purchasing, id: PurchaseId): ?Purchase = 65 | if purchasing.purchases.hasKey(id): 66 | some purchasing.purchases[id] 67 | else: 68 | none Purchase 69 | 70 | func getPurchaseIds*(purchasing: Purchasing): seq[PurchaseId] = 71 | var pIds: seq[PurchaseId] = @[] 72 | for key in purchasing.purchases.keys: 73 | pIds.add(key) 74 | return pIds 75 | -------------------------------------------------------------------------------- /codex/purchasing/purchaseid.nim: -------------------------------------------------------------------------------- 1 | import std/hashes 2 | import ../logutils 3 | 4 | type PurchaseId* = distinct array[32, byte] 5 | 6 | logutils.formatIt(LogFormat.textLines, PurchaseId): 7 | it.short0xHexLog 8 | logutils.formatIt(LogFormat.json, PurchaseId): 9 | it.to0xHexLog 10 | 11 | proc hash*(x: PurchaseId): Hash {.borrow.} 12 | proc `==`*(x, y: PurchaseId): bool {.borrow.} 13 | proc toHex*(x: PurchaseId): string = 14 | array[32, byte](x).toHex 15 | -------------------------------------------------------------------------------- /codex/purchasing/statemachine.nim: -------------------------------------------------------------------------------- 1 | import ../utils/asyncstatemachine 2 | import ../market 3 | import ../clock 4 | import ../errors 5 | 6 | export market 7 | export clock 8 | export asyncstatemachine 9 | 10 | type 11 | Purchase* = ref object of Machine 12 | future*: Future[void] 13 | market*: Market 14 | clock*: Clock 15 | requestId*: RequestId 16 | request*: ?StorageRequest 17 | 18 | PurchaseState* = ref object of State 19 | PurchaseError* = object of CodexError 20 | -------------------------------------------------------------------------------- /codex/purchasing/states/cancelled.nim: -------------------------------------------------------------------------------- 1 | import pkg/metrics 2 | 3 | import ../../logutils 4 | import ../../utils/exceptions 5 | import ../statemachine 6 | import ./error 7 | 8 | declareCounter(codex_purchases_cancelled, "codex purchases cancelled") 9 | 10 | logScope: 11 | topics = "marketplace purchases cancelled" 12 | 13 | type PurchaseCancelled* = ref object of PurchaseState 14 | 15 | method `$`*(state: PurchaseCancelled): string = 16 | "cancelled" 17 | 18 | method run*( 19 | state: PurchaseCancelled, machine: Machine 20 | ): Future[?State] {.async: (raises: []).} = 21 | codex_purchases_cancelled.inc() 22 | let purchase = Purchase(machine) 23 | 24 | try: 25 | warn "Request cancelled, withdrawing remaining funds", 26 | requestId = purchase.requestId 27 | await purchase.market.withdrawFunds(purchase.requestId) 28 | 29 | let error = newException(Timeout, "Purchase cancelled due to timeout") 30 | purchase.future.fail(error) 31 | except CancelledError as e: 32 | trace "PurchaseCancelled.run was cancelled", error = e.msgDetail 33 | except CatchableError as e: 34 | error "Error during PurchaseCancelled.run", error = e.msgDetail 35 | return some State(PurchaseErrored(error: e)) 36 | -------------------------------------------------------------------------------- /codex/purchasing/states/error.nim: -------------------------------------------------------------------------------- 1 | import pkg/metrics 2 | import ../statemachine 3 | import ../../utils/exceptions 4 | import ../../logutils 5 | 6 | declareCounter(codex_purchases_error, "codex purchases error") 7 | 8 | logScope: 9 | topics = "marketplace purchases errored" 10 | 11 | type PurchaseErrored* = ref object of PurchaseState 12 | error*: ref CatchableError 13 | 14 | method `$`*(state: PurchaseErrored): string = 15 | "errored" 16 | 17 | method run*( 18 | state: PurchaseErrored, machine: Machine 19 | ): Future[?State] {.async: (raises: []).} = 20 | codex_purchases_error.inc() 21 | let purchase = Purchase(machine) 22 | 23 | error "Purchasing error", 24 | error = state.error.msgDetail, requestId = purchase.requestId 25 | 26 | purchase.future.fail(state.error) 27 | -------------------------------------------------------------------------------- /codex/purchasing/states/failed.nim: -------------------------------------------------------------------------------- 1 | import pkg/metrics 2 | import ../statemachine 3 | import ../../logutils 4 | import ../../utils/exceptions 5 | import ./error 6 | 7 | declareCounter(codex_purchases_failed, "codex purchases failed") 8 | 9 | type PurchaseFailed* = ref object of PurchaseState 10 | 11 | method `$`*(state: PurchaseFailed): string = 12 | "failed" 13 | 14 | method run*( 15 | state: PurchaseFailed, machine: Machine 16 | ): Future[?State] {.async: (raises: []).} = 17 | codex_purchases_failed.inc() 18 | let purchase = Purchase(machine) 19 | 20 | try: 21 | warn "Request failed, withdrawing remaining funds", requestId = purchase.requestId 22 | await purchase.market.withdrawFunds(purchase.requestId) 23 | except CancelledError as e: 24 | trace "PurchaseFailed.run was cancelled", error = e.msgDetail 25 | except CatchableError as e: 26 | error "Error during PurchaseFailed.run", error = e.msgDetail 27 | return some State(PurchaseErrored(error: e)) 28 | 29 | let error = newException(PurchaseError, "Purchase failed") 30 | return some State(PurchaseErrored(error: error)) 31 | -------------------------------------------------------------------------------- /codex/purchasing/states/finished.nim: -------------------------------------------------------------------------------- 1 | import pkg/metrics 2 | 3 | import ../statemachine 4 | import ../../utils/exceptions 5 | import ../../logutils 6 | import ./error 7 | 8 | declareCounter(codex_purchases_finished, "codex purchases finished") 9 | 10 | logScope: 11 | topics = "marketplace purchases finished" 12 | 13 | type PurchaseFinished* = ref object of PurchaseState 14 | 15 | method `$`*(state: PurchaseFinished): string = 16 | "finished" 17 | 18 | method run*( 19 | state: PurchaseFinished, machine: Machine 20 | ): Future[?State] {.async: (raises: []).} = 21 | codex_purchases_finished.inc() 22 | let purchase = Purchase(machine) 23 | try: 24 | info "Purchase finished, withdrawing remaining funds", 25 | requestId = purchase.requestId 26 | await purchase.market.withdrawFunds(purchase.requestId) 27 | 28 | purchase.future.complete() 29 | except CancelledError as e: 30 | trace "PurchaseFinished.run was cancelled", error = e.msgDetail 31 | except CatchableError as e: 32 | error "Error during PurchaseFinished.run", error = e.msgDetail 33 | return some State(PurchaseErrored(error: e)) 34 | -------------------------------------------------------------------------------- /codex/purchasing/states/pending.nim: -------------------------------------------------------------------------------- 1 | import pkg/metrics 2 | import ../../logutils 3 | import ../../utils/exceptions 4 | import ../statemachine 5 | import ./submitted 6 | import ./error 7 | 8 | declareCounter(codex_purchases_pending, "codex purchases pending") 9 | 10 | type PurchasePending* = ref object of PurchaseState 11 | 12 | method `$`*(state: PurchasePending): string = 13 | "pending" 14 | 15 | method run*( 16 | state: PurchasePending, machine: Machine 17 | ): Future[?State] {.async: (raises: []).} = 18 | codex_purchases_pending.inc() 19 | let purchase = Purchase(machine) 20 | try: 21 | let request = !purchase.request 22 | await purchase.market.requestStorage(request) 23 | return some State(PurchaseSubmitted()) 24 | except CancelledError as e: 25 | trace "PurchasePending.run was cancelled", error = e.msgDetail 26 | except CatchableError as e: 27 | error "Error during PurchasePending.run", error = e.msgDetail 28 | return some State(PurchaseErrored(error: e)) 29 | -------------------------------------------------------------------------------- /codex/purchasing/states/started.nim: -------------------------------------------------------------------------------- 1 | import pkg/metrics 2 | 3 | import ../../logutils 4 | import ../../utils/exceptions 5 | import ../statemachine 6 | import ./finished 7 | import ./failed 8 | import ./error 9 | 10 | declareCounter(codex_purchases_started, "codex purchases started") 11 | 12 | logScope: 13 | topics = "marketplace purchases started" 14 | 15 | type PurchaseStarted* = ref object of PurchaseState 16 | 17 | method `$`*(state: PurchaseStarted): string = 18 | "started" 19 | 20 | method run*( 21 | state: PurchaseStarted, machine: Machine 22 | ): Future[?State] {.async: (raises: []).} = 23 | codex_purchases_started.inc() 24 | let purchase = Purchase(machine) 25 | 26 | let clock = purchase.clock 27 | let market = purchase.market 28 | info "All required slots filled, purchase started", requestId = purchase.requestId 29 | 30 | let failed = newFuture[void]() 31 | proc callback(_: RequestId) = 32 | failed.complete() 33 | 34 | var ended: Future[void] 35 | try: 36 | let subscription = await market.subscribeRequestFailed(purchase.requestId, callback) 37 | 38 | # Ensure that we're past the request end by waiting an additional second 39 | ended = clock.waitUntil((await market.getRequestEnd(purchase.requestId)) + 1) 40 | let fut = await one(ended, failed) 41 | await subscription.unsubscribe() 42 | if fut.id == failed.id: 43 | ended.cancelSoon() 44 | return some State(PurchaseFailed()) 45 | else: 46 | failed.cancelSoon() 47 | return some State(PurchaseFinished()) 48 | except CancelledError as e: 49 | ended.cancelSoon() 50 | failed.cancelSoon() 51 | trace "PurchaseStarted.run was cancelled", error = e.msgDetail 52 | except CatchableError as e: 53 | error "Error during PurchaseStarted.run", error = e.msgDetail 54 | return some State(PurchaseErrored(error: e)) 55 | -------------------------------------------------------------------------------- /codex/purchasing/states/submitted.nim: -------------------------------------------------------------------------------- 1 | import pkg/metrics 2 | 3 | import ../../logutils 4 | import ../../utils/exceptions 5 | import ../statemachine 6 | import ./started 7 | import ./cancelled 8 | import ./error 9 | 10 | logScope: 11 | topics = "marketplace purchases submitted" 12 | 13 | declareCounter(codex_purchases_submitted, "codex purchases submitted") 14 | 15 | type PurchaseSubmitted* = ref object of PurchaseState 16 | 17 | method `$`*(state: PurchaseSubmitted): string = 18 | "submitted" 19 | 20 | method run*( 21 | state: PurchaseSubmitted, machine: Machine 22 | ): Future[?State] {.async: (raises: []).} = 23 | codex_purchases_submitted.inc() 24 | let purchase = Purchase(machine) 25 | let request = !purchase.request 26 | let market = purchase.market 27 | let clock = purchase.clock 28 | 29 | info "Request submitted, waiting for slots to be filled", 30 | requestId = purchase.requestId 31 | 32 | proc wait() {.async.} = 33 | let done = newAsyncEvent() 34 | proc callback(_: RequestId) = 35 | done.fire() 36 | 37 | let subscription = await market.subscribeFulfillment(request.id, callback) 38 | await done.wait() 39 | await subscription.unsubscribe() 40 | 41 | proc withTimeout(future: Future[void]) {.async.} = 42 | let expiry = (await market.requestExpiresAt(request.id)) + 1 43 | trace "waiting for request fulfillment or expiry", expiry 44 | await future.withTimeout(clock, expiry) 45 | 46 | try: 47 | await wait().withTimeout() 48 | except Timeout: 49 | return some State(PurchaseCancelled()) 50 | except CancelledError as e: 51 | trace "PurchaseSubmitted.run was cancelled", error = e.msgDetail 52 | except CatchableError as e: 53 | error "Error during PurchaseSubmitted.run", error = e.msgDetail 54 | return some State(PurchaseErrored(error: e)) 55 | 56 | return some State(PurchaseStarted()) 57 | -------------------------------------------------------------------------------- /codex/purchasing/states/unknown.nim: -------------------------------------------------------------------------------- 1 | import pkg/metrics 2 | import ../../utils/exceptions 3 | import ../../logutils 4 | import ../statemachine 5 | import ./submitted 6 | import ./started 7 | import ./cancelled 8 | import ./finished 9 | import ./failed 10 | import ./error 11 | 12 | declareCounter(codex_purchases_unknown, "codex purchases unknown") 13 | 14 | type PurchaseUnknown* = ref object of PurchaseState 15 | 16 | method `$`*(state: PurchaseUnknown): string = 17 | "unknown" 18 | 19 | method run*( 20 | state: PurchaseUnknown, machine: Machine 21 | ): Future[?State] {.async: (raises: []).} = 22 | try: 23 | codex_purchases_unknown.inc() 24 | let purchase = Purchase(machine) 25 | if (request =? await purchase.market.getRequest(purchase.requestId)) and 26 | (requestState =? await purchase.market.requestState(purchase.requestId)): 27 | purchase.request = some request 28 | 29 | case requestState 30 | of RequestState.New: 31 | return some State(PurchaseSubmitted()) 32 | of RequestState.Started: 33 | return some State(PurchaseStarted()) 34 | of RequestState.Cancelled: 35 | return some State(PurchaseCancelled()) 36 | of RequestState.Finished: 37 | return some State(PurchaseFinished()) 38 | of RequestState.Failed: 39 | return some State(PurchaseFailed()) 40 | except CancelledError as e: 41 | trace "PurchaseUnknown.run was cancelled", error = e.msgDetail 42 | except CatchableError as e: 43 | error "Error during PurchaseUnknown.run", error = e.msgDetail 44 | return some State(PurchaseErrored(error: e)) 45 | -------------------------------------------------------------------------------- /codex/rng.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2021 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import pkg/upraises 11 | 12 | push: 13 | {.upraises: [].} 14 | 15 | import pkg/libp2p/crypto/crypto 16 | import pkg/bearssl/rand 17 | 18 | type 19 | RngSampleError = object of CatchableError 20 | Rng* = ref HmacDrbgContext 21 | 22 | var rng {.threadvar.}: Rng 23 | 24 | proc instance*(t: type Rng): Rng = 25 | if rng.isNil: 26 | rng = newRng() 27 | rng 28 | 29 | # Random helpers: similar as in stdlib, but with HmacDrbgContext rng 30 | # TODO: Move these somewhere else? 31 | const randMax = 18_446_744_073_709_551_615'u64 32 | 33 | proc rand*(rng: Rng, max: Natural): int = 34 | if max == 0: 35 | return 0 36 | 37 | while true: 38 | let x = rng[].generate(uint64) 39 | if x < randMax - (randMax mod (uint64(max) + 1'u64)): # against modulo bias 40 | return int(x mod (uint64(max) + 1'u64)) 41 | 42 | proc sample*[T](rng: Rng, a: openArray[T]): T = 43 | result = a[rng.rand(a.high)] 44 | 45 | proc sample*[T]( 46 | rng: Rng, sample, exclude: openArray[T] 47 | ): T {.raises: [Defect, RngSampleError].} = 48 | if sample == exclude: 49 | raise newException(RngSampleError, "Sample and exclude arrays are the same!") 50 | 51 | while true: 52 | result = rng.sample(sample) 53 | if exclude.find(result) != -1: 54 | continue 55 | 56 | break 57 | 58 | proc sample*[T]( 59 | rng: Rng, sample: openArray[T], limit: int 60 | ): seq[T] {.raises: [Defect, RngSampleError].} = 61 | if limit > sample.len: 62 | raise newException(RngSampleError, "Limit cannot be larger than sample!") 63 | 64 | for _ in 0 ..< min(sample.len, limit): 65 | result.add(rng.sample(sample, result)) 66 | 67 | proc shuffle*[T](rng: Rng, a: var openArray[T]) = 68 | for i in countdown(a.high, 1): 69 | let j = rng.rand(i) 70 | swap(a[i], a[j]) 71 | -------------------------------------------------------------------------------- /codex/sales/salescontext.nim: -------------------------------------------------------------------------------- 1 | import pkg/questionable 2 | import pkg/questionable/results 3 | import pkg/upraises 4 | import pkg/libp2p/cid 5 | 6 | import ../market 7 | import ../clock 8 | import ./slotqueue 9 | import ./reservations 10 | import ../blocktype as bt 11 | 12 | type 13 | SalesContext* = ref object 14 | market*: Market 15 | clock*: Clock 16 | # Sales-level callbacks. Closure will be overwritten each time a slot is 17 | # processed. 18 | onStore*: ?OnStore 19 | onClear*: ?OnClear 20 | onSale*: ?OnSale 21 | onProve*: ?OnProve 22 | onExpiryUpdate*: ?OnExpiryUpdate 23 | reservations*: Reservations 24 | slotQueue*: SlotQueue 25 | simulateProofFailures*: int 26 | 27 | BlocksCb* = proc(blocks: seq[bt.Block]): Future[?!void] {. 28 | gcsafe, async: (raises: [CancelledError]) 29 | .} 30 | OnStore* = proc( 31 | request: StorageRequest, slot: uint64, blocksCb: BlocksCb, isRepairing: bool 32 | ): Future[?!void] {.gcsafe, async: (raises: [CancelledError]).} 33 | OnProve* = proc(slot: Slot, challenge: ProofChallenge): Future[?!Groth16Proof] {. 34 | gcsafe, async: (raises: [CancelledError]) 35 | .} 36 | OnExpiryUpdate* = proc(rootCid: Cid, expiry: SecondsSince1970): Future[?!void] {. 37 | gcsafe, async: (raises: [CancelledError]) 38 | .} 39 | OnClear* = proc(request: StorageRequest, slotIndex: uint64) {.gcsafe, raises: [].} 40 | OnSale* = proc(request: StorageRequest, slotIndex: uint64) {.gcsafe, raises: [].} 41 | -------------------------------------------------------------------------------- /codex/sales/salesdata.nim: -------------------------------------------------------------------------------- 1 | import pkg/chronos 2 | import ../contracts/requests 3 | import ../market 4 | import ./reservations 5 | 6 | type SalesData* = ref object 7 | requestId*: RequestId 8 | ask*: StorageAsk 9 | request*: ?StorageRequest 10 | slotIndex*: uint64 11 | cancelled*: Future[void] 12 | reservation*: ?Reservation 13 | -------------------------------------------------------------------------------- /codex/sales/statemachine.nim: -------------------------------------------------------------------------------- 1 | import pkg/questionable 2 | import pkg/upraises 3 | import ../errors 4 | import ../utils/asyncstatemachine 5 | import ../market 6 | import ../clock 7 | import ../contracts/requests 8 | 9 | export market 10 | export clock 11 | export asyncstatemachine 12 | 13 | type 14 | SaleState* = ref object of State 15 | SaleError* = object of CodexError 16 | 17 | method onCancelled*( 18 | state: SaleState, request: StorageRequest 19 | ): ?State {.base, upraises: [].} = 20 | discard 21 | 22 | method onFailed*( 23 | state: SaleState, request: StorageRequest 24 | ): ?State {.base, upraises: [].} = 25 | discard 26 | 27 | method onSlotFilled*( 28 | state: SaleState, requestId: RequestId, slotIndex: uint64 29 | ): ?State {.base, upraises: [].} = 30 | discard 31 | 32 | proc cancelledEvent*(request: StorageRequest): Event = 33 | return proc(state: State): ?State = 34 | SaleState(state).onCancelled(request) 35 | 36 | proc failedEvent*(request: StorageRequest): Event = 37 | return proc(state: State): ?State = 38 | SaleState(state).onFailed(request) 39 | 40 | proc slotFilledEvent*(requestId: RequestId, slotIndex: uint64): Event = 41 | return proc(state: State): ?State = 42 | SaleState(state).onSlotFilled(requestId, slotIndex) 43 | -------------------------------------------------------------------------------- /codex/sales/states/cancelled.nim: -------------------------------------------------------------------------------- 1 | import ../../logutils 2 | import ../../utils/exceptions 3 | import ../salesagent 4 | import ../statemachine 5 | import ./errored 6 | 7 | logScope: 8 | topics = "marketplace sales cancelled" 9 | 10 | type SaleCancelled* = ref object of SaleState 11 | 12 | method `$`*(state: SaleCancelled): string = 13 | "SaleCancelled" 14 | 15 | proc slotIsFilledByMe( 16 | market: Market, requestId: RequestId, slotIndex: uint64 17 | ): Future[bool] {.async: (raises: [CancelledError, MarketError]).} = 18 | let host = await market.getHost(requestId, slotIndex) 19 | let me = await market.getSigner() 20 | 21 | return host == me.some 22 | 23 | method run*( 24 | state: SaleCancelled, machine: Machine 25 | ): Future[?State] {.async: (raises: []).} = 26 | let agent = SalesAgent(machine) 27 | let data = agent.data 28 | let market = agent.context.market 29 | 30 | without request =? data.request: 31 | raiseAssert "no sale request" 32 | 33 | try: 34 | var returnedCollateral = UInt256.none 35 | 36 | if await slotIsFilledByMe(market, data.requestId, data.slotIndex): 37 | debug "Collecting collateral and partial payout", 38 | requestId = data.requestId, slotIndex = data.slotIndex 39 | 40 | let slot = Slot(request: request, slotIndex: data.slotIndex) 41 | let currentCollateral = await market.currentCollateral(slot.id) 42 | 43 | try: 44 | await market.freeSlot(slot.id) 45 | except SlotStateMismatchError as e: 46 | warn "Failed to free slot because slot is already free", error = e.msg 47 | 48 | returnedCollateral = currentCollateral.some 49 | 50 | if onClear =? agent.context.onClear and request =? data.request: 51 | onClear(request, data.slotIndex) 52 | 53 | if onCleanUp =? agent.onCleanUp: 54 | await onCleanUp(reprocessSlot = false, returnedCollateral = returnedCollateral) 55 | 56 | warn "Sale cancelled due to timeout", 57 | requestId = data.requestId, slotIndex = data.slotIndex 58 | except CancelledError as e: 59 | trace "SaleCancelled.run was cancelled", error = e.msgDetail 60 | except CatchableError as e: 61 | error "Error during SaleCancelled.run", error = e.msgDetail 62 | return some State(SaleErrored(error: e)) 63 | -------------------------------------------------------------------------------- /codex/sales/states/errored.nim: -------------------------------------------------------------------------------- 1 | import pkg/questionable 2 | import pkg/questionable/results 3 | import pkg/upraises 4 | 5 | import ../statemachine 6 | import ../salesagent 7 | import ../../logutils 8 | import ../../utils/exceptions 9 | 10 | logScope: 11 | topics = "marketplace sales errored" 12 | 13 | type SaleErrored* = ref object of SaleState 14 | error*: ref CatchableError 15 | reprocessSlot*: bool 16 | 17 | method `$`*(state: SaleErrored): string = 18 | "SaleErrored" 19 | 20 | method run*( 21 | state: SaleErrored, machine: Machine 22 | ): Future[?State] {.async: (raises: []).} = 23 | let agent = SalesAgent(machine) 24 | let data = agent.data 25 | let context = agent.context 26 | 27 | error "Sale error", 28 | error = state.error.msgDetail, 29 | requestId = data.requestId, 30 | slotIndex = data.slotIndex 31 | 32 | try: 33 | if onClear =? context.onClear and request =? data.request: 34 | onClear(request, data.slotIndex) 35 | 36 | if onCleanUp =? agent.onCleanUp: 37 | await onCleanUp(reprocessSlot = state.reprocessSlot) 38 | except CancelledError as e: 39 | trace "SaleErrored.run was cancelled", error = e.msgDetail 40 | except CatchableError as e: 41 | error "Error during SaleErrored.run", error = e.msgDetail 42 | -------------------------------------------------------------------------------- /codex/sales/states/failed.nim: -------------------------------------------------------------------------------- 1 | import ../../logutils 2 | import ../../utils/exceptions 3 | import ../../utils/exceptions 4 | import ../salesagent 5 | import ../statemachine 6 | import ./errored 7 | 8 | logScope: 9 | topics = "marketplace sales failed" 10 | 11 | type 12 | SaleFailed* = ref object of SaleState 13 | SaleFailedError* = object of SaleError 14 | 15 | method `$`*(state: SaleFailed): string = 16 | "SaleFailed" 17 | 18 | method run*( 19 | state: SaleFailed, machine: Machine 20 | ): Future[?State] {.async: (raises: []).} = 21 | let data = SalesAgent(machine).data 22 | let market = SalesAgent(machine).context.market 23 | 24 | without request =? data.request: 25 | raiseAssert "no sale request" 26 | 27 | try: 28 | let slot = Slot(request: request, slotIndex: data.slotIndex) 29 | debug "Removing slot from mySlots", 30 | requestId = data.requestId, slotIndex = data.slotIndex 31 | 32 | await market.freeSlot(slot.id) 33 | 34 | let error = newException(SaleFailedError, "Sale failed") 35 | return some State(SaleErrored(error: error)) 36 | except CancelledError as e: 37 | trace "SaleFailed.run was cancelled", error = e.msgDetail 38 | except CatchableError as e: 39 | error "Error during SaleFailed.run", error = e.msgDetail 40 | return some State(SaleErrored(error: e)) 41 | -------------------------------------------------------------------------------- /codex/sales/states/filling.nim: -------------------------------------------------------------------------------- 1 | import pkg/stint 2 | import ../../logutils 3 | import ../../market 4 | import ../../utils/exceptions 5 | import ../statemachine 6 | import ../salesagent 7 | import ./filled 8 | import ./cancelled 9 | import ./failed 10 | import ./ignored 11 | import ./errored 12 | 13 | logScope: 14 | topics = "marketplace sales filling" 15 | 16 | type SaleFilling* = ref object of SaleState 17 | proof*: Groth16Proof 18 | 19 | method `$`*(state: SaleFilling): string = 20 | "SaleFilling" 21 | 22 | method onCancelled*(state: SaleFilling, request: StorageRequest): ?State = 23 | return some State(SaleCancelled()) 24 | 25 | method onFailed*(state: SaleFilling, request: StorageRequest): ?State = 26 | return some State(SaleFailed()) 27 | 28 | method run*( 29 | state: SaleFilling, machine: Machine 30 | ): Future[?State] {.async: (raises: []).} = 31 | let data = SalesAgent(machine).data 32 | let market = SalesAgent(machine).context.market 33 | 34 | without (request =? data.request): 35 | raiseAssert "Request not set" 36 | 37 | logScope: 38 | requestId = data.requestId 39 | slotIndex = data.slotIndex 40 | 41 | try: 42 | without collateral =? await market.slotCollateral(data.requestId, data.slotIndex), 43 | err: 44 | error "Failure attempting to fill slot: unable to calculate collateral", 45 | error = err.msg 46 | return some State(SaleErrored(error: err)) 47 | 48 | debug "Filling slot" 49 | try: 50 | await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral) 51 | except SlotStateMismatchError as e: 52 | debug "Slot is already filled, ignoring slot" 53 | return some State(SaleIgnored(reprocessSlot: false, returnsCollateral: true)) 54 | except MarketError as e: 55 | return some State(SaleErrored(error: e)) 56 | # other CatchableErrors are handled "automatically" by the SaleState 57 | 58 | return some State(SaleFilled()) 59 | except CancelledError as e: 60 | trace "SaleFilling.run was cancelled", error = e.msgDetail 61 | except CatchableError as e: 62 | error "Error during SaleFilling.run", error = e.msgDetail 63 | return some State(SaleErrored(error: e)) 64 | -------------------------------------------------------------------------------- /codex/sales/states/finished.nim: -------------------------------------------------------------------------------- 1 | import pkg/chronos 2 | 3 | import ../../logutils 4 | import ../../utils/exceptions 5 | import ../statemachine 6 | import ../salesagent 7 | import ./cancelled 8 | import ./failed 9 | import ./errored 10 | 11 | logScope: 12 | topics = "marketplace sales finished" 13 | 14 | type SaleFinished* = ref object of SaleState 15 | returnedCollateral*: ?UInt256 16 | 17 | method `$`*(state: SaleFinished): string = 18 | "SaleFinished" 19 | 20 | method onCancelled*(state: SaleFinished, request: StorageRequest): ?State = 21 | return some State(SaleCancelled()) 22 | 23 | method onFailed*(state: SaleFinished, request: StorageRequest): ?State = 24 | return some State(SaleFailed()) 25 | 26 | method run*( 27 | state: SaleFinished, machine: Machine 28 | ): Future[?State] {.async: (raises: []).} = 29 | let agent = SalesAgent(machine) 30 | let data = agent.data 31 | 32 | without request =? data.request: 33 | raiseAssert "no sale request" 34 | 35 | info "Slot finished and paid out", 36 | requestId = data.requestId, slotIndex = data.slotIndex 37 | 38 | try: 39 | if onClear =? agent.context.onClear: 40 | onClear(request, data.slotIndex) 41 | 42 | if onCleanUp =? agent.onCleanUp: 43 | await onCleanUp(returnedCollateral = state.returnedCollateral) 44 | except CancelledError as e: 45 | trace "SaleFilled.run onCleanUp was cancelled", error = e.msgDetail 46 | except CatchableError as e: 47 | error "Error during SaleFilled.run in onCleanUp callback", error = e.msgDetail 48 | return some State(SaleErrored(error: e)) 49 | -------------------------------------------------------------------------------- /codex/sales/states/ignored.nim: -------------------------------------------------------------------------------- 1 | import pkg/chronos 2 | 3 | import ../../logutils 4 | import ../../utils/exceptions 5 | import ../statemachine 6 | import ../salesagent 7 | import ./errored 8 | 9 | logScope: 10 | topics = "marketplace sales ignored" 11 | 12 | # Ignored slots could mean there was no availability or that the slot could 13 | # not be reserved. 14 | 15 | type SaleIgnored* = ref object of SaleState 16 | reprocessSlot*: bool # readd slot to queue with `seen` flag 17 | returnsCollateral*: bool # returns collateral when a reservation was created 18 | 19 | method `$`*(state: SaleIgnored): string = 20 | "SaleIgnored" 21 | 22 | method run*( 23 | state: SaleIgnored, machine: Machine 24 | ): Future[?State] {.async: (raises: []).} = 25 | let agent = SalesAgent(machine) 26 | let data = agent.data 27 | let market = agent.context.market 28 | 29 | without request =? data.request: 30 | raiseAssert "no sale request" 31 | 32 | var returnedCollateral = UInt256.none 33 | 34 | try: 35 | if state.returnsCollateral: 36 | # The returnedCollateral is needed because a reservation could 37 | # be created and the collateral assigned to that reservation. 38 | # The returnedCollateral will be used in the cleanup function 39 | # and be passed to the deleteReservation function. 40 | let slot = Slot(request: request, slotIndex: data.slotIndex) 41 | returnedCollateral = request.ask.collateralPerSlot.some 42 | 43 | if onCleanUp =? agent.onCleanUp: 44 | await onCleanUp( 45 | reprocessSlot = state.reprocessSlot, returnedCollateral = returnedCollateral 46 | ) 47 | except CancelledError as e: 48 | trace "SaleIgnored.run was cancelled", error = e.msgDetail 49 | except CatchableError as e: 50 | error "Error during SaleIgnored.run in onCleanUp", error = e.msgDetail 51 | return some State(SaleErrored(error: e)) 52 | -------------------------------------------------------------------------------- /codex/sales/states/payout.nim: -------------------------------------------------------------------------------- 1 | import ../../logutils 2 | import ../../market 3 | import ../../utils/exceptions 4 | import ../statemachine 5 | import ../salesagent 6 | import ./cancelled 7 | import ./failed 8 | import ./finished 9 | import ./errored 10 | 11 | logScope: 12 | topics = "marketplace sales payout" 13 | 14 | type SalePayout* = ref object of SaleState 15 | 16 | method `$`*(state: SalePayout): string = 17 | "SalePayout" 18 | 19 | method onCancelled*(state: SalePayout, request: StorageRequest): ?State = 20 | return some State(SaleCancelled()) 21 | 22 | method onFailed*(state: SalePayout, request: StorageRequest): ?State = 23 | return some State(SaleFailed()) 24 | 25 | method run*( 26 | state: SalePayout, machine: Machine 27 | ): Future[?State] {.async: (raises: []).} = 28 | let data = SalesAgent(machine).data 29 | let market = SalesAgent(machine).context.market 30 | 31 | without request =? data.request: 32 | raiseAssert "no sale request" 33 | 34 | try: 35 | let slot = Slot(request: request, slotIndex: data.slotIndex) 36 | debug "Collecting finished slot's reward", 37 | requestId = data.requestId, slotIndex = data.slotIndex 38 | let currentCollateral = await market.currentCollateral(slot.id) 39 | await market.freeSlot(slot.id) 40 | 41 | return some State(SaleFinished(returnedCollateral: some currentCollateral)) 42 | except CancelledError as e: 43 | trace "SalePayout.run onCleanUp was cancelled", error = e.msgDetail 44 | except CatchableError as e: 45 | error "Error during SalePayout.run", error = e.msgDetail 46 | return some State(SaleErrored(error: e)) 47 | -------------------------------------------------------------------------------- /codex/sales/states/provingsimulated.nim: -------------------------------------------------------------------------------- 1 | import ../../conf 2 | when codex_enable_proof_failures: 3 | import std/strutils 4 | import pkg/stint 5 | import pkg/ethers 6 | 7 | import ../../contracts/marketplace 8 | import ../../contracts/requests 9 | import ../../logutils 10 | import ../../market 11 | import ../../utils/exceptions 12 | import ../salescontext 13 | import ./proving 14 | import ./errored 15 | 16 | logScope: 17 | topics = "marketplace sales simulated-proving" 18 | 19 | type SaleProvingSimulated* = ref object of SaleProving 20 | failEveryNProofs*: int 21 | proofCount: int 22 | 23 | proc onSubmitProofError(error: ref CatchableError, period: Period, slotId: SlotId) = 24 | error "Submitting invalid proof failed", period, slotId, msg = error.msgDetail 25 | 26 | method prove*( 27 | state: SaleProvingSimulated, 28 | slot: Slot, 29 | challenge: ProofChallenge, 30 | onProve: OnProve, 31 | market: Market, 32 | currentPeriod: Period, 33 | ) {.async.} = 34 | try: 35 | trace "Processing proving in simulated mode" 36 | state.proofCount += 1 37 | if state.failEveryNProofs > 0 and state.proofCount mod state.failEveryNProofs == 0: 38 | state.proofCount = 0 39 | 40 | try: 41 | warn "Submitting INVALID proof", period = currentPeriod, slotId = slot.id 42 | await market.submitProof(slot.id, Groth16Proof.default) 43 | except ProofInvalidError as e: 44 | discard # expected 45 | except CancelledError as error: 46 | raise error 47 | except CatchableError as e: 48 | onSubmitProofError(e, currentPeriod, slot.id) 49 | else: 50 | await procCall SaleProving(state).prove( 51 | slot, challenge, onProve, market, currentPeriod 52 | ) 53 | except CancelledError as e: 54 | trace "Submitting INVALID proof cancelled", error = e.msgDetail 55 | raise e 56 | except CatchableError as e: 57 | error "Submitting INVALID proof failed", error = e.msgDetail 58 | -------------------------------------------------------------------------------- /codex/sales/states/slotreserving.nim: -------------------------------------------------------------------------------- 1 | import pkg/questionable 2 | import pkg/metrics 3 | 4 | import ../../logutils 5 | import ../../market 6 | import ../../utils/exceptions 7 | import ../salesagent 8 | import ../statemachine 9 | import ./cancelled 10 | import ./failed 11 | import ./ignored 12 | import ./downloading 13 | import ./errored 14 | 15 | type SaleSlotReserving* = ref object of SaleState 16 | 17 | logScope: 18 | topics = "marketplace sales reserving" 19 | 20 | method `$`*(state: SaleSlotReserving): string = 21 | "SaleSlotReserving" 22 | 23 | method onCancelled*(state: SaleSlotReserving, request: StorageRequest): ?State = 24 | return some State(SaleCancelled()) 25 | 26 | method onFailed*(state: SaleSlotReserving, request: StorageRequest): ?State = 27 | return some State(SaleFailed()) 28 | 29 | method run*( 30 | state: SaleSlotReserving, machine: Machine 31 | ): Future[?State] {.async: (raises: []).} = 32 | let agent = SalesAgent(machine) 33 | let data = agent.data 34 | let context = agent.context 35 | let market = context.market 36 | 37 | logScope: 38 | requestId = data.requestId 39 | slotIndex = data.slotIndex 40 | 41 | try: 42 | let canReserve = await market.canReserveSlot(data.requestId, data.slotIndex) 43 | if canReserve: 44 | try: 45 | trace "Reserving slot" 46 | await market.reserveSlot(data.requestId, data.slotIndex) 47 | except SlotReservationNotAllowedError as e: 48 | debug "Slot cannot be reserved, ignoring", error = e.msg 49 | return some State(SaleIgnored(reprocessSlot: false, returnsCollateral: true)) 50 | except MarketError as e: 51 | return some State(SaleErrored(error: e)) 52 | # other CatchableErrors are handled "automatically" by the SaleState 53 | 54 | trace "Slot successfully reserved" 55 | return some State(SaleDownloading()) 56 | else: 57 | # do not re-add this slot to the queue, and return bytes from Reservation to 58 | # the Availability 59 | debug "Slot cannot be reserved, ignoring" 60 | return some State(SaleIgnored(reprocessSlot: false, returnsCollateral: true)) 61 | except CancelledError as e: 62 | trace "SaleSlotReserving.run was cancelled", error = e.msgDetail 63 | except CatchableError as e: 64 | error "Error during SaleSlotReserving.run", error = e.msgDetail 65 | return some State(SaleErrored(error: e)) 66 | -------------------------------------------------------------------------------- /codex/slots.nim: -------------------------------------------------------------------------------- 1 | import ./slots/builder 2 | import ./slots/sampler 3 | import ./slots/proofs 4 | import ./slots/types 5 | 6 | export builder, sampler, proofs, types 7 | -------------------------------------------------------------------------------- /codex/slots/builder.nim: -------------------------------------------------------------------------------- 1 | import ./builder/builder 2 | import ./converters 3 | 4 | import ../merkletree 5 | 6 | export builder, converters 7 | 8 | type Poseidon2Builder* = SlotsBuilder[Poseidon2Tree, Poseidon2Hash] 9 | -------------------------------------------------------------------------------- /codex/slots/proofs.nim: -------------------------------------------------------------------------------- 1 | import ./proofs/backends 2 | import ./proofs/prover 3 | import ./proofs/backendfactory 4 | 5 | export circomcompat, prover, backendfactory 6 | -------------------------------------------------------------------------------- /codex/slots/proofs/backends.nim: -------------------------------------------------------------------------------- 1 | import ./backends/circomcompat 2 | 3 | export circomcompat 4 | 5 | type AnyBackend* = CircomCompat 6 | -------------------------------------------------------------------------------- /codex/slots/proofs/backends/converters.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2024 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | {.push raises: [].} 11 | 12 | import pkg/circomcompat 13 | 14 | import ../../../contracts 15 | import ../../types 16 | import ../../../merkletree 17 | 18 | type 19 | CircomG1* = G1 20 | CircomG2* = G2 21 | 22 | CircomProof* = Proof 23 | CircomKey* = VerifyingKey 24 | CircomInputs* = Inputs 25 | 26 | proc toCircomInputs*(inputs: ProofInputs[Poseidon2Hash]): CircomInputs = 27 | var 28 | slotIndex = inputs.slotIndex.toF.toBytes.toArray32 29 | datasetRoot = inputs.datasetRoot.toBytes.toArray32 30 | entropy = inputs.entropy.toBytes.toArray32 31 | 32 | elms = [entropy, datasetRoot, slotIndex] 33 | 34 | let inputsPtr = allocShared0(32 * elms.len) 35 | copyMem(inputsPtr, addr elms[0], elms.len * 32) 36 | 37 | CircomInputs(elms: cast[ptr array[32, byte]](inputsPtr), len: elms.len.uint) 38 | 39 | proc releaseCircomInputs*(inputs: var CircomInputs) = 40 | if not inputs.elms.isNil: 41 | deallocShared(inputs.elms) 42 | inputs.elms = nil 43 | 44 | func toG1*(g: CircomG1): G1Point = 45 | G1Point(x: UInt256.fromBytesLE(g.x), y: UInt256.fromBytesLE(g.y)) 46 | 47 | func toG2*(g: CircomG2): G2Point = 48 | G2Point( 49 | x: Fp2Element(real: UInt256.fromBytesLE(g.x[0]), imag: UInt256.fromBytesLE(g.x[1])), 50 | y: Fp2Element(real: UInt256.fromBytesLE(g.y[0]), imag: UInt256.fromBytesLE(g.y[1])), 51 | ) 52 | 53 | func toGroth16Proof*(proof: CircomProof): Groth16Proof = 54 | Groth16Proof(a: proof.a.toG1, b: proof.b.toG2, c: proof.c.toG1) 55 | -------------------------------------------------------------------------------- /codex/slots/proofs/backendutils.nim: -------------------------------------------------------------------------------- 1 | import ./backends 2 | 3 | type BackendUtils* = ref object of RootObj 4 | 5 | method initializeCircomBackend*( 6 | self: BackendUtils, r1csFile: string, wasmFile: string, zKeyFile: string 7 | ): AnyBackend {.base, gcsafe.} = 8 | CircomCompat.init(r1csFile, wasmFile, zKeyFile) 9 | -------------------------------------------------------------------------------- /codex/slots/sampler.nim: -------------------------------------------------------------------------------- 1 | import ./sampler/sampler 2 | import ./sampler/utils 3 | 4 | import ../merkletree 5 | 6 | export sampler, utils 7 | 8 | type Poseidon2Sampler* = DataSampler[Poseidon2Tree, Poseidon2Hash] 9 | -------------------------------------------------------------------------------- /codex/slots/sampler/utils.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2024 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import std/bitops 11 | 12 | import pkg/questionable/results 13 | import pkg/constantine/math/arithmetic 14 | 15 | import ../../merkletree 16 | 17 | func extractLowBits*[n: static int](elm: BigInt[n], k: int): uint64 = 18 | doAssert(k > 0 and k <= 64) 19 | var r = 0'u64 20 | for i in 0 ..< k: 21 | let b = bit[n](elm, i) 22 | let y = uint64(b) 23 | if (y != 0): 24 | r = bitor(r, 1'u64 shl i) 25 | r 26 | 27 | func extractLowBits(fld: Poseidon2Hash, k: int): uint64 = 28 | let elm: BigInt[254] = fld.toBig() 29 | return extractLowBits(elm, k) 30 | 31 | func floorLog2*(x: int): int = 32 | doAssert (x > 0) 33 | var k = -1 34 | var y = x 35 | while (y > 0): 36 | k += 1 37 | y = y shr 1 38 | return k 39 | 40 | func ceilingLog2*(x: int): int = 41 | doAssert (x > 0) 42 | return (floorLog2(x - 1) + 1) 43 | 44 | func toBlkInSlot*(cell: Natural, numCells: Natural): Natural = 45 | let log2 = ceilingLog2(numCells) 46 | doAssert(1 shl log2 == numCells, "`numCells` is assumed to be a power of two") 47 | 48 | return cell div numCells 49 | 50 | func toCellInBlk*(cell: Natural, numCells: Natural): Natural = 51 | let log2 = ceilingLog2(numCells) 52 | doAssert(1 shl log2 == numCells, "`numCells` is assumed to be a power of two") 53 | 54 | return cell mod numCells 55 | 56 | func cellIndex*( 57 | entropy: Poseidon2Hash, slotRoot: Poseidon2Hash, numCells: Natural, counter: Natural 58 | ): Natural = 59 | let log2 = ceilingLog2(numCells) 60 | doAssert(1 shl log2 == numCells, "`numCells` is assumed to be a power of two") 61 | 62 | let hash = Sponge.digest(@[entropy, slotRoot, counter.toF], rate = 2) 63 | return int(extractLowBits(hash, log2)) 64 | 65 | func cellIndices*( 66 | entropy: Poseidon2Hash, 67 | slotRoot: Poseidon2Hash, 68 | numCells: Natural, 69 | nSamples: Natural, 70 | ): seq[Natural] = 71 | var indices: seq[Natural] 72 | for i in 1 .. nSamples: 73 | indices.add(cellIndex(entropy, slotRoot, numCells, i)) 74 | 75 | indices 76 | -------------------------------------------------------------------------------- /codex/slots/types.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2024 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | type 11 | Sample*[H] = object 12 | cellData*: seq[H] 13 | merklePaths*: seq[H] 14 | 15 | PublicInputs*[H] = object 16 | slotIndex*: int 17 | datasetRoot*: H 18 | entropy*: H 19 | 20 | ProofInputs*[H] = object 21 | entropy*: H 22 | datasetRoot*: H 23 | slotIndex*: Natural 24 | slotRoot*: H 25 | nCellsPerSlot*: Natural 26 | nSlotsPerDataSet*: Natural 27 | slotProof*: seq[H] 28 | # inclusion proof that shows that the slot root (leaf) is part of the dataset (root) 29 | samples*: seq[Sample[H]] 30 | # inclusion proofs which show that the selected cells (leafs) are part of the slot (roots) 31 | -------------------------------------------------------------------------------- /codex/stores.nim: -------------------------------------------------------------------------------- 1 | import ./stores/cachestore 2 | import ./stores/blockstore 3 | import ./stores/networkstore 4 | import ./stores/repostore 5 | import ./stores/maintenance 6 | import ./stores/keyutils 7 | import ./stores/treehelper 8 | 9 | export 10 | cachestore, blockstore, networkstore, repostore, keyutils, treehelper, maintenance 11 | -------------------------------------------------------------------------------- /codex/stores/keyutils.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2022 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import pkg/upraises 11 | push: 12 | {.upraises: [].} 13 | 14 | import std/sugar 15 | import pkg/questionable/results 16 | import pkg/datastore 17 | import pkg/libp2p 18 | import ../namespaces 19 | import ../manifest 20 | 21 | const 22 | CodexMetaKey* = Key.init(CodexMetaNamespace).tryGet 23 | CodexRepoKey* = Key.init(CodexRepoNamespace).tryGet 24 | CodexBlocksKey* = Key.init(CodexBlocksNamespace).tryGet 25 | CodexTotalBlocksKey* = Key.init(CodexBlockTotalNamespace).tryGet 26 | CodexManifestKey* = Key.init(CodexManifestNamespace).tryGet 27 | BlocksTtlKey* = Key.init(CodexBlocksTtlNamespace).tryGet 28 | BlockProofKey* = Key.init(CodexBlockProofNamespace).tryGet 29 | QuotaKey* = Key.init(CodexQuotaNamespace).tryGet 30 | QuotaUsedKey* = (QuotaKey / "used").tryGet 31 | QuotaReservedKey* = (QuotaKey / "reserved").tryGet 32 | 33 | func makePrefixKey*(postFixLen: int, cid: Cid): ?!Key = 34 | let cidKey = ?Key.init(($cid)[^postFixLen ..^ 1] & "/" & $cid) 35 | 36 | if ?cid.isManifest: 37 | success CodexManifestKey / cidKey 38 | else: 39 | success CodexBlocksKey / cidKey 40 | 41 | proc createBlockExpirationMetadataKey*(cid: Cid): ?!Key = 42 | BlocksTtlKey / $cid 43 | 44 | proc createBlockExpirationMetadataQueryKey*(): ?!Key = 45 | let queryString = ?(BlocksTtlKey / "*") 46 | Key.init(queryString) 47 | 48 | proc createBlockCidAndProofMetadataKey*(treeCid: Cid, index: Natural): ?!Key = 49 | (BlockProofKey / $treeCid).flatMap((k: Key) => k / $index) 50 | -------------------------------------------------------------------------------- /codex/stores/repostore.nim: -------------------------------------------------------------------------------- 1 | import ./repostore/store 2 | import ./repostore/types 3 | import ./repostore/coders 4 | 5 | export store, types, coders 6 | -------------------------------------------------------------------------------- /codex/stores/repostore/coders.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2024 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | ## 10 | 11 | import std/sugar 12 | import pkg/libp2p/cid 13 | import pkg/serde/json 14 | import pkg/stew/byteutils 15 | import pkg/stew/endians2 16 | 17 | import ./types 18 | import ../../errors 19 | import ../../merkletree 20 | import ../../utils/json 21 | 22 | proc encode*(t: QuotaUsage): seq[byte] = 23 | t.toJson().toBytes() 24 | 25 | proc decode*(T: type QuotaUsage, bytes: seq[byte]): ?!T = 26 | T.fromJson(bytes) 27 | 28 | proc encode*(t: BlockMetadata): seq[byte] = 29 | t.toJson().toBytes() 30 | 31 | proc decode*(T: type BlockMetadata, bytes: seq[byte]): ?!T = 32 | T.fromJson(bytes) 33 | 34 | proc encode*(t: LeafMetadata): seq[byte] = 35 | t.toJson().toBytes() 36 | 37 | proc decode*(T: type LeafMetadata, bytes: seq[byte]): ?!T = 38 | T.fromJson(bytes) 39 | 40 | proc encode*(t: DeleteResult): seq[byte] = 41 | t.toJson().toBytes() 42 | 43 | proc decode*(T: type DeleteResult, bytes: seq[byte]): ?!T = 44 | T.fromJson(bytes) 45 | 46 | proc encode*(t: StoreResult): seq[byte] = 47 | t.toJson().toBytes() 48 | 49 | proc decode*(T: type StoreResult, bytes: seq[byte]): ?!T = 50 | T.fromJson(bytes) 51 | 52 | proc encode*(i: uint64): seq[byte] = 53 | @(i.toBytesBE) 54 | 55 | proc decode*(T: type uint64, bytes: seq[byte]): ?!T = 56 | if bytes.len >= sizeof(uint64): 57 | success(uint64.fromBytesBE(bytes)) 58 | else: 59 | failure("Not enough bytes to decode `uint64`") 60 | 61 | proc encode*(i: Natural | enum): seq[byte] = 62 | cast[uint64](i).encode 63 | 64 | proc decode*(T: typedesc[Natural | enum], bytes: seq[byte]): ?!T = 65 | uint64.decode(bytes).map((ui: uint64) => cast[T](ui)) 66 | -------------------------------------------------------------------------------- /codex/stores/treehelper.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2023 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import pkg/upraises 11 | 12 | push: 13 | {.upraises: [].} 14 | 15 | import std/sugar 16 | import pkg/chronos 17 | import pkg/chronos/futures 18 | import pkg/metrics 19 | import pkg/questionable 20 | import pkg/questionable/results 21 | 22 | import ./blockstore 23 | import ../utils/asynciter 24 | import ../merkletree 25 | 26 | proc putSomeProofs*( 27 | store: BlockStore, tree: CodexTree, iter: Iter[int] 28 | ): Future[?!void] {.async.} = 29 | without treeCid =? tree.rootCid, err: 30 | return failure(err) 31 | 32 | for i in iter: 33 | if i notin 0 ..< tree.leavesCount: 34 | return failure( 35 | "Invalid leaf index " & $i & ", tree with cid " & $treeCid & " has " & 36 | $tree.leavesCount & " leaves" 37 | ) 38 | 39 | without blkCid =? tree.getLeafCid(i), err: 40 | return failure(err) 41 | 42 | without proof =? tree.getProof(i), err: 43 | return failure(err) 44 | 45 | let res = await store.putCidAndProof(treeCid, i, blkCid, proof) 46 | 47 | if err =? res.errorOption: 48 | return failure(err) 49 | 50 | success() 51 | 52 | proc putSomeProofs*( 53 | store: BlockStore, tree: CodexTree, iter: Iter[Natural] 54 | ): Future[?!void] = 55 | store.putSomeProofs(tree, iter.map((i: Natural) => i.ord)) 56 | 57 | proc putAllProofs*(store: BlockStore, tree: CodexTree): Future[?!void] = 58 | store.putSomeProofs(tree, Iter[int].new(0 ..< tree.leavesCount)) 59 | -------------------------------------------------------------------------------- /codex/streams.nim: -------------------------------------------------------------------------------- 1 | import ./streams/seekablestream 2 | import ./streams/storestream 3 | import ./streams/asyncstreamwrapper 4 | 5 | export seekablestream, storestream, asyncstreamwrapper 6 | -------------------------------------------------------------------------------- /codex/streams/seekablestream.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2022 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import pkg/libp2p/stream/lpstream 11 | import pkg/chronos 12 | 13 | import ../logutils 14 | 15 | export lpstream, chronos, logutils 16 | 17 | logScope: 18 | topics = "codex seekablestream" 19 | 20 | type SeekableStream* = ref object of LPStream 21 | offset*: int 22 | 23 | method `size`*(self: SeekableStream): int {.base.} = 24 | raiseAssert("method unimplemented") 25 | 26 | proc setPos*(self: SeekableStream, pos: int) = 27 | self.offset = pos 28 | -------------------------------------------------------------------------------- /codex/systemclock.nim: -------------------------------------------------------------------------------- 1 | import std/times 2 | import pkg/upraises 3 | import ./clock 4 | 5 | type SystemClock* = ref object of Clock 6 | 7 | method now*(clock: SystemClock): SecondsSince1970 {.upraises: [].} = 8 | let now = times.now().utc 9 | now.toTime().toUnix() 10 | -------------------------------------------------------------------------------- /codex/utils/arrayutils.nim: -------------------------------------------------------------------------------- 1 | import std/sequtils 2 | 3 | proc createDoubleArray*( 4 | outerLen, innerLen: int 5 | ): ptr UncheckedArray[ptr UncheckedArray[byte]] = 6 | # Allocate outer array 7 | result = cast[ptr UncheckedArray[ptr UncheckedArray[byte]]](allocShared0( 8 | sizeof(ptr UncheckedArray[byte]) * outerLen 9 | )) 10 | 11 | # Allocate each inner array 12 | for i in 0 ..< outerLen: 13 | result[i] = cast[ptr UncheckedArray[byte]](allocShared0(sizeof(byte) * innerLen)) 14 | 15 | proc freeDoubleArray*( 16 | arr: ptr UncheckedArray[ptr UncheckedArray[byte]], outerLen: int 17 | ) = 18 | # Free each inner array 19 | for i in 0 ..< outerLen: 20 | if not arr[i].isNil: 21 | deallocShared(arr[i]) 22 | 23 | # Free outer array 24 | if not arr.isNil: 25 | deallocShared(arr) 26 | 27 | proc makeUncheckedArray*( 28 | data: ref seq[seq[byte]] 29 | ): ptr UncheckedArray[ptr UncheckedArray[byte]] = 30 | result = cast[ptr UncheckedArray[ptr UncheckedArray[byte]]](alloc0( 31 | sizeof(ptr UncheckedArray[byte]) * data[].len 32 | )) 33 | 34 | for i, blk in data[]: 35 | if blk.len > 0: 36 | result[i] = cast[ptr UncheckedArray[byte]](addr blk[0]) 37 | else: 38 | result[i] = nil 39 | -------------------------------------------------------------------------------- /codex/utils/digest.nim: -------------------------------------------------------------------------------- 1 | from pkg/libp2p import MultiHash 2 | 3 | func digestBytes*(mhash: MultiHash): seq[byte] = 4 | ## Extract hash digestBytes 5 | ## 6 | 7 | mhash.data.buffer[mhash.dpos ..< mhash.dpos + mhash.size] 8 | -------------------------------------------------------------------------------- /codex/utils/exceptions.nim: -------------------------------------------------------------------------------- 1 | import std/strformat 2 | 3 | proc msgDetail*(e: ref CatchableError): string = 4 | var msg = e.msg 5 | if e.parent != nil: 6 | msg = fmt"{msg} Inner exception: {e.parent.msg}" 7 | return msg 8 | -------------------------------------------------------------------------------- /codex/utils/json.nim: -------------------------------------------------------------------------------- 1 | import std/options 2 | import std/typetraits 3 | from pkg/ethers import Address 4 | from pkg/libp2p import 5 | Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$` 6 | import pkg/contractabi 7 | import pkg/codexdht/discv5/node as dn 8 | import pkg/serde/json 9 | import pkg/questionable/results 10 | import ../errors 11 | 12 | export json 13 | 14 | proc fromJson*(_: type Cid, json: JsonNode): ?!Cid = 15 | expectJsonKind(Cid, JString, json) 16 | Cid.init(json.str).mapFailure 17 | 18 | func `%`*(cid: Cid): JsonNode = 19 | % $cid 20 | 21 | func `%`*(obj: PeerId): JsonNode = 22 | % $obj 23 | 24 | func `%`*(obj: SignedPeerRecord): JsonNode = 25 | % $obj 26 | 27 | func `%`*(obj: dn.Address): JsonNode = 28 | % $obj 29 | 30 | func `%`*(obj: AddressInfo): JsonNode = 31 | % $obj.address 32 | 33 | func `%`*(obj: MultiAddress): JsonNode = 34 | % $obj 35 | 36 | func `%`*(address: ethers.Address): JsonNode = 37 | % $address 38 | -------------------------------------------------------------------------------- /codex/utils/keyutils.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2022 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import pkg/upraises 11 | push: 12 | {.upraises: [].} 13 | 14 | import pkg/questionable/results 15 | import pkg/libp2p/crypto/crypto 16 | 17 | import ./fileutils 18 | import ../errors 19 | import ../logutils 20 | import ../rng 21 | 22 | export crypto 23 | 24 | type 25 | CodexKeyError = object of CodexError 26 | CodexKeyUnsafeError = object of CodexKeyError 27 | 28 | proc setupKey*(path: string): ?!PrivateKey = 29 | if not path.fileAccessible({AccessFlags.Find}): 30 | info "Creating a private key and saving it" 31 | let 32 | res = ?PrivateKey.random(Rng.instance()[]).mapFailure(CodexKeyError) 33 | bytes = ?res.getBytes().mapFailure(CodexKeyError) 34 | 35 | ?path.secureWriteFile(bytes).mapFailure(CodexKeyError) 36 | return PrivateKey.init(bytes).mapFailure(CodexKeyError) 37 | 38 | info "Found a network private key" 39 | if not ?checkSecureFile(path).mapFailure(CodexKeyError): 40 | warn "The network private key file is not safe, aborting" 41 | return failure newException( 42 | CodexKeyUnsafeError, "The network private key file is not safe" 43 | ) 44 | 45 | let kb = ?path.readAllBytes().mapFailure(CodexKeyError) 46 | return PrivateKey.init(kb).mapFailure(CodexKeyError) 47 | -------------------------------------------------------------------------------- /codex/utils/natutils.nim: -------------------------------------------------------------------------------- 1 | {.push raises: [].} 2 | 3 | import 4 | std/[tables, hashes], pkg/results, pkg/stew/shims/net as stewNet, chronos, chronicles 5 | 6 | import pkg/libp2p 7 | 8 | type NatStrategy* = enum 9 | NatAny 10 | NatUpnp 11 | NatPmp 12 | NatNone 13 | 14 | type IpLimits* = object 15 | limit*: uint 16 | ips: Table[IpAddress, uint] 17 | 18 | func hash*(ip: IpAddress): Hash = 19 | case ip.family 20 | of IpAddressFamily.IPv6: 21 | hash(ip.address_v6) 22 | of IpAddressFamily.IPv4: 23 | hash(ip.address_v4) 24 | 25 | func inc*(ipLimits: var IpLimits, ip: IpAddress): bool = 26 | let val = ipLimits.ips.getOrDefault(ip, 0) 27 | if val < ipLimits.limit: 28 | ipLimits.ips[ip] = val + 1 29 | true 30 | else: 31 | false 32 | 33 | func dec*(ipLimits: var IpLimits, ip: IpAddress) = 34 | let val = ipLimits.ips.getOrDefault(ip, 0) 35 | if val == 1: 36 | ipLimits.ips.del(ip) 37 | elif val > 1: 38 | ipLimits.ips[ip] = val - 1 39 | 40 | func isGlobalUnicast*(address: TransportAddress): bool = 41 | if address.isGlobal() and address.isUnicast(): true else: false 42 | 43 | func isGlobalUnicast*(address: IpAddress): bool = 44 | let a = initTAddress(address, Port(0)) 45 | a.isGlobalUnicast() 46 | 47 | proc getRouteIpv4*(): Result[IpAddress, cstring] = 48 | # Avoiding Exception with initTAddress and can't make it work with static. 49 | # Note: `publicAddress` is only used an "example" IP to find the best route, 50 | # no data is send over the network to this IP! 51 | let 52 | publicAddress = TransportAddress( 53 | family: AddressFamily.IPv4, address_v4: [1'u8, 1, 1, 1], port: Port(0) 54 | ) 55 | route = getBestRoute(publicAddress) 56 | 57 | if route.source.isUnspecified(): 58 | err("No best ipv4 route found") 59 | else: 60 | let ip = 61 | try: 62 | route.source.address() 63 | except ValueError as e: 64 | # This should not occur really. 65 | error "Address conversion error", exception = e.name, msg = e.msg 66 | return err("Invalid IP address") 67 | ok(ip) 68 | -------------------------------------------------------------------------------- /codex/utils/options.nim: -------------------------------------------------------------------------------- 1 | import std/macros 2 | import pkg/questionable 3 | import pkg/questionable/operators 4 | 5 | export questionable 6 | 7 | proc `as`*[T](value: T, U: type): ?U = 8 | ## Casts a value to another type, returns an Option. 9 | ## When the cast succeeds, the option will contain the casted value. 10 | ## When the cast fails, the option will have no value. 11 | 12 | # In Nim 2.0.x, check 42.some as int == none(int) 13 | # Maybe because some 42.some looks like Option[Option[int]] 14 | # So we check first that the value is an option of the expected type. 15 | # In that case, we do not need to do anything, just return the value as it is. 16 | when value is Option[U]: 17 | return value 18 | 19 | when value is U: 20 | return some value 21 | elif value is ref object: 22 | if value of U: 23 | return some U(value) 24 | 25 | Option.liftBinary `as` 26 | 27 | # Template that wraps type with `Option[]` only if it is already not `Option` type 28 | template WrapOption*(input: untyped): type = 29 | when input is Option: 30 | input 31 | else: 32 | Option[input] 33 | 34 | macro createType(t: typedesc): untyped = 35 | var objectType = getType(t) 36 | 37 | # Work around for https://github.com/nim-lang/Nim/issues/23112 38 | while objectType.kind == nnkBracketExpr and objectType[0].eqIdent"typeDesc": 39 | objectType = getType(objectType[1]) 40 | 41 | expectKind(objectType, NimNodeKind.nnkObjectTy) 42 | var fields = nnkRecList.newTree() 43 | 44 | # Generates the list of fields that are wrapped in `Option[T]`. 45 | # Technically wrapped with `WrapOption` which is template used to prevent 46 | # re-wrapping already filed which is `Option[T]`. 47 | for field in objectType[2]: 48 | let fieldType = getTypeInst(field) 49 | let newFieldNode = nnkIdentDefs.newTree( 50 | ident($field), nnkCall.newTree(ident("WrapOption"), fieldType), newEmptyNode() 51 | ) 52 | 53 | fields.add(newFieldNode) 54 | 55 | # Creates new object type T with the fields lists from steps above. 56 | let tSym = genSym(nskType, "T") 57 | nnkStmtList.newTree( 58 | nnkTypeSection.newTree( 59 | nnkTypeDef.newTree( 60 | tSym, 61 | newEmptyNode(), 62 | nnkObjectTy.newTree(newEmptyNode(), newEmptyNode(), fields), 63 | ) 64 | ), 65 | tSym, 66 | ) 67 | 68 | template Optionalize*(t: typed): untyped = 69 | ## Takes object type and wraps all the first level fields into 70 | ## Option type unless it is already Option type. 71 | createType(t) 72 | -------------------------------------------------------------------------------- /codex/utils/poseidon2digest.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2023 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import pkg/poseidon2 11 | import pkg/questionable/results 12 | import pkg/libp2p/multihash 13 | import pkg/stew/byteutils 14 | 15 | import ../merkletree 16 | 17 | func spongeDigest*( 18 | _: type Poseidon2Hash, bytes: openArray[byte], rate: static int = 2 19 | ): ?!Poseidon2Hash = 20 | ## Hashes chunks of data with a sponge of rate 1 or 2. 21 | ## 22 | 23 | success Sponge.digest(bytes, rate) 24 | 25 | func spongeDigest*( 26 | _: type Poseidon2Hash, bytes: openArray[Bn254Fr], rate: static int = 2 27 | ): ?!Poseidon2Hash = 28 | ## Hashes chunks of elements with a sponge of rate 1 or 2. 29 | ## 30 | 31 | success Sponge.digest(bytes, rate) 32 | 33 | func digestTree*( 34 | _: type Poseidon2Tree, bytes: openArray[byte], chunkSize: int 35 | ): ?!Poseidon2Tree = 36 | ## Hashes chunks of data with a sponge of rate 2, and combines the 37 | ## resulting chunk hashes in a merkle root. 38 | ## 39 | 40 | # doAssert not(rate == 1 or rate == 2), "rate can only be 1 or 2" 41 | 42 | if not chunkSize > 0: 43 | return failure("chunkSize must be greater than 0") 44 | 45 | var index = 0 46 | var leaves: seq[Poseidon2Hash] 47 | while index < bytes.len: 48 | let start = index 49 | let finish = min(index + chunkSize, bytes.len) 50 | let digest = ?Poseidon2Hash.spongeDigest(bytes.toOpenArray(start, finish - 1), 2) 51 | leaves.add(digest) 52 | index += chunkSize 53 | return Poseidon2Tree.init(leaves) 54 | 55 | func digest*( 56 | _: type Poseidon2Tree, bytes: openArray[byte], chunkSize: int 57 | ): ?!Poseidon2Hash = 58 | ## Hashes chunks of data with a sponge of rate 2, and combines the 59 | ## resulting chunk hashes in a merkle root. 60 | ## 61 | 62 | (?Poseidon2Tree.digestTree(bytes, chunkSize)).root 63 | 64 | func digestMhash*( 65 | _: type Poseidon2Tree, bytes: openArray[byte], chunkSize: int 66 | ): ?!MultiHash = 67 | ## Hashes chunks of data with a sponge of rate 2 and 68 | ## returns the multihash of the root 69 | ## 70 | 71 | let hash = ?Poseidon2Tree.digest(bytes, chunkSize) 72 | 73 | ?MultiHash.init(Pos2Bn128MrklCodec, hash).mapFailure 74 | -------------------------------------------------------------------------------- /codex/utils/stintutils.nim: -------------------------------------------------------------------------------- 1 | import pkg/stint 2 | 3 | func fromDecimal*(T: typedesc[StUint | StInt], s: string): T {.inline.} = 4 | parse(s, type result, radix = 10) 5 | -------------------------------------------------------------------------------- /codex/utils/timer.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2023 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | ## Timer 11 | ## Used to execute a callback in a loop 12 | 13 | {.push raises: [].} 14 | 15 | import pkg/chronos 16 | 17 | import ../logutils 18 | 19 | type 20 | TimerCallback* = proc(): Future[void] {.gcsafe, async: (raises: []).} 21 | Timer* = ref object of RootObj 22 | callback: TimerCallback 23 | interval: Duration 24 | name: string 25 | loopFuture: Future[void] 26 | 27 | proc new*(T: type Timer, timerName = "Unnamed Timer"): Timer = 28 | ## Create a new Timer intance with the given name 29 | Timer(name: timerName) 30 | 31 | proc timerLoop(timer: Timer) {.async: (raises: []).} = 32 | try: 33 | while true: 34 | await timer.callback() 35 | await sleepAsync(timer.interval) 36 | except CancelledError: 37 | discard # do not propagate as timerLoop is asyncSpawned 38 | 39 | method start*( 40 | timer: Timer, callback: TimerCallback, interval: Duration 41 | ) {.gcsafe, base.} = 42 | if timer.loopFuture != nil: 43 | return 44 | trace "Timer starting: ", name = timer.name 45 | timer.callback = callback 46 | timer.interval = interval 47 | timer.loopFuture = timerLoop(timer) 48 | 49 | method stop*(timer: Timer) {.base, async: (raises: []).} = 50 | if timer.loopFuture != nil and not timer.loopFuture.finished: 51 | trace "Timer stopping: ", name = timer.name 52 | await timer.loopFuture.cancelAndWait() 53 | timer.loopFuture = nil 54 | -------------------------------------------------------------------------------- /codex/utils/trackedfutures.nim: -------------------------------------------------------------------------------- 1 | import std/tables 2 | import pkg/chronos 3 | 4 | import ../logutils 5 | 6 | {.push raises: [].} 7 | 8 | type 9 | TrackedFuture = Future[void].Raising([]) 10 | TrackedFutures* = ref object 11 | futures: Table[uint, TrackedFuture] 12 | cancelling: bool 13 | 14 | logScope: 15 | topics = "trackable futures" 16 | 17 | proc len*(self: TrackedFutures): int = 18 | self.futures.len 19 | 20 | proc removeFuture(self: TrackedFutures, future: TrackedFuture) = 21 | if not self.cancelling and not future.isNil: 22 | self.futures.del(future.id) 23 | 24 | proc track*(self: TrackedFutures, fut: TrackedFuture) = 25 | if self.cancelling: 26 | return 27 | 28 | if fut.finished: 29 | return 30 | 31 | self.futures[fut.id] = fut 32 | 33 | proc cb(udata: pointer) = 34 | self.removeFuture(fut) 35 | 36 | fut.addCallback(cb) 37 | 38 | proc cancelTracked*(self: TrackedFutures) {.async: (raises: []).} = 39 | self.cancelling = true 40 | 41 | trace "cancelling tracked futures", len = self.futures.len 42 | let cancellations = self.futures.values.toSeq.mapIt(it.cancelAndWait()) 43 | await noCancel allFutures cancellations 44 | 45 | self.futures.clear() 46 | self.cancelling = false 47 | -------------------------------------------------------------------------------- /codex/validationconfig.nim: -------------------------------------------------------------------------------- 1 | import std/strformat 2 | import pkg/questionable 3 | import pkg/questionable/results 4 | 5 | type 6 | ValidationGroups* = range[2 .. 65535] 7 | MaxSlots* = int 8 | ValidationConfig* = object 9 | maxSlots: MaxSlots 10 | groups: ?ValidationGroups 11 | groupIndex: uint16 12 | 13 | func init*( 14 | _: type ValidationConfig, 15 | maxSlots: MaxSlots, 16 | groups: ?ValidationGroups, 17 | groupIndex: uint16 = 0, 18 | ): ?!ValidationConfig = 19 | if maxSlots < 0: 20 | return failure "The value of maxSlots must be greater than " & 21 | fmt"or equal to 0! (got: {maxSlots})" 22 | if validationGroups =? groups and groupIndex >= uint16(validationGroups): 23 | return failure "The value of the group index must be less than " & 24 | fmt"validation groups! (got: {groupIndex = }, " & fmt"groups = {validationGroups})" 25 | 26 | success ValidationConfig(maxSlots: maxSlots, groups: groups, groupIndex: groupIndex) 27 | 28 | func maxSlots*(config: ValidationConfig): MaxSlots = 29 | config.maxSlots 30 | 31 | func groups*(config: ValidationConfig): ?ValidationGroups = 32 | config.groups 33 | 34 | func groupIndex*(config: ValidationConfig): uint16 = 35 | config.groupIndex 36 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | # Codex Docker Image 2 | 3 | Codex provides pre-built docker images and they are stored in the [codexstorage/nim-codex](https://hub.docker.com/repository/docker/codexstorage/nim-codex) repository. 4 | 5 | 6 | ## Run 7 | 8 | We can run Codex Docker image using CLI 9 | ```shell 10 | # Default run 11 | docker run --rm codexstorage/nim-codex 12 | 13 | # Mount local datadir 14 | docker run -v ./datadir:/datadir --rm codexstorage/nim-codex codex --data-dir=/datadir 15 | ``` 16 | 17 | And Docker Compose 18 | ```shell 19 | # Run in detached mode 20 | docker-compose up -d 21 | ``` 22 | 23 | 24 | ## Arguments 25 | 26 | Docker image is based on the [codex.Dockerfile](codex.Dockerfile) and there is 27 | ``` 28 | ENTRYPOINT ["/docker-entrypoint.sh"] 29 | CMD ["codex"] 30 | ``` 31 | 32 | It means that at the image run it will just run `codex` application without any arguments and we can pass them as a regular arguments, by overriding command 33 | ```shell 34 | docker run codexstorage/nim-codex codex --api-bindaddr=0.0.0.0 --api-port=8080 35 | ``` 36 | 37 | 38 | ## Environment variables 39 | 40 | We can configure Codex using [Environment variables](../README#environment-variables) and [docker-compose.yaml](docker-compose.yaml) file can be useful as an example. 41 | 42 | We also added a temporary environment variable `NAT_IP_AUTO` to the entrypoint which is set as `false` for releases and ` true` for regular builds. That approach is useful for Dist-Tests. 43 | ```shell 44 | # Disable NAT_IP_AUTO for regular builds 45 | docker run -e NAT_IP_AUTO=false codexstorage/nim-codex 46 | ``` 47 | 48 | 49 | ## Slim 50 | 1. Build the image using `docker build -t codexstorage/codexsetup:latest -f codex.Dockerfile ..` 51 | 2. The docker image can then be minified using [slim](https://github.com/slimtoolkit/slim). Install slim on your path and then run: 52 | ```shell 53 | slim # brings up interactive prompt 54 | >>> build --target status-im/codexsetup --http-probe-off true 55 | ``` 56 | 3. This should output an image with name `status-im/codexsetup.slim` 57 | 4. We can then bring up the image using `docker-compose up -d`. 58 | -------------------------------------------------------------------------------- /docker/codex.Dockerfile: -------------------------------------------------------------------------------- 1 | # Variables 2 | ARG BUILDER=ubuntu:24.04 3 | ARG IMAGE=${BUILDER} 4 | ARG RUST_VERSION=${RUST_VERSION:-1.79.0} 5 | ARG BUILD_HOME=/src 6 | ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4} 7 | ARG NIMFLAGS="${NIMFLAGS:-"-d:disableMarchNative"}" 8 | ARG USE_LIBBACKTRACE=${USE_LIBBACKTRACE:-1} 9 | ARG APP_HOME=/codex 10 | ARG NAT_IP_AUTO=${NAT_IP_AUTO:-false} 11 | 12 | # Build 13 | FROM ${BUILDER} AS builder 14 | ARG RUST_VERSION 15 | ARG BUILD_HOME 16 | ARG MAKE_PARALLEL 17 | ARG NIMFLAGS 18 | ARG USE_LIBBACKTRACE 19 | 20 | RUN apt-get update && apt-get install -y git cmake curl make bash lcov build-essential 21 | RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs/ | sh -s -- --default-toolchain=${RUST_VERSION} -y 22 | 23 | SHELL ["/bin/bash", "-c"] 24 | ENV BASH_ENV="/etc/bash_env" 25 | RUN echo "export PATH=$PATH:$HOME/.cargo/bin" >> $BASH_ENV 26 | 27 | WORKDIR ${BUILD_HOME} 28 | COPY . . 29 | RUN make -j ${MAKE_PARALLEL} update 30 | RUN make -j ${MAKE_PARALLEL} 31 | RUN make -j ${MAKE_PARALLEL} cirdl 32 | 33 | # Create 34 | FROM ${IMAGE} 35 | ARG BUILD_HOME 36 | ARG APP_HOME 37 | ARG NAT_IP_AUTO 38 | 39 | WORKDIR ${APP_HOME} 40 | COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin 41 | COPY --from=builder ${BUILD_HOME}/openapi.yaml . 42 | COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh / 43 | RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/* 44 | ENV NAT_IP_AUTO=${NAT_IP_AUTO} 45 | ENTRYPOINT ["/docker-entrypoint.sh"] 46 | CMD ["codex"] 47 | -------------------------------------------------------------------------------- /docker/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | codex-node1: 3 | image: codexstorage/nim-codex:sha-82b0399 4 | environment: 5 | - CODEX_LOG_LEVEL=${CODEX_LOG_LEVEL:-TRACE} 6 | - CODEX_METRICS=${CODEX_METRICS:-false} 7 | - CODEX_METRICS_ADDRESS=${CODEX_METRICS_ADDRESS:-0.0.0.0} 8 | - CODEX_METRICS_PORT=${CODEX_METRICS_PORT:-8008} 9 | - CODEX_DATA_DIR=${CODEX_DATA_DIR:-/datadir} 10 | - CODEX_LISTEN_ADDRS=${CODEX_LISTEN_ADDRS:-/ip4/0.0.0.0/tcp/2345} 11 | - CODEX_NAT=${CODEX_NAT:-10.0.0.10} 12 | - CODEX_DISC_PORT=${CODEX_DISC_PORT:-8090} 13 | - CODEX_NET_PRIVKEY=${CODEX_NET_PRIVKEY:-key} 14 | # - CODEX_BOOTSTRAP_NODE=${CODEX_BOOTSTRAP_NODE} 15 | - CODEX_MAX_PEERS=${CODEX_MAX_PEERS:-160} 16 | - CODEX_AGENT_STRING=${CODEX_AGENT_STRING:-Codex} 17 | - CODEX_API_BINDADDR=${CODEX_API_BINDADDR:-0.0.0.0} 18 | - CODEX_API_PORT=${CODEX_API_PORT:-8080} 19 | - CODEX_REPO_KIND=${CODEX_REPO_KIND:-fs} 20 | - CODEX_STORAGE_QUOTA=${CODEX_STORAGE_QUOTA:-8589934592} 21 | - CODEX_BLOCK_TTL=${CODEX_BLOCK_TTL:-0} 22 | # - CODEX_BLOCK_MI=${CODEX_BLOCK_MI} 23 | - CODEX_BLOCK_MN=${CODEX_BLOCK_MN:-1000} 24 | - CODEX_CACHE_SIZE=${CODEX_CACHE_SIZE:-0} 25 | - CODEX_PERSISTENCE=${CODEX_PERSISTENCE:-false} 26 | - CODEX_ETH_PROVIDER=${CODEX_ETH_PROVIDER:-ws://localhost:8545} 27 | # - CODEX_ETH_ACCOUNT=${CODEX_ETH_ACCOUNT} 28 | # - CODEX_MARKETPLACE_ADDRESS=${CODEX_MARKETPLACE_ADDRESS:-0x59b670e9fA9D0A427751Af201D676719a970857b} 29 | - CODEX_VALIDATOR=${CODEX_VALIDATOR:-false} 30 | - CODEX_VALIDATOR_MAX_SLOTS=${CODEX_VALIDATOR_MAX_SLOTS:-1000} 31 | - NAT_IP_AUTO=false 32 | - NAT_PUBLIC_IP_AUTO=https://ipinfo.io/ip 33 | ports: 34 | - 8080:8080/tcp # REST API 35 | - 8008:8008/tcp # Metrics 36 | - 2345:2345/tcp # libp2p 37 | - 8090:8090/udp # DHT discovery 38 | volumes: 39 | - ./datadir:/datadir:z 40 | networks: 41 | - codex 42 | networks: 43 | codex: 44 | driver: bridge 45 | -------------------------------------------------------------------------------- /env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file 4 | # and we fall back to a Zsh-specific special var to also support Zsh. 5 | REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})" 6 | ABS_PATH="$(cd ${REL_PATH}; pwd)" 7 | source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh 8 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "circom-compat": { 4 | "inputs": { 5 | "nixpkgs": [ 6 | "nixpkgs" 7 | ] 8 | }, 9 | "locked": { 10 | "lastModified": 1736521871, 11 | "narHash": "sha256-d34XNLg9NGPEOARHW+BIOAWalkHdEUAwsv3mpLZQxds=", 12 | "owner": "codex-storage", 13 | "repo": "circom-compat-ffi", 14 | "rev": "8cd4ed44fdafe59d4ec1184420639cae4c4dbab9", 15 | "type": "github" 16 | }, 17 | "original": { 18 | "owner": "codex-storage", 19 | "repo": "circom-compat-ffi", 20 | "type": "github" 21 | } 22 | }, 23 | "nixpkgs": { 24 | "locked": { 25 | "lastModified": 1736200483, 26 | "narHash": "sha256-JO+lFN2HsCwSLMUWXHeOad6QUxOuwe9UOAF/iSl1J4I=", 27 | "owner": "NixOS", 28 | "repo": "nixpkgs", 29 | "rev": "3f0a8ac25fb674611b98089ca3a5dd6480175751", 30 | "type": "github" 31 | }, 32 | "original": { 33 | "owner": "NixOS", 34 | "ref": "nixos-24.11", 35 | "repo": "nixpkgs", 36 | "type": "github" 37 | } 38 | }, 39 | "root": { 40 | "inputs": { 41 | "circom-compat": "circom-compat", 42 | "nixpkgs": "nixpkgs" 43 | } 44 | } 45 | }, 46 | "root": "root", 47 | "version": 7 48 | } 49 | -------------------------------------------------------------------------------- /metrics/assets/import.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codex-storage/nim-codex/3e17207a0b703a66001e7bd8e0bf97d226c119e5/metrics/assets/import.png -------------------------------------------------------------------------------- /metrics/assets/imported.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codex-storage/nim-codex/3e17207a0b703a66001e7bd8e0bf97d226c119e5/metrics/assets/imported.png -------------------------------------------------------------------------------- /metrics/assets/main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codex-storage/nim-codex/3e17207a0b703a66001e7bd8e0bf97d226c119e5/metrics/assets/main.png -------------------------------------------------------------------------------- /metrics/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 12s 3 | 4 | scrape_configs: 5 | - job_name: "codex" 6 | static_configs: 7 | - targets: ['127.0.0.1:8008'] 8 | - job_name: "node_exporter" 9 | static_configs: 10 | - targets: ['127.0.0.1:9100'] 11 | -------------------------------------------------------------------------------- /nix/README.md: -------------------------------------------------------------------------------- 1 | # Usage 2 | 3 | ## Shell 4 | 5 | A development shell can be started using: 6 | ```sh 7 | nix develop '.?submodules=1#' 8 | ``` 9 | 10 | ## Building 11 | 12 | To build a Codex you can use: 13 | ```sh 14 | nix build '.?submodules=1#default' 15 | ``` 16 | The `?submodules=1` part should eventually not be necessary. 17 | For more details see: 18 | https://github.com/NixOS/nix/issues/4423 19 | 20 | It can be also done without even cloning the repo: 21 | ```sh 22 | nix build 'git+https://github.com/codex-storage/nim-codex?submodules=1#' 23 | ``` 24 | 25 | ## Running 26 | 27 | ```sh 28 | nix run 'git+https://github.com/codex-storage/nim-codex?submodules=1#'' 29 | ``` 30 | 31 | ## Testing 32 | 33 | ```sh 34 | nix flake check ".?submodules=1#" 35 | ``` 36 | 37 | ## Running Nim-Codex as a service on NixOS 38 | 39 | Include nim-codex flake in your flake inputs: 40 | ```nix 41 | inputs = { 42 | nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; 43 | nim-codex-flake.url = "git+https://github.com/codex-storage/nim-codex?submodules=1#"; 44 | }; 45 | ``` 46 | 47 | To configure the service, you can use the following example: 48 | ```nix 49 | services.nim-codex = { 50 | enable = true; 51 | settings = { 52 | data-dir = "/var/lib/codex-test"; 53 | }; 54 | }; 55 | ``` 56 | The settings attribute set corresponds directly to the layout of the TOML configuration file 57 | used by nim-codex. Each option follows the same naming convention as the CLI flags, but 58 | with the -- prefix removed. For more details on the TOML file structure and options, 59 | refer to the official documentation: [nim-codex configuration file](https://docs.codex.storage/learn/run#configuration-file). -------------------------------------------------------------------------------- /nix/checksums.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import { } }: 2 | 3 | let 4 | tools = pkgs.callPackage ./tools.nix {}; 5 | sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; 6 | in pkgs.fetchFromGitHub { 7 | owner = "nim-lang"; 8 | repo = "checksums"; 9 | rev = tools.findKeyValue "^ +ChecksumsStableCommit = \"([a-f0-9]+)\"$" sourceFile; 10 | # WARNING: Requires manual updates when Nim compiler version changes. 11 | hash = "sha256-Bm5iJoT2kAvcTexiLMFBa9oU5gf7d4rWjo3OiN7obWQ="; 12 | } 13 | -------------------------------------------------------------------------------- /nix/csources.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import { } }: 2 | 3 | let 4 | tools = pkgs.callPackage ./tools.nix {}; 5 | sourceFile = ../vendor/nimbus-build-system/vendor/Nim/config/build_config.txt; 6 | in pkgs.fetchFromGitHub { 7 | owner = "nim-lang"; 8 | repo = "csources_v2"; 9 | rev = tools.findKeyValue "^nim_csourcesHash=([a-f0-9]+)$" sourceFile; 10 | # WARNING: Requires manual updates when Nim compiler version changes. 11 | hash = "sha256-UCLtoxOcGYjBdvHx7A47x6FjLMi6VZqpSs65MN7fpBs="; 12 | } 13 | -------------------------------------------------------------------------------- /nix/nimble.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import { } }: 2 | 3 | let 4 | tools = pkgs.callPackage ./tools.nix {}; 5 | sourceFile = ../vendor/nimbus-build-system/vendor/Nim/koch.nim; 6 | in pkgs.fetchFromGitHub { 7 | owner = "nim-lang"; 8 | repo = "nimble"; 9 | fetchSubmodules = true; 10 | rev = tools.findKeyValue "^ +NimbleStableCommit = \"([a-f0-9]+)\".+" sourceFile; 11 | # WARNING: Requires manual updates when Nim compiler version changes. 12 | hash = "sha256-Rz48sGUKZEAp+UySla+MlsOfsERekuGKw69Tm11fDz8="; 13 | } 14 | -------------------------------------------------------------------------------- /nix/service.nix: -------------------------------------------------------------------------------- 1 | { self, config, lib, pkgs, circomCompatPkg, ... }: 2 | 3 | let 4 | inherit (lib) 5 | types mkEnableOption mkOption mkIf literalExpression 6 | mdDoc; 7 | 8 | toml = pkgs.formats.toml { }; 9 | 10 | cfg = config.services.nim-codex; 11 | in 12 | { 13 | options = { 14 | services.nim-codex = { 15 | enable = mkEnableOption "Nim Codex Node service."; 16 | 17 | package = mkOption { 18 | type = types.package; 19 | default = pkgs.callPackage ./default.nix { src = self; inherit circomCompatPkg; }; 20 | defaultText = literalExpression "pkgs.codex"; 21 | description = mdDoc "Package to use as Nim Codex node."; 22 | }; 23 | 24 | settings = mkOption { 25 | default = { }; 26 | type = toml.type; 27 | description = ''Structured settings object that will be used to generate a TOML config file.''; 28 | }; 29 | }; 30 | }; 31 | 32 | config = mkIf cfg.enable { 33 | environment.etc = { 34 | "nim-codex/config.toml".source = toml.generate "config.toml" cfg.settings; 35 | }; 36 | systemd.services.nim-codex = { 37 | description = "Nim Codex Node"; 38 | wantedBy = [ "multi-user.target" ]; 39 | requires = [ "network.target" ]; 40 | serviceConfig = { 41 | DynamicUser = true; 42 | PrivateTmp = true; 43 | ProtectHome = true; 44 | ProtectSystem = "full"; 45 | NoNewPrivileges = true; 46 | PrivateDevices = true; 47 | MemoryDenyWriteExecute = true; 48 | ExecStart = "${cfg.package}/bin/codex --config-file=/etc/nim-codex/config.toml"; 49 | Restart = "on-failure"; 50 | }; 51 | restartIfChanged = true; 52 | restartTriggers = [ 53 | "/etc/nim-codex/config.toml" 54 | ]; 55 | }; 56 | }; 57 | } -------------------------------------------------------------------------------- /nix/tools.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import { } }: 2 | 3 | let 4 | 5 | inherit (pkgs.lib) fileContents last splitString flatten remove; 6 | inherit (builtins) map match; 7 | in { 8 | findKeyValue = regex: sourceFile: 9 | let 10 | linesFrom = file: splitString "\n" (fileContents file); 11 | matching = regex: lines: map (line: match regex line) lines; 12 | extractMatch = matches: last (flatten (remove null matches)); 13 | in 14 | extractMatch (matching regex (linesFrom sourceFile)); 15 | } 16 | -------------------------------------------------------------------------------- /redocly.yaml: -------------------------------------------------------------------------------- 1 | extends: 2 | - recommended 3 | 4 | rules: 5 | info-license: off 6 | no-required-schema-properties-undefined: error 7 | no-server-example.com: off -------------------------------------------------------------------------------- /tests/asynctest.nim: -------------------------------------------------------------------------------- 1 | import pkg/asynctest/chronos/unittest2 2 | 3 | export unittest2 4 | -------------------------------------------------------------------------------- /tests/checktest.nim: -------------------------------------------------------------------------------- 1 | import ./helpers 2 | 3 | ## Unit testing suite that calls checkTrackers in teardown to check for memory leaks using chronos trackers. 4 | template checksuite*(name, body) = 5 | suite name: 6 | proc suiteProc() = 7 | multisetup() 8 | 9 | teardown: 10 | checkTrackers() 11 | 12 | body 13 | 14 | suiteProc() 15 | 16 | template asyncchecksuite*(name, body) = 17 | suite name: 18 | proc suiteProc() = 19 | asyncmultisetup() 20 | 21 | teardown: 22 | checkTrackers() 23 | 24 | body 25 | 26 | suiteProc() 27 | 28 | export helpers 29 | -------------------------------------------------------------------------------- /tests/circuits/fixtures/proof_main.r1cs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codex-storage/nim-codex/3e17207a0b703a66001e7bd8e0bf97d226c119e5/tests/circuits/fixtures/proof_main.r1cs -------------------------------------------------------------------------------- /tests/circuits/fixtures/proof_main.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codex-storage/nim-codex/3e17207a0b703a66001e7bd8e0bf97d226c119e5/tests/circuits/fixtures/proof_main.wasm -------------------------------------------------------------------------------- /tests/circuits/fixtures/proof_main.zkey: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codex-storage/nim-codex/3e17207a0b703a66001e7bd8e0bf97d226c119e5/tests/circuits/fixtures/proof_main.zkey -------------------------------------------------------------------------------- /tests/codex/blockexchange/engine/testpayments.nim: -------------------------------------------------------------------------------- 1 | import pkg/unittest2 2 | 3 | import pkg/codex/stores 4 | import ../../examples 5 | import ../../helpers 6 | 7 | suite "Engine payments": 8 | let address = EthAddress.example 9 | let amount = 42.u256 10 | 11 | var wallet: WalletRef 12 | var peer: BlockExcPeerCtx 13 | 14 | setup: 15 | wallet = WalletRef.example 16 | peer = BlockExcPeerCtx.example 17 | peer.account = Account(address: address).some 18 | 19 | test "pays for received blocks": 20 | let payment = !wallet.pay(peer, amount) 21 | let balances = payment.state.outcome.balances(Asset) 22 | let destination = address.toDestination 23 | check !balances[destination] == amount 24 | 25 | test "no payment when no account is set": 26 | peer.account = Account.none 27 | check wallet.pay(peer, amount).isFailure 28 | 29 | test "uses same channel for consecutive payments": 30 | let payment1, payment2 = wallet.pay(peer, amount) 31 | let channel1 = payment1 .? state .? channel .? getChannelId 32 | let channel2 = payment2 .? state .? channel .? getChannelId 33 | check channel1 == channel2 34 | -------------------------------------------------------------------------------- /tests/codex/blockexchange/protobuf/testpayments.nim: -------------------------------------------------------------------------------- 1 | import pkg/chronos 2 | import pkg/stew/byteutils 3 | import pkg/codex/stores 4 | 5 | import ../../../asynctest 6 | import ../../examples 7 | import ../../helpers 8 | 9 | suite "account protobuf messages": 10 | let account = Account(address: EthAddress.example) 11 | let message = AccountMessage.init(account) 12 | 13 | test "encodes recipient of payments": 14 | check message.address == @(account.address.toArray) 15 | 16 | test "decodes recipient of payments": 17 | check Account.init(message) .? address == account.address.some 18 | 19 | test "fails to decode when address has incorrect number of bytes": 20 | var incorrect = message 21 | incorrect.address.del(0) 22 | check Account.init(incorrect).isNone 23 | 24 | suite "channel update messages": 25 | let state = SignedState.example 26 | let update = StateChannelUpdate.init(state) 27 | 28 | test "encodes a nitro signed state": 29 | check update.update == state.toJson.toBytes 30 | 31 | test "decodes a channel update": 32 | check SignedState.init(update) == state.some 33 | 34 | test "fails to decode incorrect channel update": 35 | var incorrect = update 36 | incorrect.update.del(0) 37 | check SignedState.init(incorrect).isNone 38 | -------------------------------------------------------------------------------- /tests/codex/blockexchange/protobuf/testpresence.nim: -------------------------------------------------------------------------------- 1 | import pkg/chronos 2 | 3 | import pkg/codex/blockexchange/protobuf/presence 4 | 5 | import ../../../asynctest 6 | import ../../examples 7 | import ../../helpers 8 | 9 | suite "block presence protobuf messages": 10 | let 11 | cid = Cid.example 12 | address = BlockAddress(leaf: false, cid: cid) 13 | price = UInt256.example 14 | presence = Presence(address: address, have: true, price: price) 15 | message = PresenceMessage.init(presence) 16 | 17 | test "encodes have/donthave": 18 | var presence = presence 19 | presence.have = true 20 | check PresenceMessage.init(presence).`type` == Have 21 | presence.have = false 22 | check PresenceMessage.init(presence).`type` == DontHave 23 | 24 | test "encodes price": 25 | check message.price == @(price.toBytesBE) 26 | 27 | test "decodes CID": 28 | check Presence.init(message) .? address == address.some 29 | 30 | test "decodes have/donthave": 31 | var message = message 32 | message.`type` = BlockPresenceType.Have 33 | check Presence.init(message) .? have == true.some 34 | message.`type` = BlockPresenceType.DontHave 35 | check Presence.init(message) .? have == false.some 36 | 37 | test "decodes price": 38 | check Presence.init(message) .? price == price.some 39 | 40 | test "fails to decode when price is invalid": 41 | var incorrect = message 42 | incorrect.price.add(0) 43 | check Presence.init(incorrect).isNone 44 | -------------------------------------------------------------------------------- /tests/codex/blockexchange/testdiscovery.nim: -------------------------------------------------------------------------------- 1 | import ./discovery/testdiscovery 2 | import ./discovery/testdiscoveryengine 3 | 4 | {.warning[UnusedImport]: off.} 5 | -------------------------------------------------------------------------------- /tests/codex/blockexchange/testengine.nim: -------------------------------------------------------------------------------- 1 | import ./engine/testengine 2 | import ./engine/testblockexc 3 | import ./engine/testpayments 4 | import ./engine/testadvertiser 5 | 6 | {.warning[UnusedImport]: off.} 7 | -------------------------------------------------------------------------------- /tests/codex/blockexchange/testprotobuf.nim: -------------------------------------------------------------------------------- 1 | import ./protobuf/testpayments 2 | import ./protobuf/testpresence 3 | 4 | {.warning[UnusedImport]: off.} 5 | -------------------------------------------------------------------------------- /tests/codex/helpers/always.nim: -------------------------------------------------------------------------------- 1 | import pkg/chronos 2 | 3 | template always*(condition: untyped, timeout = 50.millis): bool = 4 | proc loop(): Future[bool] {.async.} = 5 | let start = Moment.now() 6 | while true: 7 | if not condition: 8 | return false 9 | if Moment.now() > (start + timeout): 10 | return true 11 | else: 12 | await sleepAsync(1.millis) 13 | 14 | await loop() 15 | -------------------------------------------------------------------------------- /tests/codex/helpers/mockchunker.nim: -------------------------------------------------------------------------------- 1 | import pkg/chronos 2 | import pkg/codex/chunker 3 | 4 | export chunker 5 | 6 | type MockChunker* = Chunker 7 | 8 | proc new*( 9 | T: type MockChunker, 10 | dataset: openArray[byte], 11 | chunkSize: int | NBytes, 12 | pad: bool = false, 13 | ): MockChunker = 14 | ## Create a chunker that produces data 15 | ## 16 | 17 | let 18 | chunkSize = chunkSize.NBytes 19 | dataset = @dataset 20 | 21 | var consumed = 0 22 | proc reader( 23 | data: ChunkBuffer, len: int 24 | ): Future[int] {.gcsafe, async: (raises: [ChunkerError, CancelledError]).} = 25 | if consumed >= dataset.len: 26 | return 0 27 | 28 | var read = 0 29 | while read < len and read < chunkSize.int and (consumed + read) < dataset.len: 30 | data[read] = dataset[consumed + read] 31 | read.inc 32 | 33 | consumed += read 34 | return read 35 | 36 | Chunker.new(reader = reader, pad = pad, chunkSize = chunkSize) 37 | -------------------------------------------------------------------------------- /tests/codex/helpers/mockclock.nim: -------------------------------------------------------------------------------- 1 | import std/times 2 | import pkg/chronos 3 | import codex/clock 4 | 5 | export clock 6 | 7 | type 8 | MockClock* = ref object of Clock 9 | time: SecondsSince1970 10 | waiting: seq[Waiting] 11 | 12 | Waiting = ref object 13 | until: SecondsSince1970 14 | future: Future[void] 15 | 16 | func new*(_: type MockClock, time: SecondsSince1970 = getTime().toUnix): MockClock = 17 | ## Create a mock clock instance 18 | MockClock(time: time) 19 | 20 | proc set*(clock: MockClock, time: SecondsSince1970) = 21 | clock.time = time 22 | var index = 0 23 | while index < clock.waiting.len: 24 | if clock.waiting[index].until <= clock.time: 25 | clock.waiting[index].future.complete() 26 | clock.waiting.del(index) 27 | else: 28 | inc index 29 | 30 | proc advance*(clock: MockClock, seconds: int64) = 31 | clock.set(clock.time + seconds) 32 | 33 | method now*(clock: MockClock): SecondsSince1970 = 34 | clock.time 35 | 36 | method waitUntil*(clock: MockClock, time: SecondsSince1970) {.async.} = 37 | if time > clock.now(): 38 | let future = newFuture[void]() 39 | clock.waiting.add(Waiting(until: time, future: future)) 40 | await future 41 | 42 | proc isWaiting*(clock: MockClock): bool = 43 | clock.waiting.len > 0 44 | -------------------------------------------------------------------------------- /tests/codex/helpers/mockdiscovery.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2022 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import pkg/chronos 11 | import pkg/libp2p 12 | import pkg/questionable 13 | import pkg/codex/discovery 14 | import pkg/contractabi/address as ca 15 | 16 | type MockDiscovery* = ref object of Discovery 17 | findBlockProvidersHandler*: proc( 18 | d: MockDiscovery, cid: Cid 19 | ): Future[seq[SignedPeerRecord]] {.async: (raises: [CancelledError]).} 20 | 21 | publishBlockProvideHandler*: 22 | proc(d: MockDiscovery, cid: Cid): Future[void] {.async: (raises: [CancelledError]).} 23 | 24 | findHostProvidersHandler*: proc( 25 | d: MockDiscovery, host: ca.Address 26 | ): Future[seq[SignedPeerRecord]] {.async: (raises: [CancelledError]).} 27 | 28 | publishHostProvideHandler*: proc(d: MockDiscovery, host: ca.Address): Future[void] {. 29 | async: (raises: [CancelledError]) 30 | .} 31 | 32 | proc new*(T: type MockDiscovery): MockDiscovery = 33 | MockDiscovery() 34 | 35 | proc findPeer*( 36 | d: Discovery, peerId: PeerId 37 | ): Future[?PeerRecord] {.async: (raises: [CancelledError]).} = 38 | ## mock find a peer - always return none 39 | ## 40 | return none(PeerRecord) 41 | 42 | method find*( 43 | d: MockDiscovery, cid: Cid 44 | ): Future[seq[SignedPeerRecord]] {.async: (raises: [CancelledError]).} = 45 | if isNil(d.findBlockProvidersHandler): 46 | return 47 | 48 | return await d.findBlockProvidersHandler(d, cid) 49 | 50 | method provide*( 51 | d: MockDiscovery, cid: Cid 52 | ): Future[void] {.async: (raises: [CancelledError]).} = 53 | if isNil(d.publishBlockProvideHandler): 54 | return 55 | 56 | await d.publishBlockProvideHandler(d, cid) 57 | 58 | method find*( 59 | d: MockDiscovery, host: ca.Address 60 | ): Future[seq[SignedPeerRecord]] {.async: (raises: [CancelledError]).} = 61 | if isNil(d.findHostProvidersHandler): 62 | return 63 | 64 | return await d.findHostProvidersHandler(d, host) 65 | 66 | method provide*( 67 | d: MockDiscovery, host: ca.Address 68 | ): Future[void] {.async: (raises: [CancelledError]).} = 69 | if isNil(d.publishHostProvideHandler): 70 | return 71 | 72 | await d.publishHostProvideHandler(d, host) 73 | -------------------------------------------------------------------------------- /tests/codex/helpers/mockrepostore.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2023 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import std/sequtils 11 | import pkg/chronos 12 | import pkg/libp2p 13 | import pkg/questionable 14 | import pkg/questionable/results 15 | 16 | import pkg/codex/stores/repostore 17 | import pkg/codex/utils/asynciter 18 | import pkg/codex/utils/safeasynciter 19 | 20 | type MockRepoStore* = ref object of RepoStore 21 | delBlockCids*: seq[Cid] 22 | getBeMaxNumber*: int 23 | getBeOffset*: int 24 | 25 | testBlockExpirations*: seq[BlockExpiration] 26 | 27 | method delBlock*( 28 | self: MockRepoStore, cid: Cid 29 | ): Future[?!void] {.async: (raises: [CancelledError]).} = 30 | self.delBlockCids.add(cid) 31 | self.testBlockExpirations = self.testBlockExpirations.filterIt(it.cid != cid) 32 | return success() 33 | 34 | method getBlockExpirations*( 35 | self: MockRepoStore, maxNumber: int, offset: int 36 | ): Future[?!SafeAsyncIter[BlockExpiration]] {.async: (raises: [CancelledError]).} = 37 | self.getBeMaxNumber = maxNumber 38 | self.getBeOffset = offset 39 | 40 | let 41 | testBlockExpirationsCpy = @(self.testBlockExpirations) 42 | limit = min(offset + maxNumber, len(testBlockExpirationsCpy)) 43 | 44 | let 45 | iter1 = SafeAsyncIter[int].new(offset ..< limit) 46 | iter2 = map[int, BlockExpiration]( 47 | iter1, 48 | proc(i: ?!int): Future[?!BlockExpiration] {.async: (raises: [CancelledError]).} = 49 | if i =? i: 50 | return success(testBlockExpirationsCpy[i]) 51 | return failure("Unexpected error!"), 52 | ) 53 | 54 | success(iter2) 55 | -------------------------------------------------------------------------------- /tests/codex/helpers/mockreservations.nim: -------------------------------------------------------------------------------- 1 | import pkg/chronos 2 | import pkg/codex/sales 3 | import pkg/codex/stores 4 | import pkg/questionable/results 5 | import pkg/codex/clock 6 | 7 | type MockReservations* = ref object of Reservations 8 | createReservationThrowBytesOutOfBoundsError: bool 9 | createReservationThrowError: ?(ref CatchableError) 10 | 11 | proc new*(T: type MockReservations, repo: RepoStore): MockReservations = 12 | ## Create a mock clock instance 13 | MockReservations(availabilityLock: newAsyncLock(), repo: repo) 14 | 15 | proc setCreateReservationThrowBytesOutOfBoundsError*( 16 | self: MockReservations, flag: bool 17 | ) = 18 | self.createReservationThrowBytesOutOfBoundsError = flag 19 | 20 | proc setCreateReservationThrowError*( 21 | self: MockReservations, error: ?(ref CatchableError) 22 | ) = 23 | self.createReservationThrowError = error 24 | 25 | method createReservation*( 26 | self: MockReservations, 27 | availabilityId: AvailabilityId, 28 | slotSize: uint64, 29 | requestId: RequestId, 30 | slotIndex: uint64, 31 | collateralPerByte: UInt256, 32 | validUntil: SecondsSince1970, 33 | ): Future[?!Reservation] {.async: (raises: [CancelledError]).} = 34 | if self.createReservationThrowBytesOutOfBoundsError: 35 | let error = newException( 36 | BytesOutOfBoundsError, 37 | "trying to reserve an amount of bytes that is greater than the total size of the Availability", 38 | ) 39 | return failure(error) 40 | elif error =? self.createReservationThrowError: 41 | return failure(error) 42 | 43 | return await procCall createReservation( 44 | Reservations(self), 45 | availabilityId, 46 | slotSize, 47 | requestId, 48 | slotIndex, 49 | collateralPerByte, 50 | validUntil, 51 | ) 52 | -------------------------------------------------------------------------------- /tests/codex/helpers/mocksalesagent.nim: -------------------------------------------------------------------------------- 1 | import pkg/codex/sales/salesagent 2 | 3 | type MockSalesAgent = ref object of SalesAgent 4 | fulfilledCalled*: bool 5 | failedCalled*: bool 6 | slotFilledCalled*: bool 7 | 8 | method onFulfilled*(agent: SalesAgent, requestId: RequestId) = 9 | fulfilledCalled = true 10 | 11 | method onFailed*(agent: SalesAgent, requestId: RequestId) = 12 | failedCalled = true 13 | 14 | method onSlotFilled*( 15 | agent: SalesAgent, requestId: RequestId, slotIndex: uint64 16 | ) {.base.} = 17 | slotFilledCalled = true 18 | -------------------------------------------------------------------------------- /tests/codex/helpers/mockslotqueueitem.nim: -------------------------------------------------------------------------------- 1 | import pkg/codex/contracts/requests 2 | import pkg/codex/sales/slotqueue 3 | 4 | type MockSlotQueueItem* = object 5 | requestId*: RequestId 6 | slotIndex*: uint16 7 | slotSize*: uint64 8 | duration*: uint64 9 | pricePerBytePerSecond*: UInt256 10 | collateral*: UInt256 11 | expiry*: uint64 12 | seen*: bool 13 | 14 | proc toSlotQueueItem*(item: MockSlotQueueItem): SlotQueueItem = 15 | SlotQueueItem.init( 16 | requestId = item.requestId, 17 | slotIndex = item.slotIndex, 18 | ask = StorageAsk( 19 | slotSize: item.slotSize, 20 | duration: item.duration, 21 | pricePerBytePerSecond: item.pricePerBytePerSecond, 22 | ), 23 | expiry = item.expiry, 24 | seen = item.seen, 25 | collateral = item.collateral, 26 | ) 27 | -------------------------------------------------------------------------------- /tests/codex/helpers/mocktimer.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2023 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import pkg/chronos 11 | 12 | import codex/utils/timer 13 | 14 | type MockTimer* = ref object of Timer 15 | startCalled*: int 16 | stopCalled*: int 17 | mockInterval*: Duration 18 | mockCallback: timer.TimerCallback 19 | 20 | proc new*(T: type MockTimer): MockTimer = 21 | ## Create a mocked Timer instance 22 | MockTimer(startCalled: 0, stopCalled: 0) 23 | 24 | method start*(mockTimer: MockTimer, callback: timer.TimerCallback, interval: Duration) = 25 | mockTimer.mockCallback = callback 26 | mockTimer.mockInterval = interval 27 | inc mockTimer.startCalled 28 | 29 | method stop*(mockTimer: MockTimer) {.async: (raises: []).} = 30 | inc mockTimer.stopCalled 31 | 32 | method invokeCallback*(mockTimer: MockTimer) {.async, base.} = 33 | await mockTimer.mockCallback() 34 | -------------------------------------------------------------------------------- /tests/codex/helpers/nodeutils.nim: -------------------------------------------------------------------------------- 1 | import std/sequtils 2 | 3 | import pkg/chronos 4 | import pkg/libp2p 5 | import pkg/libp2p/errors 6 | 7 | import pkg/codex/discovery 8 | import pkg/codex/stores 9 | import pkg/codex/blocktype as bt 10 | import pkg/codex/blockexchange 11 | 12 | import ../examples 13 | 14 | type NodesComponents* = 15 | tuple[ 16 | switch: Switch, 17 | blockDiscovery: Discovery, 18 | wallet: WalletRef, 19 | network: BlockExcNetwork, 20 | localStore: BlockStore, 21 | peerStore: PeerCtxStore, 22 | pendingBlocks: PendingBlocksManager, 23 | discovery: DiscoveryEngine, 24 | engine: BlockExcEngine, 25 | networkStore: NetworkStore, 26 | ] 27 | 28 | proc generateNodes*( 29 | num: Natural, blocks: openArray[bt.Block] = [] 30 | ): seq[NodesComponents] = 31 | for i in 0 ..< num: 32 | let 33 | switch = newStandardSwitch(transportFlags = {ServerFlags.ReuseAddr}) 34 | discovery = Discovery.new( 35 | switch.peerInfo.privateKey, 36 | announceAddrs = 37 | @[ 38 | MultiAddress.init("/ip4/127.0.0.1/tcp/0").expect( 39 | "Should return multiaddress" 40 | ) 41 | ], 42 | ) 43 | wallet = WalletRef.example 44 | network = BlockExcNetwork.new(switch) 45 | localStore = CacheStore.new(blocks.mapIt(it)) 46 | peerStore = PeerCtxStore.new() 47 | pendingBlocks = PendingBlocksManager.new() 48 | advertiser = Advertiser.new(localStore, discovery) 49 | blockDiscovery = 50 | DiscoveryEngine.new(localStore, peerStore, network, discovery, pendingBlocks) 51 | engine = BlockExcEngine.new( 52 | localStore, wallet, network, blockDiscovery, advertiser, peerStore, 53 | pendingBlocks, 54 | ) 55 | networkStore = NetworkStore.new(engine, localStore) 56 | 57 | switch.mount(network) 58 | 59 | let nc: NodesComponents = ( 60 | switch, discovery, wallet, network, localStore, peerStore, pendingBlocks, 61 | blockDiscovery, engine, networkStore, 62 | ) 63 | 64 | result.add(nc) 65 | 66 | proc connectNodes*(nodes: seq[Switch]) {.async.} = 67 | for dialer in nodes: 68 | for node in nodes: 69 | if dialer.peerInfo.peerId != node.peerInfo.peerId: 70 | await dialer.connect(node.peerInfo.peerId, node.peerInfo.addrs) 71 | 72 | proc connectNodes*(nodes: seq[NodesComponents]) {.async.} = 73 | await connectNodes(nodes.mapIt(it.switch)) 74 | -------------------------------------------------------------------------------- /tests/codex/helpers/randomchunker.nim: -------------------------------------------------------------------------------- 1 | import std/sequtils 2 | 3 | import pkg/chronos 4 | 5 | import pkg/codex/chunker 6 | import pkg/codex/rng 7 | 8 | export chunker 9 | 10 | type RandomChunker* = Chunker 11 | 12 | proc new*( 13 | T: type RandomChunker, 14 | rng: Rng, 15 | chunkSize: int | NBytes, 16 | size: int | NBytes, 17 | pad = false, 18 | ): RandomChunker = 19 | ## Create a chunker that produces random data 20 | ## 21 | 22 | let 23 | size = size.int 24 | chunkSize = chunkSize.NBytes 25 | 26 | var consumed = 0 27 | proc reader( 28 | data: ChunkBuffer, len: int 29 | ): Future[int] {.async: (raises: [ChunkerError, CancelledError]), gcsafe.} = 30 | var alpha = toSeq(byte('A') .. byte('z')) 31 | 32 | if consumed >= size: 33 | return 0 34 | 35 | var read = 0 36 | while read < len and (pad or read < size - consumed): 37 | rng.shuffle(alpha) 38 | for a in alpha: 39 | if read >= len or (not pad and read >= size - consumed): 40 | break 41 | 42 | data[read] = a 43 | read.inc 44 | 45 | consumed += read 46 | return read 47 | 48 | Chunker.new(reader = reader, pad = pad, chunkSize = chunkSize) 49 | -------------------------------------------------------------------------------- /tests/codex/merkletree/helpers.nim: -------------------------------------------------------------------------------- 1 | import pkg/constantine/platforms/abstractions 2 | 3 | import pkg/codex/merkletree 4 | import ../helpers 5 | 6 | export merkletree, helpers 7 | 8 | converter toBool*(x: CTBool): bool = 9 | bool(x) 10 | 11 | proc `==`*(a, b: Poseidon2Tree): bool = 12 | (a.leavesCount == b.leavesCount) and (a.levels == b.levels) and (a.layers == b.layers) 13 | 14 | proc `==`*(a, b: Poseidon2Proof): bool = 15 | (a.nleaves == b.nleaves) and (a.index == b.index) and (a.path.len == b.path.len) and 16 | (a.path == b.path) 17 | 18 | proc `==`*(a, b: CodexTree): bool = 19 | (a.mcodec == b.mcodec) and (a.leavesCount == b.leavesCount) and (a.levels == b.levels) 20 | 21 | proc `==`*(a, b: CodexProof): bool = 22 | (a.mcodec == b.mcodec) and (a.nleaves == b.nleaves) and (a.path == b.path) and 23 | (a.index == b.index) 24 | -------------------------------------------------------------------------------- /tests/codex/merkletree/testcodexcoders.nim: -------------------------------------------------------------------------------- 1 | import pkg/unittest2 2 | 3 | import pkg/questionable/results 4 | import pkg/stew/byteutils 5 | 6 | import pkg/codex/merkletree 7 | import ./helpers 8 | 9 | const data = [ 10 | "00000000000000000000000000000001".toBytes, 11 | "00000000000000000000000000000002".toBytes, 12 | "00000000000000000000000000000003".toBytes, 13 | "00000000000000000000000000000004".toBytes, 14 | "00000000000000000000000000000005".toBytes, 15 | "00000000000000000000000000000006".toBytes, 16 | "00000000000000000000000000000007".toBytes, 17 | "00000000000000000000000000000008".toBytes, 18 | "00000000000000000000000000000009".toBytes, "00000000000000000000000000000010".toBytes, 19 | ] 20 | 21 | suite "merkletree - coders": 22 | test "encoding and decoding a tree yields the same tree": 23 | let 24 | tree = CodexTree.init(Sha256HashCodec, data).tryGet() 25 | encodedBytes = tree.encode() 26 | decodedTree = CodexTree.decode(encodedBytes).tryGet() 27 | 28 | check: 29 | tree == decodedTree 30 | 31 | test "encoding and decoding a proof yields the same proof": 32 | let 33 | tree = CodexTree.init(Sha256HashCodec, data).tryGet() 34 | proof = tree.getProof(4).tryGet() 35 | 36 | check: 37 | proof.verify(tree.leaves[4], tree.root.tryGet).isOk 38 | 39 | let 40 | encodedBytes = proof.encode() 41 | decodedProof = CodexProof.decode(encodedBytes).tryGet() 42 | 43 | check: 44 | proof == decodedProof 45 | -------------------------------------------------------------------------------- /tests/codex/merkletree/testmerkledigest.nim: -------------------------------------------------------------------------------- 1 | import std/sequtils 2 | import std/random 3 | 4 | import pkg/unittest2 5 | import pkg/poseidon2 6 | import pkg/poseidon2/sponge 7 | 8 | import pkg/questionable/results 9 | 10 | import pkg/codex/merkletree 11 | import pkg/codex/utils/poseidon2digest 12 | 13 | import ./helpers 14 | 15 | suite "Digest - MerkleTree": 16 | const KB = 1024 17 | 18 | test "Hashes chunks of data with sponge, and combines them in merkle root": 19 | let bytes = newSeqWith(64 * KB, rand(byte)) 20 | var leaves: seq[Poseidon2Hash] 21 | for i in 0 ..< 32: 22 | let 23 | chunk = bytes[(i * 2 * KB) ..< ((i + 1) * 2 * KB)] 24 | digest = Sponge.digest(chunk, rate = 2) 25 | leaves.add(digest) 26 | 27 | let 28 | digest = Poseidon2Tree.digest(bytes, chunkSize = 2 * KB).tryGet 29 | spongeDigest = SpongeMerkle.digest(bytes, chunkSize = 2 * KB) 30 | codexPosTree = Poseidon2Tree.init(leaves).tryGet 31 | rootDigest = codexPosTree.root.tryGet 32 | 33 | check: 34 | bool(digest == spongeDigest) 35 | bool(digest == rootDigest) 36 | 37 | test "Handles partial chunk at the end": 38 | let bytes = newSeqWith(63 * KB, rand(byte)) 39 | var leaves: seq[Poseidon2Hash] 40 | for i in 0 ..< 31: 41 | let 42 | chunk = bytes[(i * 2 * KB) ..< ((i + 1) * 2 * KB)] 43 | digest = Sponge.digest(chunk, rate = 2) 44 | leaves.add(digest) 45 | 46 | let partialChunk = bytes[(62 * KB) ..< (63 * KB)] 47 | leaves.add(Sponge.digest(partialChunk, rate = 2)) 48 | 49 | let 50 | digest = Poseidon2Tree.digest(bytes, chunkSize = 2 * KB).tryGet 51 | spongeDigest = SpongeMerkle.digest(bytes, chunkSize = 2 * KB) 52 | codexPosTree = Poseidon2Tree.init(leaves).tryGet 53 | rootDigest = codexPosTree.root.tryGet 54 | 55 | check: 56 | bool(digest == spongeDigest) 57 | bool(digest == rootDigest) 58 | -------------------------------------------------------------------------------- /tests/codex/merkletree/testposeidon2tree.nim: -------------------------------------------------------------------------------- 1 | import std/sequtils 2 | 3 | import pkg/unittest2 4 | import pkg/poseidon2 5 | import pkg/poseidon2/io 6 | import pkg/questionable/results 7 | import pkg/results 8 | import pkg/stew/byteutils 9 | import pkg/stew/arrayops 10 | 11 | import pkg/codex/merkletree 12 | 13 | import ./generictreetests 14 | import ./helpers 15 | 16 | const data = [ 17 | "0000000000000000000000000000001".toBytes, 18 | "0000000000000000000000000000002".toBytes, 19 | "0000000000000000000000000000003".toBytes, 20 | "0000000000000000000000000000004".toBytes, 21 | "0000000000000000000000000000005".toBytes, 22 | "0000000000000000000000000000006".toBytes, 23 | "0000000000000000000000000000007".toBytes, 24 | "0000000000000000000000000000008".toBytes, 25 | "0000000000000000000000000000009".toBytes, 26 | # note one less to account for padding of field elements 27 | ] 28 | 29 | suite "Test Poseidon2Tree": 30 | var expectedLeaves: seq[Poseidon2Hash] 31 | 32 | setup: 33 | expectedLeaves = toSeq(data.concat().elements(Poseidon2Hash)) 34 | 35 | test "Should fail init tree from empty leaves": 36 | check: 37 | Poseidon2Tree.init(leaves = newSeq[Poseidon2Hash](0)).isErr 38 | 39 | test "Init tree from poseidon2 leaves": 40 | let tree = Poseidon2Tree.init(leaves = expectedLeaves).tryGet 41 | 42 | check: 43 | tree.leaves == expectedLeaves 44 | 45 | test "Init tree from byte leaves": 46 | let tree = Poseidon2Tree.init( 47 | leaves = expectedLeaves.mapIt(array[31, byte].initCopyFrom(it.toBytes)) 48 | ).tryGet 49 | 50 | check: 51 | tree.leaves == expectedLeaves 52 | 53 | test "Should build from nodes": 54 | let 55 | tree = Poseidon2Tree.init(leaves = expectedLeaves).tryGet 56 | fromNodes = Poseidon2Tree.fromNodes( 57 | nodes = toSeq(tree.nodes), nleaves = tree.leavesCount 58 | ).tryGet 59 | 60 | check: 61 | tree == fromNodes 62 | 63 | let 64 | compressor = proc( 65 | x, y: Poseidon2Hash, key: PoseidonKeysEnum 66 | ): Poseidon2Hash {.noSideEffect.} = 67 | compress(x, y, key.toKey) 68 | 69 | makeTree = proc(data: seq[Poseidon2Hash]): Poseidon2Tree = 70 | Poseidon2Tree.init(leaves = data).tryGet 71 | 72 | testGenericTree( 73 | "Poseidon2Tree", 74 | toSeq(data.concat().elements(Poseidon2Hash)), 75 | zero, 76 | compressor, 77 | makeTree, 78 | ) 79 | -------------------------------------------------------------------------------- /tests/codex/sales/helpers/periods.nim: -------------------------------------------------------------------------------- 1 | import pkg/codex/market 2 | import ../../helpers/mockclock 3 | 4 | proc advanceToNextPeriod*(clock: MockClock, market: Market) {.async.} = 5 | let periodicity = await market.periodicity() 6 | let period = periodicity.periodOf(clock.now().Timestamp) 7 | let periodEnd = periodicity.periodEnd(period) 8 | clock.set(periodEnd.toSecondsSince1970 + 1) 9 | -------------------------------------------------------------------------------- /tests/codex/sales/states/testdownloading.nim: -------------------------------------------------------------------------------- 1 | import pkg/unittest2 2 | import pkg/questionable 3 | import pkg/codex/contracts/requests 4 | import pkg/codex/sales/states/cancelled 5 | import pkg/codex/sales/states/downloading 6 | import pkg/codex/sales/states/failed 7 | import pkg/codex/sales/states/filled 8 | import ../../examples 9 | import ../../helpers 10 | 11 | suite "sales state 'downloading'": 12 | let request = StorageRequest.example 13 | let slotIndex = request.ask.slots div 2 14 | var state: SaleDownloading 15 | 16 | setup: 17 | state = SaleDownloading.new() 18 | 19 | test "switches to cancelled state when request expires": 20 | let next = state.onCancelled(request) 21 | check !next of SaleCancelled 22 | 23 | test "switches to failed state when request fails": 24 | let next = state.onFailed(request) 25 | check !next of SaleFailed 26 | 27 | test "switches to filled state when slot is filled": 28 | let next = state.onSlotFilled(request.id, slotIndex) 29 | check !next of SaleFilled 30 | -------------------------------------------------------------------------------- /tests/codex/sales/states/testerrored.nim: -------------------------------------------------------------------------------- 1 | import pkg/questionable 2 | import pkg/chronos 3 | import pkg/codex/contracts/requests 4 | import pkg/codex/sales/states/errored 5 | import pkg/codex/sales/salesagent 6 | import pkg/codex/sales/salescontext 7 | import pkg/codex/market 8 | 9 | import ../../../asynctest 10 | import ../../examples 11 | import ../../helpers 12 | import ../../helpers/mockmarket 13 | import ../../helpers/mockclock 14 | 15 | asyncchecksuite "sales state 'errored'": 16 | let request = StorageRequest.example 17 | let slotIndex = request.ask.slots div 2 18 | let market = MockMarket.new() 19 | let clock = MockClock.new() 20 | 21 | var state: SaleErrored 22 | var agent: SalesAgent 23 | var reprocessSlotWas = false 24 | 25 | setup: 26 | let onCleanUp = proc( 27 | reprocessSlot = false, returnedCollateral = UInt256.none 28 | ) {.async: (raises: []).} = 29 | reprocessSlotWas = reprocessSlot 30 | 31 | let context = SalesContext(market: market, clock: clock) 32 | agent = newSalesAgent(context, request.id, slotIndex, request.some) 33 | agent.onCleanUp = onCleanUp 34 | state = SaleErrored(error: newException(ValueError, "oh no!")) 35 | 36 | test "calls onCleanUp with reprocessSlot = true": 37 | state = SaleErrored(error: newException(ValueError, "oh no!"), reprocessSlot: true) 38 | discard await state.run(agent) 39 | check eventually reprocessSlotWas == true 40 | -------------------------------------------------------------------------------- /tests/codex/sales/states/testfilled.nim: -------------------------------------------------------------------------------- 1 | import pkg/questionable/results 2 | 3 | import pkg/codex/clock 4 | import pkg/codex/contracts/requests 5 | import pkg/codex/sales 6 | import pkg/codex/sales/salesagent 7 | import pkg/codex/sales/salescontext 8 | import pkg/codex/sales/states/filled 9 | import pkg/codex/sales/states/errored 10 | import pkg/codex/sales/states/proving 11 | 12 | import ../../../asynctest 13 | import ../../helpers/mockmarket 14 | import ../../examples 15 | import ../../helpers 16 | 17 | suite "sales state 'filled'": 18 | let request = StorageRequest.example 19 | let slotIndex = request.ask.slots div 2 20 | 21 | var market: MockMarket 22 | var slot: MockSlot 23 | var agent: SalesAgent 24 | var state: SaleFilled 25 | var onExpiryUpdatePassedExpiry: SecondsSince1970 26 | 27 | setup: 28 | market = MockMarket.new() 29 | slot = MockSlot( 30 | requestId: request.id, 31 | host: Address.example, 32 | slotIndex: slotIndex, 33 | proof: Groth16Proof.default, 34 | ) 35 | 36 | market.requestEnds[request.id] = 321 37 | onExpiryUpdatePassedExpiry = -1 38 | let onExpiryUpdate = proc( 39 | rootCid: Cid, expiry: SecondsSince1970 40 | ): Future[?!void] {.async: (raises: [CancelledError]).} = 41 | onExpiryUpdatePassedExpiry = expiry 42 | return success() 43 | let context = SalesContext(market: market, onExpiryUpdate: some onExpiryUpdate) 44 | 45 | agent = newSalesAgent(context, request.id, slotIndex, some request) 46 | state = SaleFilled.new() 47 | 48 | test "switches to proving state when slot is filled by me": 49 | slot.host = await market.getSigner() 50 | market.filled = @[slot] 51 | let next = await state.run(agent) 52 | check !next of SaleProving 53 | 54 | test "calls onExpiryUpdate with request end": 55 | slot.host = await market.getSigner() 56 | market.filled = @[slot] 57 | 58 | let expectedExpiry = 123 59 | market.requestEnds[request.id] = expectedExpiry 60 | let next = await state.run(agent) 61 | check !next of SaleProving 62 | check onExpiryUpdatePassedExpiry == expectedExpiry 63 | 64 | test "switches to error state when slot is filled by another host": 65 | slot.host = Address.example 66 | market.filled = @[slot] 67 | let next = await state.run(agent) 68 | check !next of SaleErrored 69 | -------------------------------------------------------------------------------- /tests/codex/sales/states/testfilling.nim: -------------------------------------------------------------------------------- 1 | import pkg/questionable 2 | import pkg/codex/contracts/requests 3 | import pkg/codex/sales/states/filling 4 | import pkg/codex/sales/states/cancelled 5 | import pkg/codex/sales/states/failed 6 | import pkg/codex/sales/states/ignored 7 | import pkg/codex/sales/states/errored 8 | import pkg/codex/sales/salesagent 9 | import pkg/codex/sales/salescontext 10 | import ../../../asynctest 11 | import ../../examples 12 | import ../../helpers 13 | import ../../helpers/mockmarket 14 | import ../../helpers/mockclock 15 | 16 | suite "sales state 'filling'": 17 | let request = StorageRequest.example 18 | let slotIndex = request.ask.slots div 2 19 | var state: SaleFilling 20 | var market: MockMarket 21 | var clock: MockClock 22 | var agent: SalesAgent 23 | 24 | setup: 25 | clock = MockClock.new() 26 | market = MockMarket.new() 27 | let context = SalesContext(market: market, clock: clock) 28 | agent = newSalesAgent(context, request.id, slotIndex, request.some) 29 | state = SaleFilling.new() 30 | 31 | test "switches to cancelled state when request expires": 32 | let next = state.onCancelled(request) 33 | check !next of SaleCancelled 34 | 35 | test "switches to failed state when request fails": 36 | let next = state.onFailed(request) 37 | check !next of SaleFailed 38 | 39 | test "run switches to ignored when slot is not free": 40 | let error = newException( 41 | SlotStateMismatchError, "Failed to fill slot because the slot is not free" 42 | ) 43 | market.setErrorOnFillSlot(error) 44 | market.requested.add(request) 45 | market.slotState[request.slotId(slotIndex)] = SlotState.Filled 46 | 47 | let next = !(await state.run(agent)) 48 | check next of SaleIgnored 49 | check SaleIgnored(next).reprocessSlot == false 50 | 51 | test "run switches to errored with other error ": 52 | let error = newException(MarketError, "some error") 53 | market.setErrorOnFillSlot(error) 54 | market.requested.add(request) 55 | market.slotState[request.slotId(slotIndex)] = SlotState.Filled 56 | 57 | let next = !(await state.run(agent)) 58 | check next of SaleErrored 59 | 60 | let errored = SaleErrored(next) 61 | check errored.error == error 62 | -------------------------------------------------------------------------------- /tests/codex/sales/states/testfinished.nim: -------------------------------------------------------------------------------- 1 | import pkg/questionable 2 | import pkg/codex/contracts/requests 3 | import pkg/codex/sales/states/finished 4 | import pkg/codex/sales/states/cancelled 5 | import pkg/codex/sales/states/failed 6 | import pkg/codex/sales/salesagent 7 | import pkg/codex/sales/salescontext 8 | import pkg/codex/market 9 | 10 | import ../../../asynctest 11 | import ../../examples 12 | import ../../helpers 13 | import ../../helpers/mockmarket 14 | import ../../helpers/mockclock 15 | 16 | asyncchecksuite "sales state 'finished'": 17 | let request = StorageRequest.example 18 | let slotIndex = request.ask.slots div 2 19 | let clock = MockClock.new() 20 | 21 | let currentCollateral = UInt256.example 22 | 23 | var market: MockMarket 24 | var state: SaleFinished 25 | var agent: SalesAgent 26 | var reprocessSlotWas = bool.none 27 | var returnedCollateralValue = UInt256.none 28 | var saleCleared = bool.none 29 | 30 | setup: 31 | market = MockMarket.new() 32 | let onCleanUp = proc( 33 | reprocessSlot = false, returnedCollateral = UInt256.none 34 | ) {.async: (raises: []).} = 35 | reprocessSlotWas = some reprocessSlot 36 | returnedCollateralValue = returnedCollateral 37 | 38 | let context = SalesContext(market: market, clock: clock) 39 | agent = newSalesAgent(context, request.id, slotIndex, request.some) 40 | agent.onCleanUp = onCleanUp 41 | agent.context.onClear = some proc(request: StorageRequest, idx: uint64) = 42 | saleCleared = some true 43 | state = SaleFinished(returnedCollateral: some currentCollateral) 44 | 45 | test "switches to cancelled state when request expires": 46 | let next = state.onCancelled(request) 47 | check !next of SaleCancelled 48 | 49 | test "switches to failed state when request fails": 50 | let next = state.onFailed(request) 51 | check !next of SaleFailed 52 | 53 | test "calls onCleanUp with reprocessSlot = true, and returnedCollateral = currentCollateral": 54 | discard await state.run(agent) 55 | check eventually reprocessSlotWas == some false 56 | check eventually returnedCollateralValue == some currentCollateral 57 | check eventually saleCleared == some true 58 | -------------------------------------------------------------------------------- /tests/codex/sales/states/testignored.nim: -------------------------------------------------------------------------------- 1 | import pkg/questionable 2 | import pkg/chronos 3 | import pkg/codex/contracts/requests 4 | import pkg/codex/sales/states/ignored 5 | import pkg/codex/sales/salesagent 6 | import pkg/codex/sales/salescontext 7 | import pkg/codex/market 8 | 9 | import ../../../asynctest 10 | import ../../examples 11 | import ../../helpers 12 | import ../../helpers/mockmarket 13 | import ../../helpers/mockclock 14 | 15 | asyncchecksuite "sales state 'ignored'": 16 | let request = StorageRequest.example 17 | let slotIndex = request.ask.slots div 2 18 | let market = MockMarket.new() 19 | let clock = MockClock.new() 20 | let currentCollateral = UInt256.example 21 | 22 | var state: SaleIgnored 23 | var agent: SalesAgent 24 | var reprocessSlotWas = false 25 | var returnedCollateralValue: ?UInt256 26 | 27 | setup: 28 | let onCleanUp = proc( 29 | reprocessSlot = false, returnedCollateral = UInt256.none 30 | ) {.async: (raises: []).} = 31 | reprocessSlotWas = reprocessSlot 32 | returnedCollateralValue = returnedCollateral 33 | 34 | let context = SalesContext(market: market, clock: clock) 35 | agent = newSalesAgent(context, request.id, slotIndex, request.some) 36 | agent.onCleanUp = onCleanUp 37 | state = SaleIgnored.new() 38 | returnedCollateralValue = UInt256.none 39 | reprocessSlotWas = false 40 | 41 | test "calls onCleanUp with values assigned to SaleIgnored": 42 | state = SaleIgnored(reprocessSlot: true) 43 | discard await state.run(agent) 44 | check eventually reprocessSlotWas == true 45 | check eventually returnedCollateralValue.isNone 46 | 47 | test "returns collateral when returnsCollateral is true": 48 | state = SaleIgnored(reprocessSlot: false, returnsCollateral: true) 49 | discard await state.run(agent) 50 | check eventually returnedCollateralValue.isSome 51 | -------------------------------------------------------------------------------- /tests/codex/sales/states/testpayout.nim: -------------------------------------------------------------------------------- 1 | import pkg/questionable 2 | import pkg/chronos 3 | import pkg/codex/contracts/requests 4 | import pkg/codex/sales/states/payout 5 | import pkg/codex/sales/states/finished 6 | import pkg/codex/sales/salesagent 7 | import pkg/codex/sales/salescontext 8 | import pkg/codex/market 9 | 10 | import ../../../asynctest 11 | import ../../examples 12 | import ../../helpers 13 | import ../../helpers/mockmarket 14 | import ../../helpers/mockclock 15 | 16 | asyncchecksuite "sales state 'payout'": 17 | let request = StorageRequest.example 18 | let slotIndex = request.ask.slots div 2 19 | let clock = MockClock.new() 20 | 21 | let currentCollateral = UInt256.example 22 | 23 | var market: MockMarket 24 | var state: SalePayout 25 | var agent: SalesAgent 26 | 27 | setup: 28 | market = MockMarket.new() 29 | 30 | let context = SalesContext(market: market, clock: clock) 31 | agent = newSalesAgent(context, request.id, slotIndex, request.some) 32 | state = SalePayout.new() 33 | 34 | test "switches to 'finished' state and provides returnedCollateral": 35 | market.fillSlot( 36 | requestId = request.id, 37 | slotIndex = slotIndex, 38 | proof = Groth16Proof.default, 39 | host = Address.example, 40 | collateral = currentCollateral, 41 | ) 42 | let next = await state.run(agent) 43 | check !next of SaleFinished 44 | check SaleFinished(!next).returnedCollateral == some currentCollateral 45 | -------------------------------------------------------------------------------- /tests/codex/sales/teststates.nim: -------------------------------------------------------------------------------- 1 | import ./states/testunknown 2 | import ./states/testdownloading 3 | import ./states/testfilling 4 | import ./states/testpayout 5 | import ./states/testfinished 6 | import ./states/testinitialproving 7 | import ./states/testfilled 8 | import ./states/testproving 9 | import ./states/testsimulatedproving 10 | import ./states/testcancelled 11 | import ./states/testerrored 12 | import ./states/testignored 13 | import ./states/testpreparing 14 | import ./states/testslotreserving 15 | 16 | {.warning[UnusedImport]: off.} 17 | -------------------------------------------------------------------------------- /tests/codex/slots/testbackends.nim: -------------------------------------------------------------------------------- 1 | import ./backends/testcircomcompat 2 | 3 | {.warning[UnusedImport]: off.} 4 | -------------------------------------------------------------------------------- /tests/codex/slots/testconverters.nim: -------------------------------------------------------------------------------- 1 | import pkg/chronos 2 | import pkg/poseidon2 3 | import pkg/poseidon2/io 4 | import pkg/constantine/math/io/io_fields 5 | import pkg/questionable/results 6 | import pkg/codex/merkletree 7 | import pkg/codex/slots/converters 8 | 9 | import ../../asynctest 10 | import ../examples 11 | import ../merkletree/helpers 12 | 13 | let hash: Poseidon2Hash = toF(12345) 14 | 15 | suite "Converters": 16 | test "CellBlock cid": 17 | let 18 | cid = toCellCid(hash).tryGet() 19 | value = fromCellCid(cid).tryGet() 20 | 21 | check: 22 | hash.toDecimal() == value.toDecimal() 23 | 24 | test "Slot cid": 25 | let 26 | cid = toSlotCid(hash).tryGet() 27 | value = fromSlotCid(cid).tryGet() 28 | 29 | check: 30 | hash.toDecimal() == value.toDecimal() 31 | 32 | test "Verify cid": 33 | let 34 | cid = toVerifyCid(hash).tryGet() 35 | value = fromVerifyCid(cid).tryGet() 36 | 37 | check: 38 | hash.toDecimal() == value.toDecimal() 39 | 40 | test "Proof": 41 | let 42 | codexProof = toEncodableProof(Poseidon2Proof.example).tryGet() 43 | poseidonProof = toVerifiableProof(codexProof).tryGet() 44 | 45 | check: 46 | Poseidon2Proof.example == poseidonProof 47 | -------------------------------------------------------------------------------- /tests/codex/slots/testsampler.nim: -------------------------------------------------------------------------------- 1 | import ./sampler/testsampler 2 | import ./sampler/testutils 3 | 4 | {.warning[UnusedImport]: off.} 5 | -------------------------------------------------------------------------------- /tests/codex/stores/repostore/testcoders.nim: -------------------------------------------------------------------------------- 1 | import std/random 2 | 3 | import pkg/unittest2 4 | import pkg/stew/objects 5 | import pkg/questionable 6 | import pkg/questionable/results 7 | 8 | import pkg/codex/clock 9 | import pkg/codex/stores/repostore/types 10 | import pkg/codex/stores/repostore/coders 11 | 12 | import ../../helpers 13 | 14 | suite "Test coders": 15 | proc rand(T: type NBytes): T = 16 | rand(Natural).NBytes 17 | 18 | proc rand(E: type[enum]): E = 19 | let ordinals = enumRangeInt64(E) 20 | E(ordinals[rand(ordinals.len - 1)]) 21 | 22 | proc rand(T: type QuotaUsage): T = 23 | QuotaUsage(used: rand(NBytes), reserved: rand(NBytes)) 24 | 25 | proc rand(T: type BlockMetadata): T = 26 | BlockMetadata( 27 | expiry: rand(SecondsSince1970), size: rand(NBytes), refCount: rand(Natural) 28 | ) 29 | 30 | proc rand(T: type DeleteResult): T = 31 | DeleteResult(kind: rand(DeleteResultKind), released: rand(NBytes)) 32 | 33 | proc rand(T: type StoreResult): T = 34 | StoreResult(kind: rand(StoreResultKind), used: rand(NBytes)) 35 | 36 | test "Natural encode/decode": 37 | for val in newSeqWith[Natural](100, rand(Natural)) & @[Natural.low, Natural.high]: 38 | check: 39 | success(val) == Natural.decode(encode(val)) 40 | 41 | test "QuotaUsage encode/decode": 42 | for val in newSeqWith[QuotaUsage](100, rand(QuotaUsage)): 43 | check: 44 | success(val) == QuotaUsage.decode(encode(val)) 45 | 46 | test "BlockMetadata encode/decode": 47 | for val in newSeqWith[BlockMetadata](100, rand(BlockMetadata)): 48 | check: 49 | success(val) == BlockMetadata.decode(encode(val)) 50 | 51 | test "DeleteResult encode/decode": 52 | for val in newSeqWith[DeleteResult](100, rand(DeleteResult)): 53 | check: 54 | success(val) == DeleteResult.decode(encode(val)) 55 | 56 | test "StoreResult encode/decode": 57 | for val in newSeqWith[StoreResult](100, rand(StoreResult)): 58 | check: 59 | success(val) == StoreResult.decode(encode(val)) 60 | -------------------------------------------------------------------------------- /tests/codex/stores/testcachestore.nim: -------------------------------------------------------------------------------- 1 | import std/strutils 2 | 3 | import pkg/chronos 4 | import pkg/stew/byteutils 5 | import pkg/questionable/results 6 | import pkg/codex/stores/cachestore 7 | import pkg/codex/chunker 8 | 9 | import ./commonstoretests 10 | 11 | import ../../asynctest 12 | import ../helpers 13 | 14 | suite "Cache Store": 15 | var 16 | newBlock, newBlock1, newBlock2, newBlock3: Block 17 | store: CacheStore 18 | 19 | setup: 20 | newBlock = Block.new("New Kids on the Block".toBytes()).tryGet() 21 | newBlock1 = Block.new("1".repeat(100).toBytes()).tryGet() 22 | newBlock2 = Block.new("2".repeat(100).toBytes()).tryGet() 23 | newBlock3 = Block.new("3".repeat(100).toBytes()).tryGet() 24 | store = CacheStore.new() 25 | 26 | test "constructor": 27 | # cache size cannot be smaller than chunk size 28 | expect ValueError: 29 | discard CacheStore.new(cacheSize = 1, chunkSize = 2) 30 | 31 | store = CacheStore.new(cacheSize = 100, chunkSize = 1) 32 | check store.currentSize == 0'nb 33 | 34 | store = CacheStore.new(@[newBlock1, newBlock2, newBlock3]) 35 | check store.currentSize == 300'nb 36 | 37 | # initial cache blocks total more than cache size, currentSize should 38 | # never exceed max cache size 39 | store = CacheStore.new( 40 | blocks = @[newBlock1, newBlock2, newBlock3], cacheSize = 200, chunkSize = 1 41 | ) 42 | check store.currentSize == 200'nb 43 | 44 | # cache size cannot be less than chunks size 45 | expect ValueError: 46 | discard CacheStore.new(cacheSize = 99, chunkSize = 100) 47 | 48 | test "putBlock": 49 | (await store.putBlock(newBlock1)).tryGet() 50 | check (await store.hasBlock(newBlock1.cid)).tryGet() 51 | 52 | # block size bigger than entire cache 53 | store = CacheStore.new(cacheSize = 99, chunkSize = 98) 54 | (await store.putBlock(newBlock1)).tryGet() 55 | check not (await store.hasBlock(newBlock1.cid)).tryGet() 56 | 57 | # block being added causes removal of LRU block 58 | store = 59 | CacheStore.new(@[newBlock1, newBlock2, newBlock3], cacheSize = 200, chunkSize = 1) 60 | check: 61 | not (await store.hasBlock(newBlock1.cid)).tryGet() 62 | (await store.hasBlock(newBlock2.cid)).tryGet() 63 | (await store.hasBlock(newBlock2.cid)).tryGet() 64 | store.currentSize.int == newBlock2.data.len + newBlock3.data.len # 200 65 | 66 | commonBlockStoreTests( 67 | "Cache", 68 | proc(): BlockStore = 69 | BlockStore(CacheStore.new(cacheSize = 1000, chunkSize = 1)), 70 | ) 71 | -------------------------------------------------------------------------------- /tests/codex/stores/testqueryiterhelper.nim: -------------------------------------------------------------------------------- 1 | import std/sugar 2 | 3 | import pkg/results 4 | import pkg/questionable 5 | import pkg/chronos 6 | import pkg/datastore/typedds 7 | import pkg/datastore/sql/sqliteds 8 | import pkg/codex/stores/queryiterhelper 9 | import pkg/codex/utils/asynciter 10 | 11 | import ../../asynctest 12 | import ../helpers 13 | 14 | proc encode(s: string): seq[byte] = 15 | s.toBytes() 16 | 17 | proc decode(T: type string, bytes: seq[byte]): ?!T = 18 | success(string.fromBytes(bytes)) 19 | 20 | asyncchecksuite "Test QueryIter helper": 21 | var tds: TypedDatastore 22 | 23 | setupAll: 24 | tds = TypedDatastore.init(SQLiteDatastore.new(Memory).tryGet()) 25 | 26 | teardownAll: 27 | (await tds.close()).tryGet 28 | 29 | test "Should auto-dispose when QueryIter finishes": 30 | let 31 | source = {"a": "11", "b": "22"}.toTable 32 | Root = Key.init("/queryitertest").tryGet() 33 | 34 | for k, v in source: 35 | let key = (Root / k).tryGet() 36 | (await tds.put(key, v)).tryGet() 37 | 38 | var 39 | disposed = false 40 | queryIter = (await query[string](tds, Query.init(Root))).tryGet() 41 | 42 | let iterDispose: IterDispose = queryIter.dispose 43 | queryIter.dispose = () => (disposed = true; iterDispose()) 44 | 45 | let 46 | iter1 = (await toAsyncIter[string](queryIter)).tryGet() 47 | iter2 = await filterSuccess[string](iter1) 48 | 49 | var items = initTable[string, string]() 50 | 51 | for fut in iter2: 52 | let item = await fut 53 | 54 | items[item.key.value] = item.value 55 | 56 | check: 57 | items == source 58 | disposed == true 59 | queryIter.finished == true 60 | iter1.finished == true 61 | iter2.finished == true 62 | -------------------------------------------------------------------------------- /tests/codex/testblockexchange.nim: -------------------------------------------------------------------------------- 1 | import ./blockexchange/testengine 2 | import ./blockexchange/testnetwork 3 | import ./blockexchange/testpeerctxstore 4 | import ./blockexchange/testdiscovery 5 | import ./blockexchange/testprotobuf 6 | import ./blockexchange/testpendingblocks 7 | 8 | {.warning[UnusedImport]: off.} 9 | -------------------------------------------------------------------------------- /tests/codex/testclock.nim: -------------------------------------------------------------------------------- 1 | import pkg/unittest2 2 | 3 | import codex/clock 4 | import ./helpers 5 | 6 | suite "Clock": 7 | proc testConversion(seconds: SecondsSince1970) = 8 | let asBytes = seconds.toBytes 9 | 10 | let restored = asBytes.toSecondsSince1970 11 | 12 | check restored == seconds 13 | 14 | test "SecondsSince1970 should support bytes conversions": 15 | let secondsToTest: seq[int64] = @[int64.high, int64.low, 0, 1, 12345, -1, -12345] 16 | 17 | for seconds in secondsToTest: 18 | testConversion(seconds) 19 | -------------------------------------------------------------------------------- /tests/codex/testindexingstrategy.nim: -------------------------------------------------------------------------------- 1 | import std/sequtils 2 | import pkg/chronos 3 | 4 | import pkg/codex/utils/asynciter 5 | 6 | import ../asynctest 7 | import ./helpers 8 | 9 | import pkg/codex/indexingstrategy 10 | 11 | for offset in @[0, 1, 2, 100]: 12 | suite "Indexing strategies (Offset: " & $offset & ")": 13 | let 14 | firstIndex = 0 + offset 15 | lastIndex = 12 + offset 16 | nIters = 3 17 | linear = LinearStrategy.init(firstIndex, lastIndex, nIters) 18 | stepped = SteppedStrategy.init(firstIndex, lastIndex, nIters) 19 | 20 | test "linear": 21 | check: 22 | toSeq(linear.getIndicies(0)) == @[0, 1, 2, 3, 4].mapIt(it + offset) 23 | toSeq(linear.getIndicies(1)) == @[5, 6, 7, 8, 9].mapIt(it + offset) 24 | toSeq(linear.getIndicies(2)) == @[10, 11, 12].mapIt(it + offset) 25 | 26 | test "stepped": 27 | check: 28 | toSeq(stepped.getIndicies(0)) == @[0, 3, 6, 9, 12].mapIt(it + offset) 29 | toSeq(stepped.getIndicies(1)) == @[1, 4, 7, 10].mapIt(it + offset) 30 | toSeq(stepped.getIndicies(2)) == @[2, 5, 8, 11].mapIt(it + offset) 31 | 32 | suite "Indexing strategies": 33 | let 34 | linear = LinearStrategy.init(0, 10, 3) 35 | stepped = SteppedStrategy.init(0, 10, 3) 36 | 37 | test "smallest range 0": 38 | let 39 | l = LinearStrategy.init(0, 0, 1) 40 | s = SteppedStrategy.init(0, 0, 1) 41 | check: 42 | toSeq(l.getIndicies(0)) == @[0] 43 | toSeq(s.getIndicies(0)) == @[0] 44 | 45 | test "smallest range 1": 46 | let 47 | l = LinearStrategy.init(0, 1, 1) 48 | s = SteppedStrategy.init(0, 1, 1) 49 | check: 50 | toSeq(l.getIndicies(0)) == @[0, 1] 51 | toSeq(s.getIndicies(0)) == @[0, 1] 52 | 53 | test "first index must be smaller than last index": 54 | expect IndexingWrongIndexError: 55 | discard LinearStrategy.init(10, 0, 1) 56 | 57 | test "iterations must be greater than zero": 58 | expect IndexingWrongIterationsError: 59 | discard LinearStrategy.init(0, 10, 0) 60 | 61 | test "should split elements evenly when possible": 62 | let l = LinearStrategy.init(0, 11, 3) 63 | check: 64 | toSeq(l.getIndicies(0)) == @[0, 1, 2, 3].mapIt(it) 65 | toSeq(l.getIndicies(1)) == @[4, 5, 6, 7].mapIt(it) 66 | toSeq(l.getIndicies(2)) == @[8, 9, 10, 11].mapIt(it) 67 | 68 | test "linear - oob": 69 | expect IndexingError: 70 | discard linear.getIndicies(3) 71 | 72 | test "stepped - oob": 73 | expect IndexingError: 74 | discard stepped.getIndicies(3) 75 | -------------------------------------------------------------------------------- /tests/codex/testmerkletree.nim: -------------------------------------------------------------------------------- 1 | import ./merkletree/testcodextree 2 | import ./merkletree/testposeidon2tree 3 | import ./merkletree/testcodexcoders 4 | import ./merkletree/testmerkledigest 5 | 6 | {.warning[UnusedImport]: off.} 7 | -------------------------------------------------------------------------------- /tests/codex/testnat.nim: -------------------------------------------------------------------------------- 1 | import std/[unittest, options, net], stew/shims/net as stewNet 2 | import pkg/chronos 3 | import pkg/libp2p/[multiaddress, multihash, multicodec] 4 | import pkg/results 5 | 6 | import ../../codex/nat 7 | import ../../codex/utils/natutils 8 | import ../../codex/utils 9 | 10 | suite "NAT Address Tests": 11 | test "nattedAddress with local addresses": 12 | # Setup test data 13 | let 14 | udpPort = Port(1234) 15 | natConfig = NatConfig(hasExtIp: true, extIp: parseIpAddress("8.8.8.8")) 16 | 17 | # Create test addresses 18 | localAddr = MultiAddress.init("/ip4/127.0.0.1/tcp/5000").expect("valid multiaddr") 19 | anyAddr = MultiAddress.init("/ip4/0.0.0.0/tcp/5000").expect("valid multiaddr") 20 | publicAddr = 21 | MultiAddress.init("/ip4/192.168.1.1/tcp/5000").expect("valid multiaddr") 22 | 23 | # Expected results 24 | let 25 | expectedDiscoveryAddrs = 26 | @[ 27 | MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), 28 | MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), 29 | MultiAddress.init("/ip4/8.8.8.8/udp/1234").expect("valid multiaddr"), 30 | ] 31 | expectedlibp2pAddrs = 32 | @[ 33 | MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), 34 | MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), 35 | MultiAddress.init("/ip4/8.8.8.8/tcp/5000").expect("valid multiaddr"), 36 | ] 37 | 38 | #ipv6Addr = MultiAddress.init("/ip6/::1/tcp/5000").expect("valid multiaddr") 39 | addrs = @[localAddr, anyAddr, publicAddr] 40 | 41 | # Test address remapping 42 | let (libp2pAddrs, discoveryAddrs) = nattedAddress(natConfig, addrs, udpPort) 43 | 44 | # Verify results 45 | check(discoveryAddrs == expectedDiscoveryAddrs) 46 | check(libp2pAddrs == expectedlibp2pAddrs) 47 | -------------------------------------------------------------------------------- /tests/codex/testnode.nim: -------------------------------------------------------------------------------- 1 | import ./node/testnode 2 | import ./node/testcontracts 3 | 4 | {.warning[UnusedImport]: off.} 5 | -------------------------------------------------------------------------------- /tests/codex/testsales.nim: -------------------------------------------------------------------------------- 1 | import ./sales/testsales 2 | import ./sales/teststates 3 | import ./sales/testreservations 4 | import ./sales/testslotqueue 5 | import ./sales/testsalesagent 6 | 7 | {.warning[UnusedImport]: off.} 8 | -------------------------------------------------------------------------------- /tests/codex/testslots.nim: -------------------------------------------------------------------------------- 1 | import ./slots/testslotbuilder 2 | import ./slots/testsampler 3 | import ./slots/testconverters 4 | import ./slots/testbackends 5 | import ./slots/testprover 6 | import ./slots/testbackendfactory 7 | 8 | {.warning[UnusedImport]: off.} 9 | -------------------------------------------------------------------------------- /tests/codex/teststores.nim: -------------------------------------------------------------------------------- 1 | import ./stores/testcachestore 2 | import ./stores/testrepostore 3 | import ./stores/testmaintenance 4 | import ./stores/testqueryiterhelper 5 | 6 | {.warning[UnusedImport]: off.} 7 | -------------------------------------------------------------------------------- /tests/codex/testsystemclock.nim: -------------------------------------------------------------------------------- 1 | import std/times 2 | 3 | import pkg/unittest2 4 | import pkg/codex/systemclock 5 | import ./helpers 6 | 7 | suite "SystemClock": 8 | test "Should get now": 9 | let clock = SystemClock.new() 10 | 11 | let expectedNow = times.now().utc 12 | let now = clock.now() 13 | 14 | check now == expectedNow.toTime().toUnix() 15 | -------------------------------------------------------------------------------- /tests/codex/testutils.nim: -------------------------------------------------------------------------------- 1 | import ./utils/testoptions 2 | import ./utils/testkeyutils 3 | import ./utils/testasyncstatemachine 4 | import ./utils/testasynciter 5 | import ./utils/testsafeasynciter 6 | import ./utils/testtimer 7 | import ./utils/testtrackedfutures 8 | 9 | {.warning[UnusedImport]: off.} 10 | -------------------------------------------------------------------------------- /tests/codex/utils/testkeyutils.nim: -------------------------------------------------------------------------------- 1 | import std/os 2 | 3 | import pkg/unittest2 4 | import pkg/codex/utils/keyutils 5 | 6 | import ../helpers 7 | 8 | when defined(windows): 9 | import stew/windows/acl 10 | 11 | suite "keyutils": 12 | let path = getTempDir() / "CodexTest" 13 | 14 | setup: 15 | os.createDir(path) 16 | 17 | teardown: 18 | os.removeDir(path) 19 | 20 | test "creates a key file when it does not exist yet": 21 | check setupKey(path / "keyfile").isOk 22 | check fileExists(path / "keyfile") 23 | 24 | test "stores key in a file that's only readable by the user": 25 | discard setupKey(path / "keyfile").get() 26 | when defined(posix): 27 | check getFilePermissions(path / "keyfile") == {fpUserRead, fpUserWrite} 28 | when defined(windows): 29 | check checkCurrentUserOnlyACL(path / "keyfile").get() 30 | 31 | test "reads key file when it does exist": 32 | let key = setupKey(path / "keyfile").get() 33 | check setupKey(path / "keyfile").get() == key 34 | -------------------------------------------------------------------------------- /tests/codex/utils/testoptions.nim: -------------------------------------------------------------------------------- 1 | import pkg/unittest2 2 | import pkg/codex/utils/options 3 | 4 | import ../helpers 5 | 6 | suite "optional casts": 7 | test "casting value to same type works": 8 | check 42 as int == some 42 9 | 10 | test "casting value to unrelated type evaluates to None": 11 | check 42 as string == string.none 12 | 13 | test "casting value to subtype works": 14 | type 15 | BaseType = ref object of RootObj 16 | SubType = ref object of BaseType 17 | 18 | let x: BaseType = SubType() 19 | check x as SubType == SubType(x).some 20 | 21 | test "casting to unrelated subtype evaluates to None": 22 | type 23 | BaseType = ref object of RootObj 24 | SubType = ref object of BaseType 25 | OtherType = ref object of BaseType 26 | 27 | let x: BaseType = SubType() 28 | check x as OtherType == OtherType.none 29 | 30 | test "casting works on optional types": 31 | check 42.some as int == some 42 32 | check 42.some as string == string.none 33 | check int.none as int == int.none 34 | 35 | suite "Optionalize": 36 | test "does not except non-object types": 37 | static: 38 | doAssert not compiles(Optionalize(int)) 39 | 40 | test "converts object fields to option": 41 | type BaseType = object 42 | a: int 43 | b: bool 44 | c: string 45 | d: Option[string] 46 | 47 | type OptionalizedType = Optionalize(BaseType) 48 | 49 | check OptionalizedType.a is Option[int] 50 | check OptionalizedType.b is Option[bool] 51 | check OptionalizedType.c is Option[string] 52 | check OptionalizedType.d is Option[string] 53 | -------------------------------------------------------------------------------- /tests/codex/utils/testtimer.nim: -------------------------------------------------------------------------------- 1 | ## Nim-Codex 2 | ## Copyright (c) 2023 Status Research & Development GmbH 3 | ## Licensed under either of 4 | ## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) 5 | ## * MIT license ([LICENSE-MIT](LICENSE-MIT)) 6 | ## at your option. 7 | ## This file may not be copied, modified, or distributed except according to 8 | ## those terms. 9 | 10 | import pkg/chronos 11 | 12 | import codex/utils/timer 13 | 14 | import ../../asynctest 15 | import ../helpers 16 | 17 | asyncchecksuite "Timer": 18 | var timer1: Timer 19 | var timer2: Timer 20 | var output: string 21 | var numbersState = 0 22 | var lettersState = 'a' 23 | 24 | proc numbersCallback(): Future[void] {.async: (raises: []).} = 25 | output &= $numbersState 26 | inc numbersState 27 | 28 | proc lettersCallback(): Future[void] {.async: (raises: []).} = 29 | output &= $lettersState 30 | inc lettersState 31 | 32 | proc startNumbersTimer() = 33 | timer1.start(numbersCallback, 10.milliseconds) 34 | 35 | proc startLettersTimer() = 36 | timer2.start(lettersCallback, 10.milliseconds) 37 | 38 | setup: 39 | timer1 = Timer.new() 40 | timer2 = Timer.new() 41 | 42 | output = "" 43 | numbersState = 0 44 | lettersState = 'a' 45 | 46 | teardown: 47 | await timer1.stop() 48 | await timer2.stop() 49 | 50 | test "Start timer1 should execute callback": 51 | startNumbersTimer() 52 | check eventually(output == "0", pollInterval = 10) 53 | 54 | test "Start timer1 should execute callback multiple times": 55 | startNumbersTimer() 56 | check eventually(output == "012", pollInterval = 10) 57 | 58 | test "Starting timer1 multiple times has no impact": 59 | startNumbersTimer() 60 | startNumbersTimer() 61 | startNumbersTimer() 62 | check eventually(output == "01234", pollInterval = 10) 63 | 64 | test "Stop timer1 should stop execution of the callback": 65 | startNumbersTimer() 66 | check eventually(output == "012", pollInterval = 10) 67 | await timer1.stop() 68 | await sleepAsync(30.milliseconds) 69 | let stoppedOutput = output 70 | await sleepAsync(30.milliseconds) 71 | check output == stoppedOutput 72 | 73 | test "Starting both timers should execute callbacks sequentially": 74 | startNumbersTimer() 75 | startLettersTimer() 76 | check eventually(output == "0a1b2c3d4e", pollInterval = 10) 77 | -------------------------------------------------------------------------------- /tests/codex/utils/testutils.nim: -------------------------------------------------------------------------------- 1 | import pkg/unittest2 2 | 3 | import pkg/codex/utils 4 | 5 | suite "findIt": 6 | setup: 7 | type AnObject = object 8 | attribute1*: int 9 | 10 | var objList = 11 | @[ 12 | AnObject(attribute1: 1), 13 | AnObject(attribute1: 3), 14 | AnObject(attribute1: 5), 15 | AnObject(attribute1: 3), 16 | ] 17 | 18 | test "should retur index of first object matching predicate": 19 | assert objList.findIt(it.attribute1 == 3) == 1 20 | 21 | test "should return -1 when no object matches predicate": 22 | assert objList.findIt(it.attribute1 == 15) == -1 23 | 24 | suite "parseDuration": 25 | test "should parse durations": 26 | var res: Duration # caller must still know if 'b' refers to bytes|bits 27 | check parseDuration("10Hr", res) == 3 28 | check res == hours(10) 29 | check parseDuration("64min", res) == 3 30 | check res == minutes(64) 31 | check parseDuration("7m/block", res) == 2 # '/' stops parse 32 | check res == minutes(7) # 1 shl 30, forced binary metric 33 | check parseDuration("3d", res) == 2 # '/' stops parse 34 | check res == days(3) # 1 shl 30, forced binary metric 35 | -------------------------------------------------------------------------------- /tests/config.nims: -------------------------------------------------------------------------------- 1 | --path: 2 | ".." 3 | --threads: 4 | on 5 | --tlsEmulation: 6 | off 7 | 8 | when not defined(chronicles_log_level): 9 | --define: 10 | "chronicles_log_level:NONE" # compile all log statements 11 | --define: 12 | "chronicles_sinks:textlines[dynamic]" # allow logs to be filtered at runtime 13 | --"import": 14 | "logging" # ensure that logging is ignored at runtime 15 | -------------------------------------------------------------------------------- /tests/contracts/deployment.nim: -------------------------------------------------------------------------------- 1 | import std/os 2 | import std/options 3 | import pkg/ethers 4 | import pkg/codex/contracts/marketplace 5 | 6 | const hardhatMarketAddress = 7 | Address.init("0x322813Fd9A801c5507c9de605d63CEA4f2CE6c44").get() 8 | const hardhatMarketWithDummyVerifier = 9 | Address.init("0xa85233C63b9Ee964Add6F2cffe00Fd84eb32338f").get() 10 | const marketAddressEnvName = "CODEX_MARKET_ADDRESS" 11 | 12 | proc address*(_: type Marketplace, dummyVerifier = false): Address = 13 | if existsEnv(marketAddressEnvName): 14 | without address =? Address.init(getEnv(marketAddressEnvName)): 15 | raiseAssert "Invalid env. variable marketplace contract address" 16 | 17 | return address 18 | 19 | if dummyVerifier: hardhatMarketWithDummyVerifier else: hardhatMarketAddress 20 | -------------------------------------------------------------------------------- /tests/contracts/examples.nim: -------------------------------------------------------------------------------- 1 | import pkg/ethers 2 | import ../examples 3 | 4 | export examples 5 | 6 | proc example*(_: type Address): Address = 7 | Address(array[20, byte].example) 8 | -------------------------------------------------------------------------------- /tests/contracts/nim.cfg: -------------------------------------------------------------------------------- 1 | --path:"../.." 2 | -------------------------------------------------------------------------------- /tests/contracts/testClock.nim: -------------------------------------------------------------------------------- 1 | import std/times 2 | import pkg/chronos 3 | import codex/contracts/clock 4 | import codex/utils/json 5 | import ../ethertest 6 | 7 | ethersuite "On-Chain Clock": 8 | var clock: OnChainClock 9 | 10 | setup: 11 | clock = OnChainClock.new(ethProvider) 12 | await clock.start() 13 | 14 | teardown: 15 | await clock.stop() 16 | 17 | test "returns the current time of the EVM": 18 | let latestBlock = (!await ethProvider.getBlock(BlockTag.latest)) 19 | let timestamp = latestBlock.timestamp.truncate(int64) 20 | check clock.now() == timestamp 21 | 22 | test "updates time with timestamp of new blocks": 23 | let future = (getTime() + 42.years).toUnix 24 | discard await ethProvider.send("evm_setNextBlockTimestamp", @[%future]) 25 | discard await ethProvider.send("evm_mine") 26 | 27 | # Ensure that state updates are sync with WS 28 | discard await ethProvider.getBlock(BlockTag.latest) 29 | 30 | check eventually clock.now() == future 31 | 32 | test "can wait until a certain time is reached by the chain": 33 | let future = clock.now() + 42 # seconds 34 | let waiting = clock.waitUntil(future) 35 | discard await ethProvider.send("evm_setNextBlockTimestamp", @[%future]) 36 | discard await ethProvider.send("evm_mine") 37 | check await waiting.withTimeout(chronos.milliseconds(500)) 38 | 39 | test "can wait until a certain time is reached by the wall-clock": 40 | let future = clock.now() + 1 # seconds 41 | let waiting = clock.waitUntil(future) 42 | check await waiting.withTimeout(chronos.seconds(2)) 43 | 44 | test "handles starting multiple times": 45 | await clock.start() 46 | await clock.start() 47 | 48 | test "handles stopping multiple times": 49 | await clock.stop() 50 | await clock.stop() 51 | -------------------------------------------------------------------------------- /tests/contracts/testDeployment.nim: -------------------------------------------------------------------------------- 1 | import pkg/ethers 2 | import codex/contracts/deployment 3 | import codex/conf 4 | import codex/contracts 5 | import pkg/codex/utils/natutils 6 | 7 | import ../asynctest 8 | import ../checktest 9 | 10 | type MockProvider = ref object of Provider 11 | chainId*: UInt256 12 | 13 | method getChainId*( 14 | provider: MockProvider 15 | ): Future[UInt256] {.async: (raises: [ProviderError, CancelledError]).} = 16 | return provider.chainId 17 | 18 | proc configFactory(): CodexConf = 19 | CodexConf( 20 | cmd: StartUpCmd.persistence, 21 | nat: NatConfig(hasExtIp: false, nat: NatNone), 22 | metricsAddress: parseIpAddress("127.0.0.1"), 23 | ) 24 | 25 | proc configFactory(marketplace: Option[EthAddress]): CodexConf = 26 | CodexConf( 27 | cmd: StartUpCmd.persistence, 28 | nat: NatConfig(hasExtIp: false, nat: NatNone), 29 | metricsAddress: parseIpAddress("127.0.0.1"), 30 | marketplaceAddress: marketplace, 31 | ) 32 | 33 | asyncchecksuite "Deployment": 34 | let provider = MockProvider() 35 | 36 | test "uses conf value as priority": 37 | let deployment = Deployment.new( 38 | provider, 39 | configFactory(EthAddress.init("0x59b670e9fA9D0A427751Af201D676719a970aaaa")), 40 | ) 41 | provider.chainId = 1.u256 42 | 43 | let address = await deployment.address(Marketplace) 44 | check address.isSome 45 | check $(!address) == "0x59b670e9fa9d0a427751af201d676719a970aaaa" 46 | 47 | test "uses chainId hardcoded values as fallback": 48 | let deployment = Deployment.new(provider, configFactory()) 49 | provider.chainId = 167005.u256 50 | 51 | let address = await deployment.address(Marketplace) 52 | check address.isSome 53 | check $(!address) == "0x948cf9291b77bd7ad84781b9047129addf1b894f" 54 | 55 | test "return none for unknown networks": 56 | let deployment = Deployment.new(provider, configFactory()) 57 | provider.chainId = 1.u256 58 | 59 | let address = await deployment.address(Marketplace) 60 | check address.isNone 61 | -------------------------------------------------------------------------------- /tests/contracts/time.nim: -------------------------------------------------------------------------------- 1 | import pkg/ethers 2 | import pkg/serde/json 3 | 4 | proc blockTime*(provider: Provider, tag: BlockTag): Future[UInt256] {.async.} = 5 | return (!await provider.getBlock(tag)).timestamp 6 | 7 | proc currentTime*(provider: Provider): Future[UInt256] {.async.} = 8 | return await provider.blockTime(BlockTag.pending) 9 | 10 | proc advanceTime*(provider: JsonRpcProvider, seconds: UInt256) {.async.} = 11 | discard await provider.send("evm_increaseTime", @[%("0x" & seconds.toHex)]) 12 | discard await provider.send("evm_mine") 13 | 14 | proc advanceTimeTo*(provider: JsonRpcProvider, timestamp: UInt256) {.async.} = 15 | if (await provider.currentTime()) != timestamp: 16 | discard 17 | await provider.send("evm_setNextBlockTimestamp", @[%("0x" & timestamp.toHex)]) 18 | discard await provider.send("evm_mine") 19 | -------------------------------------------------------------------------------- /tests/coverage.nim: -------------------------------------------------------------------------------- 1 | include ./testCodex 2 | -------------------------------------------------------------------------------- /tests/coverage.nims: -------------------------------------------------------------------------------- 1 | switch("debugger", "native") 2 | switch("lineDir", "on") 3 | switch("define", "debug") 4 | switch("verbosity", "0") 5 | switch("hints", "off") 6 | switch("warnings", "off") 7 | # switches for compiling with coverage 8 | switch("passC", "-fprofile-arcs") 9 | switch("passC", "-ftest-coverage") 10 | switch("passL", "-fprofile-arcs") 11 | switch("passL", "-ftest-coverage") 12 | -------------------------------------------------------------------------------- /tests/ethertest.nim: -------------------------------------------------------------------------------- 1 | import std/json 2 | import pkg/ethers 3 | import pkg/chronos 4 | 5 | import ./asynctest 6 | import ./checktest 7 | 8 | ## Unit testing suite that sets up an Ethereum testing environment. 9 | ## Injects a `ethProvider` instance, and a list of `accounts`. 10 | ## Calls the `evm_snapshot` and `evm_revert` methods to ensure that any 11 | ## changes to the blockchain do not persist. 12 | template ethersuite*(name, body) = 13 | asyncchecksuite name: 14 | var ethProvider {.inject, used.}: JsonRpcProvider 15 | var accounts {.inject, used.}: seq[Address] 16 | var snapshot: JsonNode 17 | 18 | setup: 19 | ethProvider = JsonRpcProvider.new("ws://localhost:8545") 20 | snapshot = await send(ethProvider, "evm_snapshot") 21 | accounts = await ethProvider.listAccounts() 22 | teardown: 23 | discard await send(ethProvider, "evm_revert", @[snapshot]) 24 | 25 | await ethProvider.close() 26 | 27 | body 28 | 29 | export asynctest 30 | export ethers except `%` 31 | -------------------------------------------------------------------------------- /tests/fixtures/test.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codex-storage/nim-codex/3e17207a0b703a66001e7bd8e0bf97d226c119e5/tests/fixtures/test.jpg -------------------------------------------------------------------------------- /tests/helpers.nim: -------------------------------------------------------------------------------- 1 | import helpers/multisetup 2 | import helpers/trackers 3 | import helpers/templeveldb 4 | import std/times 5 | import std/sequtils, chronos 6 | 7 | export multisetup, trackers, templeveldb 8 | 9 | ### taken from libp2p errorhelpers.nim 10 | proc allFuturesThrowing*(args: varargs[FutureBase]): Future[void] = 11 | # This proc is only meant for use in tests / not suitable for general use. 12 | # - Swallowing errors arbitrarily instead of aggregating them is bad design 13 | # - It raises `CatchableError` instead of the union of the `futs` errors, 14 | # inflating the caller's `raises` list unnecessarily. `macro` could fix it 15 | let futs = @args 16 | ( 17 | proc() {.async: (raises: [CatchableError]).} = 18 | await allFutures(futs) 19 | var firstErr: ref CatchableError 20 | for fut in futs: 21 | if fut.failed: 22 | let err = fut.error() 23 | if err of CancelledError: 24 | raise err 25 | if firstErr == nil: 26 | firstErr = err 27 | if firstErr != nil: 28 | raise firstErr 29 | )() 30 | 31 | proc allFuturesThrowing*[T](futs: varargs[Future[T]]): Future[void] = 32 | allFuturesThrowing(futs.mapIt(FutureBase(it))) 33 | 34 | proc allFuturesThrowing*[T, E]( # https://github.com/nim-lang/Nim/issues/23432 35 | futs: varargs[InternalRaisesFuture[T, E]] 36 | ): Future[void] = 37 | allFuturesThrowing(futs.mapIt(FutureBase(it))) 38 | -------------------------------------------------------------------------------- /tests/helpers/multisetup.nim: -------------------------------------------------------------------------------- 1 | import pkg/chronos 2 | 3 | # Allow multiple setups and teardowns in a test suite 4 | template asyncmultisetup*() = 5 | var setups: seq[proc(): Future[void].Raising([AsyncExceptionError]) {.gcsafe.}] 6 | var teardowns: seq[proc(): Future[void].Raising([AsyncExceptionError]) {.gcsafe.}] 7 | 8 | setup: 9 | for setup in setups: 10 | await setup() 11 | 12 | teardown: 13 | for teardown in teardowns: 14 | await teardown() 15 | 16 | template setup(setupBody) {.inject, used.} = 17 | setups.add( 18 | proc() {.async: (handleException: true, raises: [AsyncExceptionError]).} = 19 | setupBody 20 | ) 21 | 22 | template teardown(teardownBody) {.inject, used.} = 23 | teardowns.insert( 24 | proc() {.async: (handleException: true, raises: [AsyncExceptionError]).} = 25 | teardownBody 26 | ) 27 | 28 | template multisetup*() = 29 | var setups: seq[proc() {.gcsafe.}] 30 | var teardowns: seq[proc() {.gcsafe.}] 31 | 32 | setup: 33 | for setup in setups: 34 | setup() 35 | 36 | teardown: 37 | for teardown in teardowns: 38 | teardown() 39 | 40 | template setup(setupBody) {.inject, used.} = 41 | let setupProc = proc() = 42 | setupBody 43 | setups.add(setupProc) 44 | 45 | template teardown(teardownBody) {.inject, used.} = 46 | teardowns.insert( 47 | proc() = 48 | teardownBody 49 | ) 50 | -------------------------------------------------------------------------------- /tests/helpers/templeveldb.nim: -------------------------------------------------------------------------------- 1 | import os 2 | import std/monotimes 3 | import pkg/datastore 4 | import pkg/chronos 5 | import pkg/questionable/results 6 | 7 | type TempLevelDb* = ref object 8 | currentPath: string 9 | ds: LevelDbDatastore 10 | 11 | var number = 0 12 | 13 | proc newDb*(self: TempLevelDb): Datastore = 14 | if self.currentPath.len > 0: 15 | raiseAssert("TempLevelDb already active.") 16 | self.currentPath = getTempDir() / "templeveldb" / $number / $getMonoTime() 17 | inc number 18 | createDir(self.currentPath) 19 | self.ds = LevelDbDatastore.new(self.currentPath).tryGet() 20 | return self.ds 21 | 22 | proc destroyDb*(self: TempLevelDb): Future[void] {.async.} = 23 | if self.currentPath.len == 0: 24 | raiseAssert("TempLevelDb not active.") 25 | try: 26 | (await self.ds.close()).tryGet() 27 | finally: 28 | removeDir(self.currentPath) 29 | self.currentPath = "" 30 | -------------------------------------------------------------------------------- /tests/helpers/trackers.nim: -------------------------------------------------------------------------------- 1 | import pkg/codex/streams/storestream 2 | import pkg/unittest2 3 | 4 | # From lip2p/tests/helpers 5 | const trackerNames = [StoreStreamTrackerName] 6 | 7 | iterator testTrackers*(extras: openArray[string] = []): TrackerBase = 8 | for name in trackerNames: 9 | let t = getTracker(name) 10 | if not isNil(t): 11 | yield t 12 | for name in extras: 13 | let t = getTracker(name) 14 | if not isNil(t): 15 | yield t 16 | 17 | proc checkTracker*(name: string) = 18 | var tracker = getTracker(name) 19 | if tracker.isLeaked(): 20 | checkpoint tracker.dump() 21 | fail() 22 | 23 | proc checkTrackers*() = 24 | for tracker in testTrackers(): 25 | if tracker.isLeaked(): 26 | checkpoint tracker.dump() 27 | fail() 28 | try: 29 | GC_fullCollect() 30 | except: 31 | discard 32 | -------------------------------------------------------------------------------- /tests/integration/clioption.nim: -------------------------------------------------------------------------------- 1 | type CliOption* = object 2 | key*: string # option key, including `--` 3 | value*: string # option value 4 | 5 | proc `$`*(option: CliOption): string = 6 | var res = option.key 7 | if option.value.len > 0: 8 | res &= "=" & option.value 9 | return res 10 | -------------------------------------------------------------------------------- /tests/integration/codexprocess.nim: -------------------------------------------------------------------------------- 1 | import pkg/questionable 2 | import pkg/questionable/results 3 | import pkg/confutils 4 | import pkg/chronicles 5 | import pkg/chronos/asyncproc 6 | import pkg/ethers 7 | import pkg/libp2p 8 | import std/os 9 | import std/strutils 10 | import codex/conf 11 | import ./codexclient 12 | import ./nodeprocess 13 | 14 | export codexclient 15 | export chronicles 16 | export nodeprocess 17 | 18 | logScope: 19 | topics = "integration testing codex process" 20 | 21 | type CodexProcess* = ref object of NodeProcess 22 | client: ?CodexClient 23 | 24 | method workingDir(node: CodexProcess): string = 25 | return currentSourcePath() / ".." / ".." / ".." 26 | 27 | method executable(node: CodexProcess): string = 28 | return "build" / "codex" 29 | 30 | method startedOutput(node: CodexProcess): string = 31 | return "REST service started" 32 | 33 | method processOptions(node: CodexProcess): set[AsyncProcessOption] = 34 | return {AsyncProcessOption.StdErrToStdOut} 35 | 36 | method outputLineEndings(node: CodexProcess): string {.raises: [].} = 37 | return "\n" 38 | 39 | method onOutputLineCaptured(node: CodexProcess, line: string) {.raises: [].} = 40 | discard 41 | 42 | proc dataDir(node: CodexProcess): string = 43 | let config = CodexConf.load(cmdLine = node.arguments, quitOnFailure = false) 44 | return config.dataDir.string 45 | 46 | proc ethAccount*(node: CodexProcess): Address = 47 | let config = CodexConf.load(cmdLine = node.arguments, quitOnFailure = false) 48 | without ethAccount =? config.ethAccount: 49 | raiseAssert "eth account not set" 50 | return Address(ethAccount) 51 | 52 | proc apiUrl*(node: CodexProcess): string = 53 | let config = CodexConf.load(cmdLine = node.arguments, quitOnFailure = false) 54 | return "http://" & config.apiBindAddress & ":" & $config.apiPort & "/api/codex/v1" 55 | 56 | proc client*(node: CodexProcess): CodexClient = 57 | if client =? node.client: 58 | return client 59 | let client = CodexClient.new(node.apiUrl) 60 | node.client = some client 61 | return client 62 | 63 | method stop*(node: CodexProcess) {.async.} = 64 | logScope: 65 | nodeName = node.name 66 | 67 | await procCall NodeProcess(node).stop() 68 | 69 | trace "stopping codex client" 70 | if client =? node.client: 71 | await client.close() 72 | node.client = none CodexClient 73 | 74 | method removeDataDir*(node: CodexProcess) = 75 | removeDir(node.dataDir) 76 | -------------------------------------------------------------------------------- /tests/integration/hardhatconfig.nim: -------------------------------------------------------------------------------- 1 | type HardhatConfig* = object 2 | logFile*: bool 3 | debugEnabled*: bool 4 | 5 | proc debug*(self: HardhatConfig, enabled = true): HardhatConfig = 6 | ## output log in stdout 7 | var config = self 8 | config.debugEnabled = enabled 9 | return config 10 | 11 | proc withLogFile*(self: HardhatConfig, logToFile: bool = true): HardhatConfig = 12 | var config = self 13 | config.logFile = logToFile 14 | return config 15 | -------------------------------------------------------------------------------- /tests/integration/nodeconfig.nim: -------------------------------------------------------------------------------- 1 | import pkg/chronicles 2 | import pkg/questionable 3 | 4 | export chronicles 5 | 6 | type NodeConfig* = ref object of RootObj 7 | logFile*: bool 8 | logLevel*: ?LogLevel 9 | debugEnabled*: bool 10 | 11 | proc debug*[T: NodeConfig](config: T, enabled = true): T = 12 | ## output log in stdout 13 | var startConfig = config 14 | startConfig.debugEnabled = enabled 15 | return startConfig 16 | 17 | proc withLogFile*[T: NodeConfig](config: T, logToFile: bool = true): T = 18 | var startConfig = config 19 | startConfig.logFile = logToFile 20 | return startConfig 21 | 22 | proc withLogLevel*[T: NodeConfig](config: NodeConfig, level: LogLevel): T = 23 | var startConfig = config 24 | startConfig.logLevel = some level 25 | return startConfig 26 | -------------------------------------------------------------------------------- /tests/integration/nodeconfigs.nim: -------------------------------------------------------------------------------- 1 | import pkg/questionable 2 | import ./codexconfig 3 | import ./hardhatconfig 4 | 5 | type NodeConfigs* = object 6 | clients*: ?CodexConfigs 7 | providers*: ?CodexConfigs 8 | validators*: ?CodexConfigs 9 | hardhat*: ?HardhatConfig 10 | -------------------------------------------------------------------------------- /tests/integration/testblockexpiration.nim: -------------------------------------------------------------------------------- 1 | import ../examples 2 | import ./multinodes 3 | 4 | multinodesuite "Node block expiration tests": 5 | var content: seq[byte] 6 | 7 | setup: 8 | content = await RandomChunker.example(blocks = 8) 9 | 10 | test "node retains not-expired file", 11 | NodeConfigs( 12 | clients: CodexConfigs 13 | .init(nodes = 1) 14 | .withBlockTtl(0, 10) 15 | .withBlockMaintenanceInterval(0, 1).some, 16 | providers: CodexConfigs.none, 17 | ): 18 | let client = clients()[0] 19 | let clientApi = client.client 20 | 21 | let contentId = (await clientApi.upload(content)).get 22 | 23 | await sleepAsync(2.seconds) 24 | 25 | let download = await clientApi.download(contentId, local = true) 26 | 27 | check: 28 | download.isOk 29 | download.get == string.fromBytes(content) 30 | 31 | test "node deletes expired file", 32 | NodeConfigs( 33 | clients: CodexConfigs 34 | .init(nodes = 1) 35 | .withBlockTtl(0, 1) 36 | .withBlockMaintenanceInterval(0, 1).some, 37 | providers: CodexConfigs.none, 38 | ): 39 | let client = clients()[0] 40 | let clientApi = client.client 41 | 42 | let contentId = (await clientApi.upload(content)).get 43 | 44 | await sleepAsync(3.seconds) 45 | 46 | let download = await clientApi.download(contentId, local = true) 47 | 48 | check: 49 | download.isFailure 50 | download.error.msg == "404" 51 | -------------------------------------------------------------------------------- /tests/integration/testecbug.nim: -------------------------------------------------------------------------------- 1 | from pkg/libp2p import Cid, init 2 | import ../examples 3 | import ./marketplacesuite 4 | import ./nodeconfigs 5 | import ./hardhatconfig 6 | 7 | marketplacesuite "Bug #821 - node crashes during erasure coding": 8 | test "should be able to create storage request and download dataset", 9 | NodeConfigs( 10 | clients: CodexConfigs 11 | .init(nodes = 1) 12 | # .debug() # uncomment to enable console log output.debug() 13 | .withLogFile() 14 | # uncomment to output log file to tests/integration/logs/ //_.log 15 | .withLogTopics("node", "erasure", "marketplace").some, 16 | providers: CodexConfigs.init(nodes = 0).some, 17 | ): 18 | let 19 | pricePerBytePerSecond = 1.u256 20 | duration = 20.periods 21 | collateralPerByte = 1.u256 22 | expiry = 10.periods 23 | data = await RandomChunker.example(blocks = 8) 24 | client = clients()[0] 25 | clientApi = client.client 26 | 27 | let cid = (await clientApi.upload(data)).get 28 | 29 | var requestId = none RequestId 30 | proc onStorageRequested(eventResult: ?!StorageRequested) = 31 | assert not eventResult.isErr 32 | requestId = some (!eventResult).requestId 33 | 34 | let subscription = await marketplace.subscribe(StorageRequested, onStorageRequested) 35 | 36 | # client requests storage but requires multiple slots to host the content 37 | let id = await clientApi.requestStorage( 38 | cid, 39 | duration = duration, 40 | pricePerBytePerSecond = pricePerBytePerSecond, 41 | expiry = expiry, 42 | collateralPerByte = collateralPerByte, 43 | nodes = 3, 44 | tolerance = 1, 45 | ) 46 | 47 | check eventually(requestId.isSome, timeout = expiry.int * 1000) 48 | 49 | let 50 | request = await marketplace.getRequest(requestId.get) 51 | cidFromRequest = request.content.cid 52 | downloaded = await clientApi.downloadBytes(cidFromRequest, local = true) 53 | 54 | check downloaded.isOk 55 | check downloaded.get.toHex == data.toHex 56 | 57 | await subscription.unsubscribe() 58 | -------------------------------------------------------------------------------- /tests/integration/twonodes.nim: -------------------------------------------------------------------------------- 1 | import std/macros 2 | import pkg/questionable 3 | import ./multinodes 4 | import ./codexconfig 5 | import ./codexprocess 6 | import ./codexclient 7 | import ./nodeconfigs 8 | 9 | export codexclient 10 | export multinodes 11 | 12 | template twonodessuite*(name: string, body: untyped) = 13 | multinodesuite name: 14 | let twoNodesConfig {.inject, used.} = 15 | NodeConfigs(clients: CodexConfigs.init(nodes = 2).some) 16 | 17 | var node1 {.inject, used.}: CodexProcess 18 | var node2 {.inject, used.}: CodexProcess 19 | var client1 {.inject, used.}: CodexClient 20 | var client2 {.inject, used.}: CodexClient 21 | var account1 {.inject, used.}: Address 22 | var account2 {.inject, used.}: Address 23 | 24 | setup: 25 | account1 = accounts[0] 26 | account2 = accounts[1] 27 | 28 | node1 = clients()[0] 29 | node2 = clients()[1] 30 | 31 | client1 = node1.client 32 | client2 = node2.client 33 | 34 | body 35 | -------------------------------------------------------------------------------- /tests/logging.nim: -------------------------------------------------------------------------------- 1 | when not defined(nimscript): 2 | import pkg/codex/logutils 3 | 4 | proc ignoreLogging(level: LogLevel, message: LogOutputStr) = 5 | discard 6 | 7 | defaultChroniclesStream.output.writer = ignoreLogging 8 | 9 | {.warning[UnusedImport]: off.} 10 | {.used.} 11 | -------------------------------------------------------------------------------- /tests/testCodex.nim: -------------------------------------------------------------------------------- 1 | import ./codex/teststores 2 | import ./codex/testblockexchange 3 | import ./codex/testasyncheapqueue 4 | import ./codex/testchunking 5 | import ./codex/testlogutils 6 | import ./codex/testmanifest 7 | import ./codex/testnode 8 | import ./codex/teststorestream 9 | import ./codex/testpurchasing 10 | import ./codex/testsales 11 | import ./codex/testerasure 12 | import ./codex/testutils 13 | import ./codex/testclock 14 | import ./codex/testsystemclock 15 | import ./codex/testvalidation 16 | import ./codex/testasyncstreamwrapper 17 | import ./codex/testmerkletree 18 | import ./codex/testslots 19 | import ./codex/testindexingstrategy 20 | 21 | {.warning[UnusedImport]: off.} 22 | -------------------------------------------------------------------------------- /tests/testContracts.nim: -------------------------------------------------------------------------------- 1 | import ./contracts/testContracts 2 | import ./contracts/testMarket 3 | import ./contracts/testDeployment 4 | import ./contracts/testClock 5 | import ./contracts/testProvider 6 | 7 | {.warning[UnusedImport]: off.} 8 | -------------------------------------------------------------------------------- /tests/testIntegration.nim: -------------------------------------------------------------------------------- 1 | import ./integration/testcli 2 | import ./integration/testrestapi 3 | import ./integration/testrestapivalidation 4 | import ./integration/testupdownload 5 | import ./integration/testsales 6 | import ./integration/testpurchasing 7 | import ./integration/testblockexpiration 8 | import ./integration/testmarketplace 9 | import ./integration/testproofs 10 | import ./integration/testvalidator 11 | import ./integration/testecbug 12 | 13 | {.warning[UnusedImport]: off.} 14 | -------------------------------------------------------------------------------- /tests/testTools.nim: -------------------------------------------------------------------------------- 1 | import ./tools/cirdl/testcirdl 2 | 3 | {.warning[UnusedImport]: off.} 4 | -------------------------------------------------------------------------------- /tests/tools/cirdl/testcirdl.nim: -------------------------------------------------------------------------------- 1 | import std/os 2 | import std/osproc 3 | import std/options 4 | import pkg/chronos 5 | import pkg/codex/contracts 6 | import ../../asynctest 7 | import ../../contracts/deployment 8 | 9 | suite "tools/cirdl": 10 | const 11 | cirdl = "build" / "cirdl" 12 | workdir = "." 13 | 14 | test "circuit download tool": 15 | let 16 | circuitPath = "testcircuitpath" 17 | rpcEndpoint = "ws://localhost:8545" 18 | marketplaceAddress = Marketplace.address 19 | 20 | discard existsOrCreateDir(circuitPath) 21 | 22 | let args = [circuitPath, rpcEndpoint, $marketplaceAddress] 23 | 24 | let process = osproc.startProcess(cirdl, workdir, args, options = {poParentStreams}) 25 | 26 | let returnCode = process.waitForExit() 27 | check returnCode == 0 28 | 29 | check: 30 | fileExists(circuitPath / "proof_main_verification_key.json") 31 | fileExists(circuitPath / "proof_main.r1cs") 32 | fileExists(circuitPath / "proof_main.wasm") 33 | fileExists(circuitPath / "proof_main.zkey") 34 | 35 | removeDir(circuitPath) 36 | -------------------------------------------------------------------------------- /tools/scripts/git_pre_commit_format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "Formatting changed files with pre-commit hook" 4 | 5 | # Regexp for grep to only choose some file extensions for formatting 6 | exts="\.\(nim\|nims\)$" 7 | 8 | # Build nph lazily 9 | make build-nph || (1>&2 echo "failed to build nph. Pre-commit formatting will not be done."; exit 0) 10 | 11 | # Format staged files 12 | git diff --cached --name-only --diff-filter=ACMR | grep "$exts" | while read file; do 13 | echo "Formatting $file" 14 | make nph/"$file" 15 | git add "$file" 16 | done 17 | -------------------------------------------------------------------------------- /vendor/urls.rules: -------------------------------------------------------------------------------- 1 | https://github.com/status-im/nim-libp2p-dht.git -> https://github.com/codex-storage/nim-codex-dht.git 2 | https://github.com/markspanbroek/questionable -> https://github.com/codex-storage/questionable 3 | https://github.com/status-im/questionable -> https://github.com/codex-storage/questionable 4 | https://github.com/status-im/asynctest -> https://github.com/codex-storage/asynctest 5 | https://github.com/status-im/nim-datastore -> https://github.com/codex-storage/nim-datastore 6 | https://github.com/cheatfate/nimcrypto -> https://github.com/status-im/nimcrypto 7 | protobufserialization -> protobuf_serialization 8 | protobufserialization -> https://github.com/status-im/nim-protobuf-serialization 9 | --------------------------------------------------------------------------------