├── .cargo └── config.toml ├── .github ├── actionlint.yaml ├── e2e │ └── docker-compose.yml ├── release-please │ ├── config.json │ └── manifest.json ├── renovate.json └── workflows │ ├── e2e.yml │ ├── lint-pr.yml │ ├── lint.yml │ ├── release.yml │ ├── run-tests.yml │ ├── snapshot.yml │ └── tests.yml ├── .gitignore ├── .prettierignore ├── API_reference.md ├── CHANGELOG.md ├── CONTRIBUTING.md ├── INSTALL.md ├── LICENSE ├── README.md ├── aqua-tests ├── .fluence │ ├── aqua-dependencies │ │ └── package-lock.json │ ├── env.yaml │ └── secrets │ │ ├── test_create_resource.txt │ │ ├── test_get_resource.txt │ │ ├── test_register_record_unregister.txt │ │ └── test_register_unregister_remote_record.txt ├── .gitignore ├── README.md ├── aqua │ └── test.aqua ├── config.py ├── fluence.yaml ├── getDefaultPeers.js ├── package-lock.json ├── package.json ├── requirements.txt ├── spell │ ├── spell.aqua │ └── spell.yaml ├── test_aqua.py └── test_fluence_cli_version.py ├── aqua ├── constants.aqua ├── misc.aqua ├── package-lock.json ├── package.json ├── registry-api.aqua ├── registry-scheduled-scripts.aqua ├── registry-service.aqua ├── resources-api.aqua └── target │ └── typescript │ ├── misc.ts │ ├── registry-api.ts │ └── registry-scheduled-scripts.ts ├── build.sh ├── distro ├── Cargo.lock ├── Cargo.toml ├── Config.toml ├── built.rs └── src │ └── lib.rs ├── examples └── archived │ └── 1-registry │ ├── .fluence │ ├── aqua │ │ ├── deals.aqua │ │ ├── hosts.aqua │ │ └── services.aqua │ ├── configs │ │ ├── nox-0_Config.toml │ │ ├── nox-1_Config.toml │ │ └── nox-2_Config.toml │ ├── docker-compose.yaml │ └── workers.yaml │ ├── .gitignore │ ├── .vscode │ └── extensions.json │ ├── Cargo.lock │ ├── Cargo.toml │ ├── README.md │ ├── fluence.yaml │ ├── provider.yaml │ └── src │ ├── aqua │ └── main.aqua │ ├── frontend │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── echo.ts │ │ └── relays.json │ └── tsconfig.json │ └── services │ └── echo_service │ ├── modules │ └── echo_service │ │ ├── Cargo.toml │ │ ├── module.yaml │ │ └── src │ │ └── main.rs │ └── service.yaml ├── images ├── availability.png ├── decentralized.png ├── discovery.png ├── mapping.png ├── registry.png └── subnetwork.png ├── rust-toolchain.toml └── service ├── Cargo.lock ├── Cargo.toml ├── Config.toml ├── README.md ├── build.rs ├── build.sh ├── clippy.toml └── src ├── config.rs ├── defaults.rs ├── error.rs ├── key.rs ├── key_api.rs ├── key_storage_impl.rs ├── main.rs ├── misc.rs ├── record.rs ├── record_api.rs ├── record_storage_impl.rs ├── results.rs ├── storage_impl.rs ├── tests └── mod.rs ├── tetraplets_checkers.rs ├── tombstone.rs ├── tombstone_api.rs └── tombstone_storage_impl.rs /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [registries] 2 | fluence = { index = "git://crates.fluence.dev/index" } 3 | -------------------------------------------------------------------------------- /.github/actionlint.yaml: -------------------------------------------------------------------------------- 1 | self-hosted-runner: 2 | labels: 3 | - builder 4 | -------------------------------------------------------------------------------- /.github/e2e/docker-compose.yml: -------------------------------------------------------------------------------- 1 | networks: 2 | nox: 3 | driver: bridge 4 | ipam: 5 | config: 6 | - subnet: 10.50.10.0/24 7 | 8 | services: 9 | nox-1: 10 | image: ${NOX_IMAGE} 11 | ports: 12 | - 7771:7771 13 | - 9991:9991 14 | command: 15 | - --aqua-pool-size=2 16 | - -t=7771 17 | - -w=9991 18 | - -x=10.50.10.10 19 | - --external-maddrs 20 | - /dns4/nox-1/tcp/7771 21 | - /dns4/nox-1/tcp/9991/ws 22 | - --allow-private-ips 23 | - --local 24 | # - --bootstraps=/dns/nox-1/tcp/7771 25 | # 12D3KooWBM3SdXWqGaawQDGQ6JprtwswEg3FWGvGhmgmMez1vRbR 26 | - -k=hK62afickoeP2uZbmSkAYXxxqP8ozq16VRN7qfTP719EHC5V5tjrtW57BSjUr8GvsEXmJRbtejUWyPZ2rZMyQdq 27 | networks: 28 | nox: 29 | ipv4_address: 10.50.10.10 30 | 31 | nox-2: 32 | image: ${NOX_IMAGE} 33 | ports: 34 | - 7772:7772 35 | - 9992:9992 36 | command: 37 | - --aqua-pool-size=2 38 | - -t=7772 39 | - -w=9992 40 | - -x=10.50.10.20 41 | - --external-maddrs 42 | - /dns4/nox-2/tcp/7772 43 | - /dns4/nox-2/tcp/9992/ws 44 | - --allow-private-ips 45 | - --bootstraps=/dns/nox-1/tcp/7771 46 | # 12D3KooWQdpukY3p2DhDfUfDgphAqsGu5ZUrmQ4mcHSGrRag6gQK 47 | - -k=2WijTVdhVRzyZamWjqPx4V4iNMrajegNMwNa2PmvPSZV6RRpo5M2fsPWdQr22HVRubuJhhSw8BrWiGt6FPhFAuXy 48 | networks: 49 | nox: 50 | ipv4_address: 10.50.10.20 51 | 52 | nox-3: 53 | image: ${NOX_IMAGE} 54 | ports: 55 | - 7773:7773 56 | - 9993:9993 57 | command: 58 | - --aqua-pool-size=2 59 | - -t=7773 60 | - -w=9993 61 | - -x=10.50.10.30 62 | - --external-maddrs 63 | - /dns4/nox-3/tcp/7773 64 | - /dns4/nox-3/tcp/9993/ws 65 | - --allow-private-ips 66 | - --bootstraps=/dns/nox-1/tcp/7771 67 | # 12D3KooWRT8V5awYdEZm6aAV9HWweCEbhWd7df4wehqHZXAB7yMZ 68 | - -k=2n2wBVanBeu2GWtvKBdrYK9DJAocgG3PrTUXMharq6TTfxqTL4sLdXL9BF23n6rsnkAY5pR9vBtx2uWYDQAiZdrX 69 | networks: 70 | nox: 71 | ipv4_address: 10.50.10.30 72 | 73 | nox-4: 74 | image: ${NOX_IMAGE} 75 | ports: 76 | - 7774:7774 77 | - 9994:9994 78 | command: 79 | - --aqua-pool-size=2 80 | - -t=7774 81 | - -w=9994 82 | - -x=10.50.10.40 83 | - --external-maddrs 84 | - /dns4/nox-4/tcp/7774 85 | - /dns4/nox-4/tcp/9994/ws 86 | - --allow-private-ips 87 | - --bootstraps=/dns/nox-1/tcp/7771 88 | # 12D3KooWBzLSu9RL7wLP6oUowzCbkCj2AGBSXkHSJKuq4wwTfwof 89 | - -k=4zp8ucAikkjB8CmkufYiFBW4QCDUCbQG7yMjviX7W8bMyN5rfChQ2Pi5QCWThrCTbAm9uq5nbFbxtFcNZq3De4dX 90 | networks: 91 | nox: 92 | ipv4_address: 10.50.10.40 93 | 94 | nox-5: 95 | image: ${NOX_IMAGE} 96 | ports: 97 | - 7775:7775 98 | - 9995:9995 99 | command: 100 | - --aqua-pool-size=2 101 | - -t=7775 102 | - -w=9995 103 | - -x=10.50.10.50 104 | - --external-maddrs 105 | - /dns4/nox-5/tcp/7775 106 | - /dns4/nox-5/tcp/9995/ws 107 | - --allow-private-ips 108 | - --bootstraps=/dns/nox-1/tcp/7771 109 | # 12D3KooWBf6hFgrnXwHkBnwPGMysP3b1NJe5HGtAWPYfwmQ2MBiU 110 | - -k=3ry26rm5gkJXvdqRH4FoM3ezWq4xVVsBQF7wtKq4E4pbuaa6p1F84tNqifUS7DdfJL9hs2gcdW64Wc342vHZHMUp 111 | networks: 112 | nox: 113 | ipv4_address: 10.50.10.50 114 | 115 | nox-6: 116 | image: ${NOX_IMAGE} 117 | ports: 118 | - 7776:7776 119 | - 9996:9996 120 | command: 121 | - --aqua-pool-size=2 122 | - -t=7776 123 | - -w=9996 124 | - --bootstraps=/dns/nox-1/tcp/7771 125 | - -x=10.50.10.60 126 | - --external-maddrs 127 | - /dns4/nox-6/tcp/7776 128 | - /dns4/nox-6/tcp/9996/ws 129 | - --allow-private-ips 130 | # 12D3KooWPisGn7JhooWhggndz25WM7vQ2JmA121EV8jUDQ5xMovJ 131 | - -k=5Qh8bB1sF28uLPwr3HTvEksCeC6mAWQvebCfcgv9y6j4qKwSzNKm2tzLUg4nACUEo2KZpBw11gNCnwaAdM7o1pEn 132 | networks: 133 | nox: 134 | ipv4_address: 10.50.10.60 135 | -------------------------------------------------------------------------------- /.github/release-please/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "bootstrap-sha": "1126e01580c2b91674a6c016844e5b1b8c5b2b14", 3 | "release-type": "simple", 4 | "bump-minor-pre-major": true, 5 | "bump-patch-for-minor-pre-major": true, 6 | "packages": { 7 | ".": { 8 | "package-name": "registry", 9 | "component": "registry" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /.github/release-please/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | ".": "0.9.4" 3 | } 4 | -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base", 5 | ":semanticCommitTypeAll(chore)" 6 | ], 7 | "enabledManagers": ["cargo", "npm", "github-actions", "pip_requirements"], 8 | "rangeStrategy": "pin", 9 | "schedule": "every weekend", 10 | "respectLatest": false, 11 | "packageRules": [ 12 | { 13 | "matchManagers": ["cargo", "npm"], 14 | "matchPackagePatterns": [ 15 | "@fluencelabs/.*", 16 | "fluence-.*", 17 | "marine-.*" 18 | ], 19 | "semanticCommitType": "fix", 20 | "semanticCommitScope": "deps", 21 | "schedule": "at any time" 22 | }, 23 | { 24 | "matchManagers": ["cargo"], 25 | "groupName": "marine things", 26 | "matchPackagePatterns": [ 27 | "marine-rs-sdk", 28 | "marine-rs-sdk-test", 29 | "marine-sqlite-connector" 30 | ], 31 | }, 32 | { 33 | "matchDepTypes": ["devDependencies"], 34 | "prPriority": -1, 35 | "semanticCommitType": "chore", 36 | "semanticCommitScope": "deps" 37 | }, 38 | { 39 | "matchUpdateTypes": ["major"], 40 | "prConcurrentLimit": 1 41 | }, 42 | { 43 | "matchManagers": ["github-actions"], 44 | "groupName": "all github-actions", 45 | "prPriority": -1 46 | } 47 | ] 48 | } 49 | -------------------------------------------------------------------------------- /.github/workflows/e2e.yml: -------------------------------------------------------------------------------- 1 | name: "e2e" 2 | 3 | on: 4 | pull_request: 5 | paths-ignore: 6 | - "**.md" 7 | - ".github/**" 8 | - "!.github/workflows/e2e.yml" 9 | - "!.github/workflows/snapshot.yml" 10 | - "!.github/workflows/tests.yml" 11 | types: 12 | - "labeled" 13 | - "synchronize" 14 | - "opened" 15 | - "reopened" 16 | push: 17 | branches: 18 | - "main" 19 | paths-ignore: 20 | - "**.md" 21 | - ".github/**" 22 | - "!.github/workflows/e2e.yml" 23 | - "!.github/workflows/snapshot.yml" 24 | - "!.github/workflows/tests.yml" 25 | 26 | concurrency: 27 | group: "${{ github.workflow }}-${{ github.ref }}" 28 | cancel-in-progress: true 29 | 30 | jobs: 31 | snapshot: 32 | if: > 33 | github.event_name == 'push' || 34 | contains(github.event.pull_request.labels.*.name, 'e2e') 35 | name: "registry" 36 | uses: ./.github/workflows/snapshot.yml 37 | with: 38 | ref: ${{ github.ref }} 39 | 40 | nox: 41 | needs: 42 | - snapshot 43 | uses: fluencelabs/nox/.github/workflows/build.yml@master 44 | with: 45 | cargo-dependencies: | 46 | [ 47 | { 48 | "package": "registry-distro", 49 | "version": "=${{ needs.snapshot.outputs.cargo-version }}", 50 | "registry": "fluence", 51 | "manifest": "crates/system-services/Cargo.toml" 52 | } 53 | ] 54 | 55 | nox-snapshot: 56 | name: "nox" 57 | needs: 58 | - nox 59 | 60 | uses: fluencelabs/nox/.github/workflows/container.yml@master 61 | with: 62 | image-name: "docker.fluence.dev/registry" 63 | 64 | aqua-tests: 65 | name: "registry" 66 | needs: 67 | - nox-snapshot 68 | uses: ./.github/workflows/tests.yml 69 | with: 70 | ref: ${{ github.ref }} 71 | nox-image: "${{ needs.nox-snapshot.outputs.nox-image }}" 72 | if-no-artifacts-found: warn 73 | -------------------------------------------------------------------------------- /.github/workflows/lint-pr.yml: -------------------------------------------------------------------------------- 1 | name: lint PR 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - opened 7 | - edited 8 | - synchronize 9 | 10 | concurrency: 11 | group: "${{ github.workflow }}-${{ github.ref }}" 12 | cancel-in-progress: true 13 | 14 | jobs: 15 | pr: 16 | name: Validate PR title 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: amannn/action-semantic-pull-request@v5 20 | env: 21 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 22 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - ".github/workflows/**" 7 | - ".github/renovate.json" 8 | 9 | concurrency: 10 | group: "${{ github.workflow }}-${{ github.ref }}" 11 | cancel-in-progress: true 12 | 13 | jobs: 14 | reviewdog: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout 18 | uses: actions/checkout@v4 19 | 20 | - name: Lint actions 21 | uses: reviewdog/action-actionlint@v1 22 | env: 23 | SHELLCHECK_OPTS: "-e SC2086 -e SC2207 -e SC2128" 24 | with: 25 | reporter: github-pr-check 26 | fail_on_error: true 27 | 28 | renovate: 29 | runs-on: ubuntu-latest 30 | steps: 31 | - name: Checkout 32 | uses: actions/checkout@v4 33 | 34 | - name: Renovate Config Validator 35 | uses: tj-actions/renovate-config-validator@v2 36 | with: 37 | config_file: .github/renovate.json 38 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: "release" 2 | 3 | on: 4 | push: 5 | branches: 6 | - "main" 7 | 8 | concurrency: 9 | group: "${{ github.workflow }}-${{ github.ref }}" 10 | 11 | jobs: 12 | release-please: 13 | runs-on: ubuntu-latest 14 | 15 | outputs: 16 | release-created: ${{ steps.release.outputs['release_created'] }} 17 | tag-name: ${{ steps.release.outputs['tag_name'] }} 18 | version: ${{ steps.release.outputs['version'] }} 19 | pr: ${{ steps.release.outputs['pr'] }} 20 | 21 | steps: 22 | - name: Run release-please 23 | id: release 24 | uses: google-github-actions/release-please-action@v3 25 | with: 26 | token: ${{ secrets.FLUENCEBOT_RELEASE_PLEASE_PAT }} 27 | command: manifest 28 | config-file: .github/release-please/config.json 29 | manifest-file: .github/release-please/manifest.json 30 | 31 | - name: Show output from release-please 32 | if: steps.release.outputs.releases_created 33 | env: 34 | RELEASE_PLEASE_OUTPUT: ${{ toJSON(steps.release.outputs) }} 35 | run: echo "${RELEASE_PLEASE_OUTPUT}" | jq 36 | 37 | bump-version: 38 | if: needs.release-please.outputs.pr != null 39 | runs-on: builder 40 | needs: 41 | - release-please 42 | 43 | permissions: 44 | contents: write 45 | 46 | steps: 47 | - name: Checkout 48 | uses: actions/checkout@v4 49 | with: 50 | ref: ${{ fromJson(needs.release-please.outputs.pr).headBranchName }} 51 | 52 | - name: Setup Rust toolchain 53 | uses: dsherret/rust-toolchain-file@v1 54 | 55 | - name: Install cargo-edit 56 | uses: baptiste0928/cargo-install@v2.2.0 57 | with: 58 | crate: cargo-edit 59 | 60 | - name: Get version 61 | id: version 62 | run: | 63 | version="$(jq -r '.[]' .github/release-please/manifest.json)" 64 | echo "version=${version}" >> $GITHUB_OUTPUT 65 | 66 | - name: Set version in service 67 | working-directory: service 68 | run: cargo set-version ${{ steps.version.outputs.version }} 69 | 70 | - name: Set version in distro 71 | working-directory: distro 72 | run: cargo set-version ${{ steps.version.outputs.version }} 73 | 74 | - name: Setup node 75 | uses: actions/setup-node@v3 76 | with: 77 | node-version: "18" 78 | 79 | - name: Set version in aqua 80 | working-directory: aqua 81 | run: npm version ${{ steps.version.outputs.version }} 82 | 83 | - name: Commit version bump 84 | uses: stefanzweifel/git-auto-commit-action@v4 85 | with: 86 | commit_message: "chore: Bump registry version to ${{ steps.version.outputs.version }}" 87 | branch: ${{ fromJson(needs.release-please.outputs.pr).headBranchName }} 88 | commit_user_name: fluencebot 89 | commit_user_email: devops@fluence.one 90 | commit_author: fluencebot 91 | 92 | registry: 93 | runs-on: builder 94 | 95 | needs: release-please 96 | if: needs.release-please.outputs.release-created 97 | 98 | permissions: 99 | contents: write 100 | id-token: write 101 | 102 | steps: 103 | - name: Checkout 104 | uses: actions/checkout@v4 105 | 106 | - name: Import secrets 107 | uses: hashicorp/vault-action@v2.7.3 108 | with: 109 | url: https://vault.fluence.dev 110 | path: jwt/github 111 | role: ci 112 | method: jwt 113 | jwtGithubAudience: "https://github.com/fluencelabs" 114 | jwtTtl: 300 115 | exportToken: false 116 | secrets: | 117 | kv/npmjs/fluencebot token | NODE_AUTH_TOKEN 118 | 119 | - name: Setup Rust toolchain 120 | uses: dsherret/rust-toolchain-file@v1 121 | 122 | - name: Setup marine 123 | uses: fluencelabs/setup-marine@v1 124 | 125 | - name: Setup fcli 126 | uses: fluencelabs/setup-fluence@v1 127 | with: 128 | version: main 129 | 130 | - name: Build service 131 | run: ./build.sh 132 | working-directory: service 133 | 134 | - name: Setup node 135 | uses: actions/setup-node@v3 136 | with: 137 | node-version: "18" 138 | registry-url: "https://registry.npmjs.org" 139 | cache-dependency-path: "aqua/package-lock.json" 140 | cache: "npm" 141 | 142 | - run: npm i 143 | working-directory: aqua 144 | 145 | - run: npm run build 146 | working-directory: aqua 147 | 148 | - name: Publish to NPM registry 149 | run: npm publish --access public 150 | working-directory: aqua 151 | 152 | - name: Import secrets 153 | uses: hashicorp/vault-action@v2.7.3 154 | with: 155 | url: https://vault.fluence.dev 156 | path: jwt/github 157 | role: ci 158 | method: jwt 159 | jwtGithubAudience: "https://github.com/fluencelabs" 160 | jwtTtl: 300 161 | exportToken: false 162 | secrets: | 163 | kv/crates.io/fluencebot token | CARGO_REGISTRY_TOKEN 164 | 165 | - name: Setup Rust toolchain 166 | uses: actions-rust-lang/setup-rust-toolchain@v1 167 | 168 | - name: Install cargo-workspaces 169 | uses: baptiste0928/cargo-install@v2.2.0 170 | with: 171 | crate: cargo-workspaces 172 | 173 | - name: Build distro 174 | run: ./build.sh 175 | 176 | - name: Publish to crates.io 177 | working-directory: ./distro 178 | run: | 179 | cargo ws publish \ 180 | --no-git-commit \ 181 | --allow-dirty \ 182 | --from-git \ 183 | --skip-published \ 184 | --yes 185 | 186 | slack: 187 | if: always() 188 | name: "Notify" 189 | runs-on: ubuntu-latest 190 | 191 | needs: 192 | - release-please 193 | - registry 194 | 195 | permissions: 196 | contents: read 197 | id-token: write 198 | 199 | steps: 200 | - uses: lwhiteley/dependent-jobs-result-check@v1 201 | id: status 202 | with: 203 | statuses: failure 204 | dependencies: ${{ toJSON(needs) }} 205 | 206 | - name: Log output 207 | run: | 208 | echo "statuses:" "${{ steps.status.outputs.statuses }}" 209 | echo "jobs:" "${{ steps.status.outputs.jobs }}" 210 | echo "found any?:" "${{ steps.status.outputs.found }}" 211 | 212 | - name: Import secrets 213 | uses: hashicorp/vault-action@v2.7.3 214 | with: 215 | url: https://vault.fluence.dev 216 | path: jwt/github 217 | role: ci 218 | method: jwt 219 | jwtGithubAudience: "https://github.com/fluencelabs" 220 | jwtTtl: 300 221 | exportToken: false 222 | secrets: | 223 | kv/slack/release-please webhook | SLACK_WEBHOOK_URL 224 | 225 | - uses: ravsamhq/notify-slack-action@v2 226 | if: steps.status.outputs.found == 'true' 227 | with: 228 | status: "failure" 229 | notification_title: "*{workflow}* has {status_message}" 230 | message_format: "${{ steps.status.outputs.jobs }} {status_message} in <{repo_url}|{repo}>" 231 | footer: "<{run_url}>" 232 | -------------------------------------------------------------------------------- /.github/workflows/run-tests.yml: -------------------------------------------------------------------------------- 1 | name: "test" 2 | 3 | on: 4 | pull_request: 5 | paths-ignore: 6 | - "**.md" 7 | - ".github/**" 8 | - "!.github/workflows/run-tests.yml" 9 | - "!.github/workflows/tests.yml" 10 | - "!.github/workflows/e2e.yml" 11 | types: 12 | - "labeled" 13 | - "synchronize" 14 | - "opened" 15 | - "reopened" 16 | push: 17 | branches: 18 | - "main" 19 | paths-ignore: 20 | - "**.md" 21 | - ".github/**" 22 | - "!.github/workflows/run-tests.yml" 23 | - "!.github/workflows/tests.yml" 24 | - "!.github/workflows/e2e.yml" 25 | 26 | concurrency: 27 | group: "${{ github.workflow }}-${{ github.ref }}" 28 | cancel-in-progress: true 29 | 30 | jobs: 31 | cargo: 32 | name: "registry / Run cargo tests" 33 | runs-on: builder 34 | timeout-minutes: 60 35 | 36 | defaults: 37 | run: 38 | working-directory: service 39 | 40 | steps: 41 | - uses: actions/checkout@v4 42 | 43 | - name: Setup rust toolchain 44 | uses: actions-rust-lang/setup-rust-toolchain@v1 45 | with: 46 | cache: false 47 | 48 | - name: Setup marine 49 | uses: fluencelabs/setup-marine@v1 50 | 51 | - name: Build service 52 | run: ./build.sh 53 | 54 | - name: Run cargo fmt 55 | run: cargo fmt --all -- --check 56 | 57 | - name: Run cargo clippy 58 | run: cargo clippy -Z unstable-options --all -- -D warnings 59 | 60 | - name: Install cargo-nextest 61 | uses: baptiste0928/cargo-install@v2.2.0 62 | with: 63 | crate: cargo-nextest 64 | version: 0.9.22 65 | 66 | - name: Run cargo nextest 67 | env: 68 | NEXTEST_RETRIES: 2 69 | NEXTEST_TEST_THREADS: 1 70 | run: cargo nextest run --release --all-features --no-fail-fast 71 | 72 | lints: 73 | name: Lints 74 | runs-on: ubuntu-latest 75 | 76 | steps: 77 | - name: Checkout sources 78 | uses: actions/checkout@v4 79 | 80 | - name: Setup rust toolchain 81 | uses: actions-rust-lang/setup-rust-toolchain@v1 82 | 83 | - name: Run cargo fmt 84 | uses: actions-rs/cargo@v1 85 | with: 86 | command: fmt 87 | args: --all --manifest-path service/Cargo.toml -- --check 88 | -------------------------------------------------------------------------------- /.github/workflows/snapshot.yml: -------------------------------------------------------------------------------- 1 | name: Build snapshot 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | cargo-dependencies: 7 | description: "Cargo dependencies map" 8 | type: string 9 | default: "null" 10 | ref: 11 | description: "git ref to checkout to" 12 | type: string 13 | default: "master" 14 | snapshot: 15 | description: "Whether to publish snapshots" 16 | type: boolean 17 | default: true 18 | outputs: 19 | cargo-version: 20 | description: "Cargo snapshot version" 21 | value: ${{ jobs.snapshot.outputs.version }} 22 | 23 | jobs: 24 | snapshot: 25 | name: "Build crate" 26 | runs-on: builder 27 | timeout-minutes: 60 28 | 29 | outputs: 30 | version: "${{ steps.snapshot.outputs.version }}" 31 | 32 | permissions: 33 | contents: read 34 | id-token: write 35 | 36 | steps: 37 | - uses: actions/checkout@v4 38 | with: 39 | repository: fluencelabs/registry 40 | ref: ${{ inputs.ref }} 41 | 42 | - name: Setup Rust toolchain 43 | uses: dsherret/rust-toolchain-file@v1 44 | 45 | - name: Set dependencies 46 | if: inputs.cargo-dependencies != 'null' 47 | uses: fluencelabs/github-actions/cargo-set-dependency@main 48 | with: 49 | dependencies: ${{ inputs.cargo-dependencies }} 50 | path: service/ 51 | 52 | - name: Setup marine 53 | uses: fluencelabs/setup-marine@v1 54 | with: 55 | artifact-name: marine 56 | 57 | - name: Setup fcli 58 | uses: fluencelabs/setup-fluence@v1 59 | with: 60 | version: main 61 | 62 | - name: Import secrets 63 | if: inputs.snapshot == true 64 | uses: hashicorp/vault-action@v2.7.3 65 | with: 66 | url: https://vault.fluence.dev 67 | path: jwt/github 68 | role: ci 69 | method: jwt 70 | jwtGithubAudience: "https://github.com/fluencelabs" 71 | jwtTtl: 300 72 | exportToken: false 73 | secrets: | 74 | kv/cargo-registry/users/ci token | CARGO_REGISTRIES_FLUENCE_TOKEN 75 | 76 | - name: Setup node 77 | uses: actions/setup-node@v3 78 | with: 79 | node-version: "18" 80 | registry-url: "https://registry.npmjs.org" 81 | cache-dependency-path: "aqua/package-lock.json" 82 | cache: "npm" 83 | 84 | - run: npm i 85 | working-directory: aqua 86 | 87 | - name: Install cargo-workspaces 88 | uses: baptiste0928/cargo-install@v2.2.0 89 | with: 90 | crate: cargo-workspaces 91 | 92 | - name: Generate snapshot version 93 | id: version 94 | uses: fluencelabs/github-actions/generate-snapshot-id@main 95 | 96 | - name: Build distro 97 | run: ./build.sh 98 | 99 | - name: Publish crate snapshots 100 | id: snapshot 101 | uses: fluencelabs/github-actions/cargo-publish-snapshot@main 102 | with: 103 | id: ${{ steps.version.outputs.id }} 104 | path: distro 105 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Run tests with workflow_call 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | fluence-env: 7 | description: "Fluence enviroment to run tests agains" 8 | type: string 9 | default: "local" 10 | nox-image: 11 | description: "nox image tag" 12 | type: string 13 | default: "fluencelabs/nox:unstable" 14 | fcli-version: 15 | description: "@fluencelabs/cli version" 16 | type: string 17 | default: "main" 18 | if-no-artifacts-found: 19 | description: "What to do when no artifacts found in setup-* actions" 20 | type: string 21 | default: "error" 22 | cargo-dependencies: 23 | description: "Cargo dependencies map" 24 | type: string 25 | ref: 26 | description: "git ref to checkout to" 27 | type: string 28 | default: "main" 29 | 30 | env: 31 | CI: true 32 | FORCE_COLOR: true 33 | NOX_IMAGE: "${{ inputs.nox-image }}" 34 | FLUENCE_ENV: "${{ inputs.fluence-env }}" 35 | 36 | jobs: 37 | aqua: 38 | name: "Run aqua tests" 39 | runs-on: builder 40 | timeout-minutes: 60 41 | 42 | permissions: 43 | contents: read 44 | id-token: write 45 | 46 | steps: 47 | - name: Import secrets 48 | uses: hashicorp/vault-action@v2.7.3 49 | with: 50 | url: https://vault.fluence.dev 51 | path: jwt/github 52 | role: ci 53 | method: jwt 54 | jwtGithubAudience: "https://github.com/fluencelabs" 55 | jwtTtl: 300 56 | secrets: | 57 | kv/docker-registry/basicauth/ci username | DOCKER_USERNAME ; 58 | kv/docker-registry/basicauth/ci password | DOCKER_PASSWORD ; 59 | kv/npm-registry/basicauth/ci token | NODE_AUTH_TOKEN; 60 | 61 | - name: Checkout registry 62 | uses: actions/checkout@v4 63 | with: 64 | repository: fluencelabs/registry 65 | ref: ${{ inputs.ref }} 66 | 67 | - name: Setup node with self-hosted registry 68 | uses: actions/setup-node@v3 69 | with: 70 | node-version: "18" 71 | registry-url: "https://npm.fluence.dev" 72 | cache: "npm" 73 | cache-dependency-path: "**/package-lock.json" 74 | 75 | - name: Run npm install in aqua 76 | run: npm install 77 | working-directory: aqua 78 | 79 | - name: Run npm install in aqua-tests 80 | run: npm install 81 | working-directory: aqua-tests 82 | 83 | - name: Setup fcli 84 | uses: fluencelabs/setup-fluence@v1 85 | with: 86 | artifact: fcli 87 | version: ${{ inputs.fcli-version }} 88 | if-no-artifact-found: ${{ inputs.if-no-artifacts-found }} 89 | 90 | - name: Setup Rust toolchain 91 | uses: dsherret/rust-toolchain-file@v1 92 | 93 | - name: Setup marine 94 | uses: fluencelabs/setup-marine@v1 95 | with: 96 | artifact-name: marine 97 | 98 | - name: Set dependencies 99 | if: inputs.cargo-dependencies != '' 100 | uses: fluencelabs/github-actions/cargo-set-dependency@main 101 | with: 102 | dependencies: ${{ inputs.cargo-dependencies }} 103 | path: service/ 104 | 105 | - name: Build service 106 | env: 107 | FLUENCE_USER_DIR: "${{ github.workspace }}/tmp/.fluence" 108 | run: ./build.sh 109 | working-directory: service 110 | 111 | - name: Build distro 112 | env: 113 | FLUENCE_USER_DIR: "${{ github.workspace }}/tmp/.fluence" 114 | run: ./build.sh 115 | 116 | - name: Login to DockerHub 117 | uses: docker/login-action@v3 118 | with: 119 | registry: docker.fluence.dev 120 | username: ${{ env.DOCKER_USERNAME }} 121 | password: ${{ env.DOCKER_PASSWORD }} 122 | 123 | - name: Pull nox image 124 | run: docker pull $NOX_IMAGE 125 | 126 | - name: Run nox network 127 | uses: isbang/compose-action@v1.4.1 128 | with: 129 | compose-file: ".github/e2e/docker-compose.yml" 130 | down-flags: "--volumes" 131 | 132 | - name: Setup python 133 | uses: actions/setup-python@v4 134 | with: 135 | python-version: "3.9" 136 | cache: "pip" 137 | cache-dependency-path: aqua-tests/requirements.txt 138 | 139 | - name: Install python requirements 140 | run: pip install -r requirements.txt 141 | working-directory: aqua-tests 142 | 143 | - name: Install fcli dependencies 144 | env: 145 | FLUENCE_USER_DIR: "${{ github.workspace }}/tmp/.fluence" 146 | run: fluence dep i --no-input 147 | working-directory: aqua-tests 148 | 149 | - name: Print fcli version 150 | run: pytest -s test_fluence_cli_version.py 151 | working-directory: aqua-tests 152 | 153 | - name: Run aqua tests 154 | env: 155 | FLUENCE_USER_DIR: "${{ github.workspace }}/tmp/.fluence" 156 | NPM_CONFIG_REGISTRY: "https://npm.fluence.dev" 157 | run: pytest test_aqua.py 158 | working-directory: aqua-tests 159 | 160 | - name: Print versions to check summary 161 | if: always() 162 | working-directory: aqua-tests 163 | run: | 164 | cat <> $GITHUB_STEP_SUMMARY 165 | ## Used versions 166 | \`\`\` 167 | $(fluence dep v) 168 | \`\`\` 169 | SNAPSHOT 170 | 171 | - name: Dump container logs 172 | if: always() 173 | uses: jwalton/gh-docker-logs@v2 174 | 175 | - name: Cleanup 176 | if: always() 177 | run: | 178 | rm -rf tmp ~/.fluence 179 | sudo rm -rf registry 180 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | service/target 2 | builtin-package/*.wasm 3 | builtin-package/scheduled/*.air 4 | registry.tar.gz 5 | registry 6 | distro/target/ 7 | distro/registry-service/ 8 | 9 | **/*.rs.bk 10 | **/.idea 11 | **/artifacts 12 | **/.DS_Store 13 | **/node_modules 14 | **/dist 15 | *.drawio 16 | 17 | aqua/*.tgz 18 | examples/src/generated/** 19 | 20 | **/__pycache__ 21 | tmp 22 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | .github 2 | -------------------------------------------------------------------------------- /API_reference.md: -------------------------------------------------------------------------------- 1 | # API Reference 2 | 3 | - [API Reference](#api-reference) 4 | - [Data structures](#data-structures) 5 | - [Key](#key) 6 | - [RecordMetadata](#recordmetadata) 7 | - [Record](#record) 8 | - [Tombstone](#tombstone) 9 | - [Resources API](#resources-api) 10 | - [Overview](#overview) 11 | - [Return types](#return-types) 12 | - [Methods](#methods) 13 | - [createResource](#createresource) 14 | - [getResource](#getresource) 15 | - [getResourceId](#getresourceid) 16 | - [registerService](#registerservice) 17 | - [unregisterService](#unregisterservice) 18 | - [resolveResource](#resolveresource) 19 | - [executeOnResource](#executeonresource) 20 | 21 | ## Data structures 22 | ### Key 23 | ```rust 24 | data Key { 25 | -- base58-encoded sha256(concat(label, owner_peer_id)) 26 | id: string, 27 | -- any unique string defined by the owner 28 | label: string, 29 | -- peer id in base58 30 | owner_peer_id: string, 31 | -- timestamp of creation in seconds 32 | timestamp_created: u64, 33 | -- challenge in bytes, will be used for permissions 34 | challenge: []u8, 35 | -- challenge type, will be used for permissions 36 | challenge_type: string, 37 | -- encoded and hashed previous fields signed by `owner_peer_id` 38 | signature: []u8, 39 | } 40 | ``` 41 | 42 | This data structure can be created via [`get_key_bytes`](#get_key_bytes) and [`register_key`](#register_key), and replicated via [`republish_key`](#republish_key). For now, there is no way to remove this structure, it can only be automatically garbage-collected via [`clear_expired`](#clear_expired). In the future updates, key tombstones will be implemented and it would be possible to remove key by an owner. 43 | 44 | In terms of Resources API Keys are Resources. 45 | ### RecordMetadata 46 | ```rust 47 | data RecordMetadata { 48 | -- base58-encoded key id 49 | key_id: string, 50 | -- peer id of the issuer in base58 51 | issued_by: string, 52 | -- peer_id of hoster 53 | peer_id: string, 54 | -- timestamp in seconds 55 | timestamp_issued: u64, 56 | -- will be used for permissions 57 | solution: []u8, 58 | -- any user-defined string 59 | value: string, 60 | -- optional (length is 0 or 1), base58 relay id 61 | relay_id: []string, 62 | -- optional (length is 0 or 1), advertising service id 63 | service_id: []string, 64 | -- encoded and hashed previous fields signed by `issued_by` 65 | issuer_signature: []u8, 66 | } 67 | ``` 68 | 69 | Metadata is the main part of the Record created by issuer that contains routing information, such as optional relay id, peer id and optional service id. Key identifier is a deterministic hash of the `label` and the `owner_peer_id`. 70 | 71 | ### Record 72 | ```rust 73 | data Record { 74 | -- record metadata 75 | metadata: RecordMetadata, 76 | -- timestamp in seconds 77 | timestamp_created: u64, 78 | -- encoded and hashed previous fields signed by `metadata.peer_id` 79 | signature: []u8, 80 | } 81 | ``` 82 | 83 | Record is maintained by `metadata.peer_id` via renewing of `timestamp_created` field automatically with scheduled scripts for full-featured peers and manually for other peers (Note: here and below we mean Rust peers as full-featured and JS/TS as others). Record can be removed by issuing a tombstone or become expired and then garbage-collected. Record owner is `metadata.issued_by`. 84 | 85 | ### Tombstone 86 | ```rust 87 | data Tombstone { 88 | -- base58-encoded key id 89 | key_id: string, 90 | -- peer id of the issuer in base58 91 | issued_by: string, 92 | -- peer_id of hoster 93 | peer_id: string, 94 | -- timestamp in seconds 95 | timestamp_issued: u64, 96 | -- will be used for permissions 97 | solution: []u8, 98 | -- encoded and hashed previous fields signed by `issued_by` 99 | issuer_signature: []u8, 100 | } 101 | ``` 102 | 103 | Tombstone is a special type of record that can be issued by record owner which eventually will substitute record with lower `timestamp_issued`. Tombstones replicated alongside with keys and records and live long enough to be sure that certain records will be deleted. Tombstones are garbage-collected automatically. 104 | 105 | In Resources API [`unregisterService`](#unregisterservice) method creates Tombstone. 106 | 107 | ## Resources API 108 | ### Overview 109 | Resources API is a high-level API for Registry network protocol. It uses Kademlia for the discovery of resources and service records. Resource and corresponding service Records are identified by Resource ID, and can be found in Registry services on peers in the Kademlia neighborhood of this Resource ID. 110 | 111 | ### Return types 112 | ``` 113 | alias ResourceId: string 114 | alias Resource: Key 115 | alias Error: string 116 | ``` 117 | 118 | Notes: 119 | - ResourceId is also an alias for key id in Resources API terminology. 120 | - Every method (except getResourceId) returns a stream of errors from different peers but success of execution is defined by first part of returned values. Optional types should be checked against `nil` to determine success of execution. 121 | 122 | ### Methods 123 | #### createResource 124 | ```rust 125 | func createResource(label: string) -> ?ResourceId, *Error: 126 | ``` 127 | 128 | Creates Resource with `label` and `INIT_PEER_ID` as owner. 129 | #### getResource 130 | ```rust 131 | func getResource(resource_id: ResourceId) -> ?Resource, *Error: 132 | ``` 133 | Returns resource by corresponding `resource_id`. 134 | 135 | #### getResourceId 136 | ```rust 137 | func getResourceId(label: string, peer_id: string) -> ResourceId: 138 | ``` 139 | 140 | Returns a deterministic hash of the `label` and the `peer_id`. 141 | #### registerService 142 | ```rust 143 | func registerService( 144 | resource_id: ResourceId, 145 | value: string, 146 | peer_id: PeerId, 147 | service_id: ?string 148 | ) -> bool, *Error: 149 | ``` 150 | 151 | Registers Record issued by `INIT_PEER_ID` for service on `peer_id`. `value` is any user-defined string. 152 | #### unregisterService 153 | ```rust 154 | func unregisterService(resource_id: ResourceId, peer_id: PeerId) -> bool, *Error: 155 | ``` 156 | 157 | Prevents the record issued by `INIT_PEER_ID` from being renewed and eventually removed. 158 | #### resolveResource 159 | ```rust 160 | func resolveResource(resource_id: ResourceId, ack: i16) -> ?[]Record, *Error: 161 | ``` 162 | 163 | Returns all records registered by this `resource_id`. `ack` is a minimal number of polled peers. 164 | 165 | #### executeOnResource 166 | ```rust 167 | func executeOnResource(resource_id: ResourceId, ack: i16, call: Record -> ()) -> bool, *Error: 168 | ``` 169 | 170 | Resolves all records by given `resource_id` and execites in parallel given callback. -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [0.9.4](https://github.com/fluencelabs/registry/compare/registry-v0.9.3...registry-v0.9.4) (2024-01-04) 4 | 5 | 6 | ### Features 7 | 8 | * **registry:** Use `aqua` keyword instead of `module` ([#313](https://github.com/fluencelabs/registry/issues/313)) ([b9bce2e](https://github.com/fluencelabs/registry/commit/b9bce2e7641d0431d8199d6a104f8c3d2fe3eee5)) 9 | 10 | 11 | ### Bug Fixes 12 | 13 | * **deps:** update dependency @fluencelabs/aqua-lib to v0.9.0 ([#328](https://github.com/fluencelabs/registry/issues/328)) ([5c9af8b](https://github.com/fluencelabs/registry/commit/5c9af8bd3a36493802b8e913d917e2fbd1621977)) 14 | * **deps:** update marine things ([#307](https://github.com/fluencelabs/registry/issues/307)) ([f78212d](https://github.com/fluencelabs/registry/commit/f78212d49bca9fe30def6702ec65aa187fe9deb1)) 15 | * **deps:** update rust crate fluence-keypair to v0.10.4 ([#318](https://github.com/fluencelabs/registry/issues/318)) ([b71b85c](https://github.com/fluencelabs/registry/commit/b71b85ca1eb0472176b78c237e421ec04418e0d9)) 16 | * **deps:** update sqlite wasm to 0.18.2 ([#320](https://github.com/fluencelabs/registry/issues/320)) ([7d9327b](https://github.com/fluencelabs/registry/commit/7d9327bcfd11c2dd63b360c96fed045f3f0952c3)) 17 | * **registry:** Revert release registry 0.9.4 ([#331](https://github.com/fluencelabs/registry/issues/331)) ([e9ba1ad](https://github.com/fluencelabs/registry/commit/e9ba1ad248418e3811fa8d7653545028b7e48127)) 18 | 19 | ## [0.9.3](https://github.com/fluencelabs/registry/compare/registry-v0.9.2...registry-v0.9.3) (2023-12-21) 20 | 21 | 22 | ### Features 23 | 24 | * use new aqua packages ([#310](https://github.com/fluencelabs/registry/issues/310)) ([633d8e6](https://github.com/fluencelabs/registry/commit/633d8e6648f344487da68b610857ee9837d0c081)) 25 | 26 | 27 | ### Bug Fixes 28 | 29 | * **deps:** update dependency @fluencelabs/trust-graph to v3.1.2 ([#189](https://github.com/fluencelabs/registry/issues/189)) ([7ba20dc](https://github.com/fluencelabs/registry/commit/7ba20dcabd9747256609e9b986bd63f47c94e691)) 30 | * **spell:** update spell api ([#315](https://github.com/fluencelabs/registry/issues/315)) ([3092907](https://github.com/fluencelabs/registry/commit/3092907e5e5d38caeeda15a83ea11e0462022f41)) 31 | 32 | ## [0.9.2](https://github.com/fluencelabs/registry/compare/registry-v0.9.1...registry-v0.9.2) (2023-12-19) 33 | 34 | 35 | ### Features 36 | 37 | * update marine sdk's, sqlite conector and config ([#309](https://github.com/fluencelabs/registry/issues/309)) ([863ae55](https://github.com/fluencelabs/registry/commit/863ae55f35bbe5452b636c064f9f8b377bb10ee8)) 38 | 39 | 40 | ### Bug Fixes 41 | 42 | * **ci:** setup fcli in release step ([#305](https://github.com/fluencelabs/registry/issues/305)) ([7b89267](https://github.com/fluencelabs/registry/commit/7b892678b1003bcf0c0fc834b7b49ceb2172e388)) 43 | * **deps:** update dependency @fluencelabs/aqua-lib to v0.8.1 ([#249](https://github.com/fluencelabs/registry/issues/249)) ([66a42f7](https://github.com/fluencelabs/registry/commit/66a42f7b935e82af9133e2d5bc2c864cb4296e2f)) 44 | * **deps:** update dependency @fluencelabs/aqua-lib to v0.8.2 ([#308](https://github.com/fluencelabs/registry/issues/308)) ([c207f7f](https://github.com/fluencelabs/registry/commit/c207f7fa549702c45dd8f25d0f97d95944472e6e)) 45 | * **deps:** update dependency @fluencelabs/trust-graph to v0.4.7 ([#257](https://github.com/fluencelabs/registry/issues/257)) ([a6aeeea](https://github.com/fluencelabs/registry/commit/a6aeeea3f5eb4f06a99ec272e0f5d3b4b0a2a8a7)) 46 | 47 | ## [0.9.1](https://github.com/fluencelabs/registry/compare/registry-v0.9.0...registry-v0.9.1) (2023-12-06) 48 | 49 | 50 | ### Features 51 | 52 | * use non-npm Fluence CLI ([#302](https://github.com/fluencelabs/registry/issues/302)) ([d77fd12](https://github.com/fluencelabs/registry/commit/d77fd12b4dfe2d57ae3e35f729e35e2f6ad1c63c)) 53 | 54 | 55 | ### Bug Fixes 56 | 57 | * **deps:** update dependency @fluencelabs/cli to v0.13.0 ([#290](https://github.com/fluencelabs/registry/issues/290)) ([2a440a8](https://github.com/fluencelabs/registry/commit/2a440a8b1ff8aa922bd2faa982b8b75c9beb3bc7)) 58 | * **deps:** update rust crate marine-rs-sdk-test to v0.11.1 ([#292](https://github.com/fluencelabs/registry/issues/292)) ([2405f41](https://github.com/fluencelabs/registry/commit/2405f41702543d1ff70620923787a6a7621cc7d5)) 59 | * remove binary import ([#304](https://github.com/fluencelabs/registry/issues/304)) ([c160475](https://github.com/fluencelabs/registry/commit/c16047515751f1400cb1f7231abcc83e2f6bcf4f)) 60 | 61 | ## [0.9.0](https://github.com/fluencelabs/registry/compare/registry-v0.8.8...registry-v0.9.0) (2023-11-22) 62 | 63 | 64 | ### ⚠ BREAKING CHANGES 65 | 66 | * **subnetwork:** deprecate registry-based subnets [NET-633] ([#283](https://github.com/fluencelabs/registry/issues/283)) 67 | 68 | ### Features 69 | 70 | * **subnetwork:** deprecate registry-based subnets [NET-633] ([#283](https://github.com/fluencelabs/registry/issues/283)) ([81f15d4](https://github.com/fluencelabs/registry/commit/81f15d4eb74b730fca331f1ea4ef6b960a02f9c8)) 71 | 72 | ## [0.8.8](https://github.com/fluencelabs/registry/compare/registry-v0.8.7...registry-v0.8.8) (2023-11-07) 73 | 74 | 75 | ### Features 76 | 77 | * prepare cli update ([#270](https://github.com/fluencelabs/registry/issues/270)) ([2c29fea](https://github.com/fluencelabs/registry/commit/2c29fea09808e2f98c4f58a10a1587aa5a571ad0)) 78 | * **registry:** Use streams instead of options [LNG-277] ([#282](https://github.com/fluencelabs/registry/issues/282)) ([19f5d47](https://github.com/fluencelabs/registry/commit/19f5d47add949f62085a022a01b84c83d3fc0389)) 79 | 80 | 81 | ### Bug Fixes 82 | 83 | * **ci:** use unstable nox image ([#255](https://github.com/fluencelabs/registry/issues/255)) ([257516e](https://github.com/fluencelabs/registry/commit/257516e74ff78807f78a7570ccc9e2d685af48f9)) 84 | * **deps:** unlock and update rust crate serde to 1.0.188 ([#273](https://github.com/fluencelabs/registry/issues/273)) ([4cb1b90](https://github.com/fluencelabs/registry/commit/4cb1b90a95bdc49b87b1dd1336e604cc71444de3)) 85 | * **deps:** Update cli to 0.11.0 ([#272](https://github.com/fluencelabs/registry/issues/272)) ([0ac1b76](https://github.com/fluencelabs/registry/commit/0ac1b76fe1c0635bfa5cf1105ffaf899db36b300)) 86 | * **deps:** update dependency @fluencelabs/cli ([#276](https://github.com/fluencelabs/registry/issues/276)) ([2259425](https://github.com/fluencelabs/registry/commit/22594259767fbd5be59904eab080d74733e7ea3e)) 87 | * **deps:** update dependency @fluencelabs/cli to v0.6.0 ([#238](https://github.com/fluencelabs/registry/issues/238)) ([be441e8](https://github.com/fluencelabs/registry/commit/be441e86cbc07a51636edfd07ec0fc80933b31cf)) 88 | * **deps:** update dependency @fluencelabs/fluence-network-environment to v1.1.2 ([#277](https://github.com/fluencelabs/registry/issues/277)) ([8ff086a](https://github.com/fluencelabs/registry/commit/8ff086a206d37edaeebe986661b626277e456d95)) 89 | * **deps:** update marine things ([#278](https://github.com/fluencelabs/registry/issues/278)) ([1f44cdc](https://github.com/fluencelabs/registry/commit/1f44cdc3b1188ef9daaba33a73ee85980c0c8bc6)) 90 | * **deps:** update rust crate marine-rs-sdk to v0.9.0 ([#265](https://github.com/fluencelabs/registry/issues/265)) ([9b4142d](https://github.com/fluencelabs/registry/commit/9b4142dc951414270f5a76b0519aa749c8835eb6)) 91 | 92 | ## [0.8.7](https://github.com/fluencelabs/registry/compare/registry-v0.8.6...registry-v0.8.7) (2023-06-20) 93 | 94 | 95 | ### Features 96 | 97 | * add distro crate [fixes NET-462] ([#233](https://github.com/fluencelabs/registry/issues/233)) ([5acf1d2](https://github.com/fluencelabs/registry/commit/5acf1d230b92f6b0784314b0926b6f6c2e195307)) 98 | * Migrate Registry to spell ([#247](https://github.com/fluencelabs/registry/issues/247)) ([990b588](https://github.com/fluencelabs/registry/commit/990b588b75857d2f61b76d89999a2c1f09f861f8)) 99 | * update to node 18 ([a08ee16](https://github.com/fluencelabs/registry/commit/a08ee16ff9dc402e1388e22c57324ca975c1a94d)) 100 | 101 | ## [0.8.6](https://github.com/fluencelabs/registry/compare/registry-v0.8.5...registry-v0.8.6) (2023-05-19) 102 | 103 | 104 | ### Features 105 | 106 | * **parser:** Fix indentation ([#241](https://github.com/fluencelabs/registry/issues/241)) ([d96f5a4](https://github.com/fluencelabs/registry/commit/d96f5a4a0da7288ef6895c270fe207ea9a9f102d)) 107 | 108 | ## [0.8.5](https://github.com/fluencelabs/registry/compare/registry-v0.8.4...registry-v0.8.5) (2023-05-08) 109 | 110 | 111 | ### Features 112 | 113 | * **builtin-package:** use new blueprint ([#234](https://github.com/fluencelabs/registry/issues/234)) ([061cf2f](https://github.com/fluencelabs/registry/commit/061cf2f8186192c39946628e21e466323dc31a33)) 114 | 115 | ## [0.8.4](https://github.com/fluencelabs/registry/compare/registry-v0.8.3...registry-v0.8.4) (2023-04-19) 116 | 117 | 118 | ### Features 119 | 120 | * update aqua-lib and trust-graph versions ([#229](https://github.com/fluencelabs/registry/issues/229)) ([5e460e3](https://github.com/fluencelabs/registry/commit/5e460e3e2429df909d034193fedf2876f86b18a8)) 121 | 122 | 123 | ### Bug Fixes 124 | 125 | * **deps:** pin dependencies ([#198](https://github.com/fluencelabs/registry/issues/198)) ([e66457c](https://github.com/fluencelabs/registry/commit/e66457c0ff696330717e58e3ebb4120709281202)) 126 | * **deps:** update dependency @fluencelabs/fluence-network-environment to v1.0.14 ([#195](https://github.com/fluencelabs/registry/issues/195)) ([204af45](https://github.com/fluencelabs/registry/commit/204af450001cd6e1ed587111fcc452d41d56a705)) 127 | 128 | ## [0.8.3](https://github.com/fluencelabs/registry/compare/registry-v0.8.2...registry-v0.8.3) (2023-04-06) 129 | 130 | 131 | ### Features 132 | 133 | * **sqlite:** bump to v0.18.1 ([#218](https://github.com/fluencelabs/registry/issues/218)) ([4fd0895](https://github.com/fluencelabs/registry/commit/4fd0895ab8415b60eacb34e0a627e9d6d5b5fe2c)) 134 | 135 | ## [0.8.2](https://github.com/fluencelabs/registry/compare/registry-v0.8.1...registry-v0.8.2) (2023-03-08) 136 | 137 | 138 | ### Bug Fixes 139 | 140 | * **deps:** Update sqlite to 0.8.0 ([#205](https://github.com/fluencelabs/registry/issues/205)) ([d27f232](https://github.com/fluencelabs/registry/commit/d27f232fb44629b18fa45e45b7c33e332f5817fd)) 141 | 142 | ## [0.8.1](https://github.com/fluencelabs/registry/compare/registry-v0.8.0...registry-v0.8.1) (2023-02-24) 143 | 144 | 145 | ### Bug Fixes 146 | 147 | * **subnet:** add on HOST_PEER_ID in resolveSubnetwork ([#202](https://github.com/fluencelabs/registry/issues/202)) ([3960180](https://github.com/fluencelabs/registry/commit/3960180246471a78bacf5fa65152a52fb3d4ddf2)) 148 | 149 | ## [0.8.0](https://github.com/fluencelabs/registry/compare/registry-v0.7.1...registry-v0.8.0) (2023-02-24) 150 | 151 | 152 | ### ⚠ BREAKING CHANGES 153 | 154 | * **storage:** bump SQLite module to 0.18.0 ([#200](https://github.com/fluencelabs/registry/issues/200)) 155 | 156 | ### Bug Fixes 157 | 158 | * **storage:** bump SQLite module to 0.18.0 ([#200](https://github.com/fluencelabs/registry/issues/200)) ([f671c8a](https://github.com/fluencelabs/registry/commit/f671c8ac1514a11331ae871a7e126f1e908214f6)) 159 | 160 | ## [0.7.1](https://github.com/fluencelabs/registry/compare/registry-v0.7.0...registry-v0.7.1) (2023-02-20) 161 | 162 | 163 | ### Features 164 | 165 | * **deals:** register and resolve workers ([#197](https://github.com/fluencelabs/registry/issues/197)) ([8d49211](https://github.com/fluencelabs/registry/commit/8d492113f17ec7add582f7f2d9575fc48b5325dc)) 166 | * **tests:** Run tests using fluence cli [fixes DXJ-225] ([#165](https://github.com/fluencelabs/registry/issues/165)) ([269373f](https://github.com/fluencelabs/registry/commit/269373f0ea904c572cffa51b8d49a248822c7ff1)) 167 | 168 | 169 | ### Bug Fixes 170 | 171 | * run all tests with different secret keys [fixes DXJ-242] ([#187](https://github.com/fluencelabs/registry/issues/187)) ([9b5cfbd](https://github.com/fluencelabs/registry/commit/9b5cfbd987259a890933e516e8ec2fee58e149d8)) 172 | * **tests:** fix registry aqua tests [fixes DXJ-235] ([#178](https://github.com/fluencelabs/registry/issues/178)) ([9981043](https://github.com/fluencelabs/registry/commit/9981043448fa3a9d64353ab763f9985245a6dff0)) 173 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contribute Code 2 | 3 | You are welcome to contribute to Fluence! 4 | 5 | Things you need to know: 6 | 7 | 1. You need to **agree to the [Contributor License Agreement](https://gist.github.com/fluencelabs-org/3f4cbb3cc14c1c0fb9ad99d8f7316ed7) (CLA)**. This is a common practice in all major Open Source projects. At the current moment, we are unable to accept contributions made on behalf of a company. Only individual contributions will be accepted. 8 | 9 | 2. **Not all proposed contributions can be accepted**. Some features may, e.g., just fit a third-party add-on better. The contribution must fit the overall direction of Fluence and really improve it. The more effort you invest, the better you should clarify in advance whether the contribution fits: the best way would be to just open an issue to discuss the contribution you plan to make. 10 | 11 | ### Contributor License Agreement 12 | 13 | When you contribute, you have to be aware that your contribution is covered by **[Apache License 2.0](./LICENSE)**, but might relicensed under few other software licenses mentioned in the **Contributor License Agreement**. In particular, you need to agree to the Contributor License Agreement. If you agree to its content, you simply have to click on the link posted by the CLA assistant as a comment to the pull request. Click it to check the CLA, then accept it on the following screen if you agree to it. The CLA assistant will save this decision for upcoming contributions and will notify you if there is any change to the CLA in the meantime. 14 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | # How to Use Registry in Aqua 2 | 3 | ## How to install 4 | If you use Fluence CLI you can define dependency in `fluence.yaml`: 5 | ```yaml 6 | dependencies: 7 | npm: 8 | "@fluencelabs/registry": 0.6.2 9 | ``` 10 | And then run `fluence dep i` 11 | 12 | If you are developing from scratch without Fluence CLI, you should install it via npm: 13 | 14 | ```bash 15 | npm i @fluencelabs/registry@nightly 16 | ``` 17 | 18 | ## How to import 19 | 20 | ```rust 21 | import "@fluencelabs/registry/resources-api.aqua" 22 | import "@fluencelabs/registry/registry-service.aqua" 23 | 24 | func myFunction(resource_id: string) -> ?[]Record, *Error: 25 | result, errors <- resolveResource(resource_id, 2) 26 | <- result, errors 27 | ``` 28 | 29 | ## How to create a Resource 30 | - `createResource(label: string) -> ?ResourceId, *Error` 31 | 32 | Let's register a resource with the label `sample` by `INIT_PEER_ID`: 33 | ```rust 34 | func my_resource() -> ?ResourceId, *Error: 35 | id, errors <- createResource("sample") 36 | <- id, errors 37 | ``` 38 | 39 | - `label` is a unique string for the peer id 40 | - creation is successful if a resource id is returned 41 | - `*Error` accumulates errors from all the affected peers 42 | 43 | ## How to remove a Resource 44 | 45 | For now there is no method for Resource removal but it can be expired and garbage-collected if it doesn't have any actual records. In the future updates it can be changed. 46 | 47 | ## How to register a service 48 | ``` 49 | registerService(resource_id: ResourceId, value: string, peer_id: string service_id: ?string) -> bool, *Error 50 | ``` 51 | 52 | Let's register a local service `greeting` and pass a random string `hi` as a value: 53 | ```rust 54 | func registerLocalService(resource_id: ResourceId) -> bool, *Error: 55 | success, errors <- registerService(resource_id, "hi", INIT_PEER_ID, ?[greeting]) 56 | <- success, errors 57 | ``` 58 | 59 | 60 | Let's register a service `echo` hosted on `peer_id` and pass a random string like `sample` as a value: 61 | ```rust 62 | func registerExternalService(resource_id: ResourceId, peer_id: PeerId) -> bool, *Error: 63 | success, errors <- registerService(resource_id, "hi", peer_id, ?[greeting]) 64 | <- success, errors 65 | ``` 66 | 67 | - `value` is a user-defined string that can be used at the discretion of the user 68 | - to update the service record, you should register it again to create a record with a newer timestamp 69 | - service record will be automatically updated till deleted via `unregisterService` 70 | 71 | 72 | ## How to unregister a service 73 | ``` 74 | func unregisterService(resource_id: ResourceId, peer_id: PeerId) -> bool, *Error: 75 | ``` 76 | Let's remove a service record from a target node: 77 | ```rust 78 | func stopProvideExternalService(resource_id: ResourceId, peer_id: PeerId): 79 | unregisterService(resource_id, peer_id) 80 | ``` 81 | 82 | - it will be removed from the target node and eventually from the network 83 | 84 | ## How to resolve service records 85 | - `resolveResource(resource_id: ResourceId, ack: i16) -> ?[]Record, *Error` 86 | 87 | Let's resolve all service records for our resource_id: 88 | ```rust 89 | func getMyRecords(resource_id: ResourceId, consistency_level: i16) -> ?[]Record, *Error: 90 | records, error <- resolveResource(resource_id, consistency_level) 91 | <- records, error 92 | ``` 93 | 94 | - `ack` represents a minimal number of peers that requested for known records 95 | 96 | ## How to execute a callback on Resource 97 | - `executeOnResource(resource_id: ResourceId, ack: i16, call: Record -> ()) -> bool, *Error` 98 | 99 | ```rust 100 | func callProvider(r: Record): 101 | -- topological move to a provider via relay 102 | on r.metadata.peer_id via r.metadata.relay_id: 103 | -- resolve and call your service on a provider 104 | MyService r.metadata.service_id! 105 | MyService.do_smth() 106 | 107 | -- call on every provider 108 | func callEveryone(resource_id: ResourceId, ack: i16) -> bool, *Error: 109 | success, errors <- executeOnResource(resource_id, ack, callProvider) 110 | ``` 111 | 112 | - it is a combination of `resolveResource` and a `for` loop through records with the callback execution 113 | - it can be useful in case of broadcasting events 114 | 115 | ## Replication 116 | 117 | Resources with corresponding records and tombstones are automatically and periodically replicated to the Kademlia neighborhood of `resource_id`. 118 | 119 | ## Remarks 120 | 121 | You can redefine [`INITIAL_REPLICATION_FACTOR`](https://github.com/fluencelabs/registry/blob/main/aqua/resources-api.aqua#L10) and [`CONSISTENCY_LEVEL`](https://github.com/fluencelabs/registry/blob/main/aqua/resources-api.aqua#L11). The first constant is used to define the number of peers to which data will be replicated during the API call. This constant doesn't affect the network-wide replication factor, which is defined by Kademlia. The second constant defines the minimal number of peers requested to obtain the data. 122 | 123 | ## Use cases 124 | 125 | ### Services discovery 126 | Discover services without prior knowledge about exact peers and service identifiers. 127 | 128 | ### Service high-availability 129 | A service provided by several peers still will be available for the client in case of disconnections and other providers' failures. 130 | 131 | ![image](images/availability.png) 132 | 133 | ### Subnetwork discovery 134 | You can register a group of peers for a resource (without specifying any services). So you "tag" and group the nodes to create a subnetwork. 135 | 136 | ![image](images/subnetwork.png) 137 | 138 | ### Load balancer 139 | If you have a list of service records updated in runtime, you can create a load-balancing service based on your preferred metrics. 140 | 141 | 142 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Registry 2 | 3 | [![npm](https://img.shields.io/npm/v/@fluencelabs/registry)](https://www.npmjs.com/package/@fluencelabs/registry) 4 | 5 | 6 | ## Overview 7 | 8 | Registry is an essential part of the [Fluence network](https://fluence.network) protocol. It provides a Resources API that can be used for service advertisement and discovery. Registry is available (built-in) on every Fluence node, and it provides service advertisement and discovery. The component allows creating relationships between unique identifiers and groups of services on various peers, so that service providers can either join or disconnect anytime and be discoverable on the network. 9 | 10 | There are many [services](https://doc.fluence.dev/docs/concepts#services) in the network on different peers, and there should be a way to find and resolve these services without prior knowledge about exact identifiers. Such an approach brings robustness and flexibility to our solutions in terms of discovery, redundancy and high availability. 11 | 12 | In centralized systems, one can have centralized storage and routing, but in p2p decentralized environments, the problem becomes more challenging. Our solution for the problem is **Registry**, a purpose-driven distributed hash table (DHT), an inherent part of the [Fluence](https://fluence.dev) protocol. 13 | 14 | ![image](images/registry.png) 15 | 16 | However, Registry is not a plain key/value storage. Instead, it is a composition of the Registry service for each network participant and scheduled scripts maintaining replication and garbage collection. Thus, if you want to discover a group of services on different peers without prior knowledge, you should create a **Resource**. A resource is a group of services or peers united by some common feature. Any service is represented by a combination of `service_id` and `peer_id`, it is called a **Record**. 17 | 18 | **Why is Registry important?** 19 | 20 | Scalability, redundancy and high availability are essential parts of a decentralized system, but they are not available out of the box. To enable them, information about services should be bound with peers providing them. Also, such networks are constantly changing, and those changes should be reflected and resolvable to provide uninterruptible access. So there's a need to have a decentralized protocol to update and resolve information about routing, both global and local. 21 | 22 | 23 | ## Installation and Usage 24 | 25 | A complete workflow covering installation of Registry, creating Resources, registering services etc. can be found [here](INSTALL.md). 26 | 27 | 28 | ## Documentation 29 | 30 | Comprehensive documentation on Fluence can be found [here](https://fluence.dev). In particular, it includes [Aqua Book](https://fluence.dev/docs/aqua-book/getting-started/). Also, check our [YouTube channel](https://www.youtube.com/@fluencelabs). [This presentation](https://www.youtube.com/watch?v=Md0_Ny_5_1o&t=770s) at one of our community calls was especially dedicated to Registry. 31 | 32 | Resources API is defined in the [resources-api](./aqua/resources-api.aqua) module. Service API is defined in the [registry-service](./aqua/registry-service.aqua) module. For the details, check the [API Reference](./API_reference.md). 33 | 34 | 35 | ## Repository Structure 36 | 37 | - [**aqua-tests**](./aqua-tests) contains tests for the Aqua API and use 38 | it in e2e tests 39 | - [**aqua**](./aqua) is the Registry Aqua API 40 | - [**builtin-package**](./builtin-package) contains a build script and 41 | config files used for building a standard distro for the Rust peer 42 | builtins 43 | - [**service**](./service) is the Rust code for the Registry service 44 | 45 | 46 | ## Support 47 | 48 | Please, file an [issue](https://github.com/fluencelabs/registry/issues) if you find a bug. You can also contact us at [Discord](https://discord.com/invite/5qSnPZKh7u) or [Telegram](https://t.me/fluence_project). We will do our best to resolve the issue ASAP. 49 | 50 | 51 | ## Contributing 52 | 53 | Any interested person is welcome to contribute to the project. Please, make sure you read and follow some basic [rules](./CONTRIBUTING.md). 54 | 55 | 56 | ## License 57 | 58 | All software code is copyright (c) Fluence Labs, Inc. under the [Apache-2.0](./LICENSE) license. 59 | 60 | -------------------------------------------------------------------------------- /aqua-tests/.fluence/aqua-dependencies/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aqua-dependencies", 3 | "lockfileVersion": 3, 4 | "requires": true, 5 | "packages": { 6 | "": { 7 | "dependencies": { 8 | "@fluencelabs/aqua-lib": "0.8.1", 9 | "@fluencelabs/registry": "0.8.7", 10 | "@fluencelabs/spell": "0.5.33", 11 | "@fluencelabs/trust-graph": "3.1.2" 12 | } 13 | }, 14 | "node_modules/@fluencelabs/aqua-lib": { 15 | "version": "0.8.1", 16 | "resolved": "https://registry.npmjs.org/@fluencelabs/aqua-lib/-/aqua-lib-0.8.1.tgz", 17 | "integrity": "sha512-VLslkhi3hsNLWkgsoCyceCediqkicWphMVHZ+9eEkgMumepvo7TcqiYC14bl2LpZjn7YZ6y/OzK+Ffy8ADfKdA==" 18 | }, 19 | "node_modules/@fluencelabs/registry": { 20 | "version": "0.8.7", 21 | "resolved": "https://registry.npmjs.org/@fluencelabs/registry/-/registry-0.8.7.tgz", 22 | "integrity": "sha512-43bmb1v4p5ORvaiLBrUAl+hRPo3luxxBVrJgqTvipJa2OEg2wCRA/Wo9s4M7Lchnv3NoYLOyNTzNyFopQRKILA==", 23 | "dependencies": { 24 | "@fluencelabs/aqua-lib": "0.7.0", 25 | "@fluencelabs/trust-graph": "0.4.1" 26 | } 27 | }, 28 | "node_modules/@fluencelabs/registry/node_modules/@fluencelabs/aqua-lib": { 29 | "version": "0.7.0", 30 | "resolved": "https://registry.npmjs.org/@fluencelabs/aqua-lib/-/aqua-lib-0.7.0.tgz", 31 | "integrity": "sha512-mJEaxfAQb6ogVM4l4qw7INK6kvLA2Y161ErwL7IVeVSkKXIeYq/qio2p2au35LYvhBNsKc7XP2qc0uztCmxZzA==" 32 | }, 33 | "node_modules/@fluencelabs/registry/node_modules/@fluencelabs/trust-graph": { 34 | "version": "0.4.1", 35 | "resolved": "https://registry.npmjs.org/@fluencelabs/trust-graph/-/trust-graph-0.4.1.tgz", 36 | "integrity": "sha512-V/6ts4q/Y0uKMS6orVpPyxfdd99YFMkm9wN9U2IFtlBUWNsQZG369FK9qEizwsSRCqTchMHYs8Vh4wgZ2uRfuQ==", 37 | "dependencies": { 38 | "@fluencelabs/aqua-lib": "^0.7.0" 39 | } 40 | }, 41 | "node_modules/@fluencelabs/spell": { 42 | "version": "0.5.33", 43 | "resolved": "https://registry.npmjs.org/@fluencelabs/spell/-/spell-0.5.33.tgz", 44 | "integrity": "sha512-JZ+CWTrBXwX6DilzxsJfg39DMsQN9P/h1jyujcDwIpOKynbGCD84g5t9hsplNVH/pEZwcYtGajDH293Sg54bwA==", 45 | "dependencies": { 46 | "@fluencelabs/aqua-lib": "0.8.1" 47 | }, 48 | "engines": { 49 | "node": ">=18", 50 | "pnpm": ">=8" 51 | } 52 | }, 53 | "node_modules/@fluencelabs/trust-graph": { 54 | "version": "3.1.2", 55 | "resolved": "https://registry.npmjs.org/@fluencelabs/trust-graph/-/trust-graph-3.1.2.tgz", 56 | "integrity": "sha512-HpyHtiomh09wv6/83z+bhbkqVngIUdqNGEXRTIPg4sArVPMZ9UCXBrkQsHDRqdMUx0lBAcgB3IjlbdhkwHGaXA==", 57 | "dependencies": { 58 | "@fluencelabs/aqua-lib": "^0.5.2" 59 | } 60 | }, 61 | "node_modules/@fluencelabs/trust-graph/node_modules/@fluencelabs/aqua-lib": { 62 | "version": "0.5.2", 63 | "resolved": "https://registry.npmjs.org/@fluencelabs/aqua-lib/-/aqua-lib-0.5.2.tgz", 64 | "integrity": "sha512-fmoFFE8myhLH9d+YR0+0ZPL2YIQyR6M1woAGu5d1xXI02Sjzn4id6dE4PpxHb8cSBPRie8AwsKobHCNqGxI8oA==" 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /aqua-tests/.fluence/env.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=schemas/env.json 2 | 3 | # Defines user project preferences 4 | 5 | # Documentation: https://github.com/fluencelabs/cli/tree/main/docs/configs/env.md 6 | 7 | version: 0 8 | 9 | fluenceEnv: local 10 | -------------------------------------------------------------------------------- /aqua-tests/.fluence/secrets/test_create_resource.txt: -------------------------------------------------------------------------------- 1 | BNidntUryx+hxr7NK2z9nci23sMn3fURB6bTH1K2Ll4= -------------------------------------------------------------------------------- /aqua-tests/.fluence/secrets/test_get_resource.txt: -------------------------------------------------------------------------------- 1 | e72l3wuItcfCcQBP6Rn4L0uQRsKmyckZRbYXP1ms59Q= -------------------------------------------------------------------------------- /aqua-tests/.fluence/secrets/test_register_record_unregister.txt: -------------------------------------------------------------------------------- 1 | rZxZGGCxECt1opnXjnxrSpV2g6Qt2Fl0KTDoJkox008= -------------------------------------------------------------------------------- /aqua-tests/.fluence/secrets/test_register_unregister_remote_record.txt: -------------------------------------------------------------------------------- 1 | I/ZUMsjlt47e9LxYxbk/LamZJUzNxoBikPA+Qqy8yYA= -------------------------------------------------------------------------------- /aqua-tests/.gitignore: -------------------------------------------------------------------------------- 1 | /.fluence/schemas 2 | /.fluence/aqua-dependencies/package.json 3 | -------------------------------------------------------------------------------- /aqua-tests/README.md: -------------------------------------------------------------------------------- 1 | # Registry API tests 2 | 3 | ## How to run 4 | 5 | - `npm i` 6 | - `pip3 install -r requirements.txt` 7 | - `pip install -U pytest` 8 | - `pytest -n auto` 9 | 10 | ## Adding new test 11 | 12 | Before adding new test go to the aqua-tests dir first, then run `npm run secret` 13 | to add a new key-pair for the new test. 14 | Name it the same way the test function will be called (e.g. `test_create_resource`) 15 | This is required for tests to run in parallel. Key-pairs could've been generated on the fly 16 | but it's a bit faster to not waste time on it each time the tests are run 17 | -------------------------------------------------------------------------------- /aqua-tests/aqua/test.aqua: -------------------------------------------------------------------------------- 1 | aqua Test 2 | 3 | import "@fluencelabs/aqua-lib/builtin.aqua" 4 | import "@fluencelabs/registry/resources-api.aqua" 5 | 6 | export getResource, createResource, getResourceId, get_peer_id, registerService, resolveResource, unregisterService 7 | 8 | func get_peer_id() -> PeerId: 9 | <- INIT_PEER_ID 10 | -------------------------------------------------------------------------------- /aqua-tests/config.py: -------------------------------------------------------------------------------- 1 | 2 | def get_local(): 3 | return [ 4 | '/ip4/127.0.0.1/tcp/9991/ws/p2p/12D3KooWBM3SdXWqGaawQDGQ6JprtwswEg3FWGvGhmgmMez1vRbR', 5 | '/ip4/127.0.0.1/tcp/9992/ws/p2p/12D3KooWQdpukY3p2DhDfUfDgphAqsGu5ZUrmQ4mcHSGrRag6gQK', 6 | '/ip4/127.0.0.1/tcp/9993/ws/p2p/12D3KooWRT8V5awYdEZm6aAV9HWweCEbhWd7df4wehqHZXAB7yMZ', 7 | '/ip4/127.0.0.1/tcp/9994/ws/p2p/12D3KooWBzLSu9RL7wLP6oUowzCbkCj2AGBSXkHSJKuq4wwTfwof', 8 | '/ip4/127.0.0.1/tcp/9995/ws/p2p/12D3KooWBf6hFgrnXwHkBnwPGMysP3b1NJe5HGtAWPYfwmQ2MBiU', 9 | '/ip4/127.0.0.1/tcp/9996/ws/p2p/12D3KooWPisGn7JhooWhggndz25WM7vQ2JmA121EV8jUDQ5xMovJ' 10 | ] -------------------------------------------------------------------------------- /aqua-tests/fluence.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=.fluence/schemas/fluence.json 2 | 3 | # Defines Fluence Project, most importantly - what exactly you want to deploy and how. You can use `fluence init` command to generate a template for new Fluence project 4 | 5 | # Documentation: https://github.com/fluencelabs/fluence-cli/tree/main/docs/configs/fluence.md 6 | 7 | version: 5 8 | 9 | aquaInputPath: aqua/test.aqua 10 | 11 | dependencies: 12 | cargo: 13 | marine: 0.14.1 14 | mrepl: 0.21.3 15 | npm: 16 | '@fluencelabs/aqua-lib': 0.9.1 17 | '@fluencelabs/spell': 0.6.9 18 | '@fluencelabs/trust-graph': 0.4.11 19 | -------------------------------------------------------------------------------- /aqua-tests/getDefaultPeers.js: -------------------------------------------------------------------------------- 1 | const { 2 | krasnodar, 3 | stage, 4 | testNet, 5 | } = require('@fluencelabs/fluence-network-environment') 6 | 7 | console.log( 8 | JSON.stringify({ 9 | krasnodar, 10 | stage, 11 | testnet: testNet, 12 | }), 13 | ) 14 | -------------------------------------------------------------------------------- /aqua-tests/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aqua-tests", 3 | "lockfileVersion": 3, 4 | "requires": true, 5 | "packages": { 6 | "": { 7 | "devDependencies": { 8 | "@fluencelabs/fluence-network-environment": "1.1.2" 9 | } 10 | }, 11 | "node_modules/@fluencelabs/fluence-network-environment": { 12 | "version": "1.1.2", 13 | "resolved": "https://registry.npmjs.org/@fluencelabs/fluence-network-environment/-/fluence-network-environment-1.1.2.tgz", 14 | "integrity": "sha512-1Bp2gBy3oMEILMynFpOIFK/q2Pj792xpnb3AJs5QcTQAaHz9V2nrEI8OOPwBAFTmjmLBirXBqQQX63O+ePH7yg==", 15 | "dev": true 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /aqua-tests/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "devDependencies": { 3 | "@fluencelabs/fluence-network-environment": "1.1.2" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /aqua-tests/requirements.txt: -------------------------------------------------------------------------------- 1 | delegator.py==0.1.1 2 | pytest==7.3.0 3 | pytest-xdist==3.2.1 4 | pytest-repeat==0.9.1 5 | -------------------------------------------------------------------------------- /aqua-tests/spell/spell.aqua: -------------------------------------------------------------------------------- 1 | aqua TestSpell 2 | 3 | export spell 4 | 5 | import Op, Debug, Peer, Kademlia from "@fluencelabs/aqua-lib/builtin.aqua" 6 | import Spell from "@fluencelabs/spell/spell_service.aqua" 7 | import Compare from "@fluencelabs/aqua-lib/math.aqua" 8 | 9 | import "@fluencelabs/registry/registry-service.aqua" 10 | import "@fluencelabs/registry/registry-api.aqua" 11 | import "@fluencelabs/trust-graph/trust-graph.aqua" 12 | 13 | data SpellConfig: 14 | expired_interval: u32 15 | renew_interval: u32 16 | replicate_interval:u32 17 | 18 | -- A hack to allow using timestamp as u32 values 19 | -- Aqua doesn't allow truncating values 20 | service PeerTimeTrunc("peer"): 21 | timestamp_sec() -> u32 22 | 23 | func log_info(spell_id: string, msg: string): 24 | Spell spell_id 25 | Spell.list_push_string("logs", msg) 26 | 27 | 28 | -- clears expired records 29 | func clear_expired(now:u32): 30 | Registry.clear_expired(now) 31 | 32 | -- update stale local records 33 | func renew(now:u32): 34 | res <- Registry.get_stale_local_records(now) 35 | for r <- res.result par: 36 | signature <- getRecordSignature(r.metadata, now) 37 | putRecord(r.metadata, now, signature.signature!) 38 | 39 | -- get all old records and replicate it by routes 40 | func replicate(now:u32): 41 | res <- Registry.evict_stale(now) 42 | for r <- res.results par: 43 | k <- Op.string_to_b58(r.key.id) 44 | nodes <- Kademlia.neighborhood(k, nil, nil) 45 | for n <- nodes par: 46 | on n: 47 | tt <- Peer.timestamp_sec() 48 | key_weight <- TrustGraph.get_weight(r.key.owner_peer_id, tt) 49 | Registry.republish_key(r.key, key_weight, tt) 50 | 51 | records_weights: *WeightResult 52 | for record <- r.records: 53 | records_weights <- TrustGraph.get_weight(record.metadata.issued_by, tt) 54 | Registry.republish_records(r.records, records_weights, tt) 55 | 56 | func spell(config: SpellConfig): 57 | Spell "registry-spell" 58 | log = (msg: string): 59 | log_info("registry-spell", msg) 60 | 61 | check_and_run = (key: string, now:u32, interval: u32, job: u32 -> ()): 62 | last_run <- Spell.get_u32(key) 63 | need_to_run = !last_run.success || ((now - last_run.value) >= interval) 64 | if need_to_run == true: 65 | log(Op.concat_strings(Op.concat_strings("Running ", key), "job")) 66 | job(now) 67 | Spell.set_u32(key, now) 68 | 69 | 70 | on HOST_PEER_ID: 71 | now <- PeerTimeTrunc.timestamp_sec() 72 | check_and_run("clear_expired", now, config.expired_interval, clear_expired) 73 | check_and_run("renew", now, config.renew_interval, renew) 74 | check_and_run("replicate", now, config.replicate_interval, replicate) 75 | -------------------------------------------------------------------------------- /aqua-tests/spell/spell.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=../.fluence/schemas/spell.yaml.json 2 | 3 | # Defines a spell. You can use `fluence spell new` command to generate a template for new spell 4 | 5 | # Documentation: https://github.com/fluencelabs/fluence-cli/tree/main/docs/configs/spell.md 6 | 7 | version: 0 8 | aquaFilePath: ./spell.aqua 9 | function: spell 10 | clock: 11 | periodSec: 6000 12 | -------------------------------------------------------------------------------- /aqua-tests/test_aqua.py: -------------------------------------------------------------------------------- 1 | import delegator 2 | import random 3 | import json 4 | import os 5 | import inspect 6 | from config import get_local 7 | 8 | default_peers = json.loads(delegator.run( 9 | f"node ./getDefaultPeers.js", block=True).out) 10 | 11 | 12 | def get_relays(): 13 | env = os.environ.get("FLUENCE_ENV") 14 | if env == "local": 15 | peers = get_local() 16 | else: 17 | if env is None: 18 | env = "testnet" 19 | peers = [peer["multiaddr"] for peer in default_peers[env]] 20 | 21 | assert len(peers) != 0 22 | return peers 23 | 24 | 25 | relays = get_relays() 26 | peer_ids = [relay.split("/")[-1] for relay in relays] 27 | 28 | 29 | def get_random_list_item(ar): 30 | return ar[random.randint(0, len(ar) - 1)] 31 | 32 | 33 | def get_random_relay(): 34 | return get_random_list_item(relays) 35 | 36 | 37 | def get_random_peer_id(): 38 | return get_random_list_item(peer_ids) 39 | 40 | 41 | def get_label(): 42 | return ''.join(random.choice('0123456789ABCDEF') for i in range(16)) 43 | 44 | 45 | def run_aqua(func, args, relay=get_random_relay()): 46 | 47 | # "a" : arg1, "b" : arg2 ..... 48 | data = {chr(97 + i): arg for (i, arg) in enumerate(args)} 49 | call = f"{func}(" + ", ".join([chr(97 + i) 50 | for i in range(0, len(args))]) + ")" 51 | # inspect.stack method inspects the current execution stack as the name suggests 52 | # it's possible to infer that the minus 39th element of the stack always contains info 53 | # about the test function that is currently running. The third element is the function's name 54 | try: 55 | test_name = inspect.stack()[-39][3] 56 | except: 57 | # when running one test at a time, the stack is shorter so we need to use a different index 58 | test_name = inspect.stack()[-32][3] 59 | 60 | command = f"fluence run -k {test_name} --relay {relay} -f '{call}' --data '{json.dumps(data)}' --quiet --particle-id" 61 | print(command) 62 | c = delegator.run(command, block=True) 63 | lines = c.out.splitlines() 64 | particle_id = lines[0] if len(lines) != 0 else "" 65 | 66 | if len(c.err.strip()) != 0: 67 | print(f"{particle_id}\n{c.err}") 68 | 69 | result = '\n'.join(lines[1:]) 70 | 71 | try: 72 | result = json.loads(result) 73 | print(result) 74 | return result 75 | except: 76 | print(result) 77 | return result 78 | 79 | 80 | def create_resource(label): 81 | result, error = run_aqua("createResource", [label]) 82 | assert result != None, error 83 | return result 84 | 85 | 86 | def get_peer_id(): 87 | return run_aqua("get_peer_id", []) 88 | 89 | 90 | def test_create_resource(): 91 | label = get_label() 92 | result = create_resource(label) 93 | peer_id = get_peer_id() 94 | resource_id = run_aqua("getResourceId", [label, peer_id]) 95 | assert result == resource_id 96 | 97 | 98 | def test_get_resource(): 99 | label = get_label() 100 | resource_id = create_resource(label) 101 | peer_id = get_peer_id() 102 | result, error = run_aqua("getResource", [resource_id]) 103 | assert result != None, error 104 | assert result["id"] == resource_id, error 105 | assert result["owner_peer_id"] == peer_id, error 106 | assert result["label"] == label, error 107 | 108 | 109 | def test_register_record_unregister(): 110 | relay = get_random_relay() 111 | label = get_label() 112 | value = "some_value" 113 | peer_id = get_peer_id() 114 | service_id = "id" 115 | 116 | resource_id = create_resource(label) 117 | result, error = run_aqua( 118 | "registerService", [resource_id, value, peer_id, service_id], relay) 119 | assert result, error 120 | 121 | # we want at least 1 successful response 122 | result, error = run_aqua("resolveResource", [resource_id, 1], relay) 123 | assert result != None, error 124 | 125 | assert len(result) == 1, "records not found" 126 | 127 | record = result[0] 128 | assert record["metadata"]["key_id"] == resource_id 129 | assert record["metadata"]["issued_by"] == peer_id 130 | assert record["metadata"]["peer_id"] == peer_id 131 | assert record["metadata"]["service_id"] == [service_id] 132 | 133 | result, error = run_aqua("unregisterService", [resource_id, peer_id], 134 | relay) 135 | assert result, error 136 | 137 | result, error = run_aqua("resolveResource", [resource_id, 2], relay) 138 | assert result != None, error 139 | assert len(result) == 0 140 | 141 | 142 | def test_register_unregister_remote_record(): 143 | relay = get_random_relay() 144 | label = get_label() 145 | value = "some_value" 146 | issuer_peer_id = get_peer_id() 147 | peer_id = get_random_peer_id() 148 | service_id = "id" 149 | 150 | resource_id = create_resource(label) 151 | result, error = run_aqua( 152 | "registerService", [resource_id, value, peer_id, service_id], relay) 153 | assert result, error 154 | 155 | result, error = run_aqua("resolveResource", [resource_id, 2], relay) 156 | assert result != None, error 157 | 158 | assert len(result) == 1, "records not found" 159 | 160 | record = result[0] 161 | assert record["metadata"]["key_id"] == resource_id 162 | assert record["metadata"]["issued_by"] == issuer_peer_id 163 | assert record["metadata"]["peer_id"] == peer_id 164 | assert record["metadata"]["service_id"] == [service_id] 165 | 166 | result, error = run_aqua("unregisterService", [resource_id, peer_id], 167 | relay) 168 | assert result, error 169 | 170 | result, error = run_aqua("resolveResource", [resource_id, 2], relay) 171 | assert result != None, error 172 | assert len(result) == 0 173 | -------------------------------------------------------------------------------- /aqua-tests/test_fluence_cli_version.py: -------------------------------------------------------------------------------- 1 | import delegator 2 | 3 | 4 | def test_fluence_cli_version(): 5 | c = delegator.run(f"fluence --version", block=True) 6 | print(f"Fluence CLI version: {c.out}") 7 | assert True 8 | -------------------------------------------------------------------------------- /aqua/constants.aqua: -------------------------------------------------------------------------------- 1 | aqua Constants declares * 2 | 3 | -- the number of peers to which data will be replicated during the API call 4 | const INITIAL_REPLICATION_FACTOR = 1 5 | -- the minimal number of peers requested to obtain the data. 6 | const CONSISTENCY_LEVEL = 1 7 | -- default timeout for waiting for results 8 | const DEFAULT_TIMEOUT = 6000 9 | -------------------------------------------------------------------------------- /aqua/misc.aqua: -------------------------------------------------------------------------------- 1 | aqua Misc declares * 2 | 3 | import "@fluencelabs/aqua-lib/builtin.aqua" 4 | import "registry-service.aqua" 5 | import "constants.aqua" 6 | 7 | alias ResourceId: string 8 | alias Resource: Key 9 | alias Error: string 10 | 11 | func wait(successful: *bool, len: i16, timeout: u16) -> bool: 12 | status: *string 13 | waiting = (arr: *bool, s: *string): 14 | join arr[len - 1] 15 | s <<- "ok" 16 | 17 | waiting(successful, status) 18 | par status <- Peer.timeout(timeout, "timeout") 19 | 20 | result: *bool 21 | stat = status! 22 | if stat == "ok": 23 | result <<- true 24 | else: 25 | result <<- false 26 | 27 | <- result! 28 | 29 | -- Get peers closest to the resource_id's hash in Kademlia network 30 | -- These peers are expected to store list of providers for this key 31 | func getNeighbors(resource_id: ResourceId) -> []PeerId: 32 | k <- Op.string_to_b58(resource_id) 33 | nodes <- Kademlia.neighborhood(k, nil, nil) 34 | <- nodes 35 | 36 | func appendErrors(error1: *Error, error2: *Error): 37 | for e <- error2: 38 | error1 <<- e 39 | 40 | func getResourceHelper(resource_id: ResourceId) -> ?Resource, *Error: 41 | nodes <- getNeighbors(resource_id) 42 | result: *Resource 43 | error: *Error 44 | 45 | resources: *Key 46 | successful: *bool 47 | for n <- nodes par: 48 | on n: 49 | try: 50 | get_result <- Registry.get_key_metadata(resource_id) 51 | if get_result.success: 52 | resources <<- get_result.key 53 | successful <<- true 54 | else: 55 | e <- Op.concat_strings(get_result.error, " on ") 56 | error <- Op.concat_strings(e, n) 57 | 58 | success <- wait(successful, CONSISTENCY_LEVEL, DEFAULT_TIMEOUT) 59 | if success == false: 60 | error <<- "resource not found: timeout exceeded" 61 | else: 62 | merge_result <- Registry.merge_keys(resources) 63 | 64 | if merge_result.success: 65 | result <<- merge_result.key 66 | else: 67 | error <<- merge_result.error 68 | 69 | <- result, error 70 | -------------------------------------------------------------------------------- /aqua/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@fluencelabs/registry", 3 | "version": "0.9.4", 4 | "lockfileVersion": 3, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "@fluencelabs/registry", 9 | "version": "0.9.4", 10 | "license": "MIT", 11 | "dependencies": { 12 | "@fluencelabs/aqua-lib": "0.9.1", 13 | "@fluencelabs/trust-graph": "0.4.11" 14 | } 15 | }, 16 | "node_modules/@fluencelabs/aqua-lib": { 17 | "version": "0.9.1", 18 | "resolved": "https://registry.npmjs.org/@fluencelabs/aqua-lib/-/aqua-lib-0.9.1.tgz", 19 | "integrity": "sha512-W4KBhUkFC/kb3Mvjmhpnz0vvtr1kqpV8IWTWlbitzzS1k5FhODQYP2k7ROkDyjEztg/7Bn5eunstEHr9Sxj6qA==" 20 | }, 21 | "node_modules/@fluencelabs/trust-graph": { 22 | "version": "0.4.11", 23 | "resolved": "https://registry.npmjs.org/@fluencelabs/trust-graph/-/trust-graph-0.4.11.tgz", 24 | "integrity": "sha512-tjtwahYw9Ol+P3SQuqiiucAqaZ9ftDXqUkNeJ/SxrJYR4zFu65yzyyQhZvUQe7msc+csLbhn03cGHqK0MKV0gA==", 25 | "dependencies": { 26 | "@fluencelabs/aqua-lib": "^0.9.0" 27 | } 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /aqua/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@fluencelabs/registry", 3 | "version": "0.9.4", 4 | "description": "Aqua Registry library", 5 | "files": [ 6 | "*.aqua" 7 | ], 8 | "dependencies": { 9 | "@fluencelabs/aqua-lib": "0.9.1", 10 | "@fluencelabs/trust-graph": "0.4.11" 11 | }, 12 | "scripts": { 13 | "generate-aqua": "../service/build.sh", 14 | "build": "fluence aqua -i . --dry" 15 | }, 16 | "repository": { 17 | "type": "git", 18 | "url": "git+https://github.com/fluencelabs/registry.git", 19 | "directory": "aqua" 20 | }, 21 | "keywords": [ 22 | "aqua", 23 | "fluence" 24 | ], 25 | "author": "Fluence Labs", 26 | "license": "MIT", 27 | "bugs": { 28 | "url": "https://github.com/fluencelabs/registry/issues" 29 | }, 30 | "homepage": "https://github.com/fluencelabs/registry" 31 | } 32 | -------------------------------------------------------------------------------- /aqua/registry-api.aqua: -------------------------------------------------------------------------------- 1 | aqua RegistryApi declares * 2 | 3 | export getKeySignature, getRecordMetadata 4 | export getRecordSignature, getTombstoneSignature 5 | export registerKey, putRecord, addTombstone 6 | export getKeyMetadata, republishKey 7 | 8 | import "registry-service.aqua" 9 | import PeerId, Peer, Sig, SignResult from "@fluencelabs/aqua-lib/builtin.aqua" 10 | import "@fluencelabs/trust-graph/trust-graph.aqua" 11 | 12 | func getKeySignature(label: string, timestamp_created: u64) -> SignResult: 13 | bytes <- Registry.get_key_bytes(label, nil, timestamp_created, nil, "") 14 | on INIT_PEER_ID via HOST_PEER_ID: 15 | result <- Sig.sign(bytes) 16 | <- result 17 | 18 | func getRecordMetadata(key_id: string, value: string, peer_id: string, relay_id: []string, service_id: []string, solution: []u8) -> ?RecordMetadata, ?string: 19 | t <- Peer.timestamp_sec() 20 | bytes <- Registry.get_record_metadata_bytes(key_id, INIT_PEER_ID, t, value, peer_id, relay_id, service_id, solution) 21 | 22 | on INIT_PEER_ID via HOST_PEER_ID: 23 | sig_result <- Sig.sign(bytes) 24 | 25 | result: *RecordMetadata 26 | error: *string 27 | if sig_result.success == true: 28 | result <- Registry.create_record_metadata(key_id, INIT_PEER_ID, t, value, peer_id, relay_id, service_id, solution, sig_result.signature!) 29 | else: 30 | error <<- sig_result.error! 31 | 32 | <- result, error 33 | 34 | func getRecordSignature(metadata: RecordMetadata, timestamp_created: u64) -> SignResult: 35 | signature: *SignResult 36 | 37 | if metadata.peer_id != INIT_PEER_ID: 38 | on metadata.peer_id via HOST_PEER_ID: 39 | bytes <- Registry.get_record_bytes(metadata, timestamp_created) 40 | signature <- Sig.sign(bytes) 41 | else: 42 | on HOST_PEER_ID: 43 | bytess <- Registry.get_record_bytes(metadata, timestamp_created) 44 | on INIT_PEER_ID: 45 | signature <- Sig.sign(bytess) 46 | 47 | <- signature! 48 | 49 | func getTombstoneSignature(key_id: string, peer_id: string, timestamp_issued: u64, solution: []u8) -> SignResult: 50 | bytes <- Registry.get_tombstone_bytes(key_id, INIT_PEER_ID, peer_id, timestamp_issued, solution) 51 | on INIT_PEER_ID via HOST_PEER_ID: 52 | result <- Sig.sign(bytes) 53 | <- result 54 | 55 | func registerKey(label: string, timestamp_created: u64, signature: []u8) -> RegisterKeyResult: 56 | t <- Peer.timestamp_sec() 57 | weight <- TrustGraph.get_weight(%init_peer_id%, t) 58 | result <- Registry.register_key(label, nil, timestamp_created, nil, "", signature, weight, t) 59 | <- result 60 | 61 | func putRecord(metadata: RecordMetadata, timestamp_created: u64, signature: []u8) -> RegistryResult: 62 | t <- Peer.timestamp_sec() 63 | weight <- TrustGraph.get_weight(metadata.issued_by, t) 64 | result <- Registry.put_record(metadata, timestamp_created, signature, weight, t) 65 | <- result 66 | 67 | func addTombstone(key_id: string, peer_id: string, timestamp_issued: u64, solution: []u8, signature: []u8) -> RegistryResult: 68 | t <- Peer.timestamp_sec() 69 | result <- Registry.add_tombstone(key_id, INIT_PEER_ID, peer_id, timestamp_issued, solution, signature, t) 70 | <- result 71 | 72 | func getKeyMetadata(key_id: string) -> GetKeyMetadataResult: 73 | result <- Registry.get_key_metadata(key_id) 74 | <- result 75 | 76 | func republishKey(key: Key) -> RegistryResult: 77 | t <- Peer.timestamp_sec() 78 | weight <- TrustGraph.get_weight(key.owner_peer_id, t) 79 | result <- Registry.republish_key(key, weight, t) 80 | <- result 81 | -------------------------------------------------------------------------------- /aqua/registry-scheduled-scripts.aqua: -------------------------------------------------------------------------------- 1 | aqua Registry.Scheduled declares * 2 | 3 | export clearExpired_86400, replicate_3600, renew_43200 4 | 5 | import "registry-service.aqua" 6 | import "registry-api.aqua" 7 | import "@fluencelabs/aqua-lib/builtin.aqua" 8 | import "@fluencelabs/trust-graph/trust-graph.aqua" 9 | 10 | -- clears expired records 11 | func clearExpired_86400(): 12 | on HOST_PEER_ID: 13 | t <- Peer.timestamp_sec() 14 | Registry.clear_expired(t) 15 | 16 | -- update stale local records 17 | func renew_43200(): 18 | on HOST_PEER_ID: 19 | t <- Peer.timestamp_sec() 20 | res <- Registry.get_stale_local_records(t) 21 | for r <- res.result par: 22 | signature <- getRecordSignature(r.metadata, t) 23 | putRecord(r.metadata, t, signature.signature!) 24 | 25 | -- get all old records and replicate it by routes 26 | func replicate_3600(): 27 | on HOST_PEER_ID: 28 | t <- Peer.timestamp_sec() 29 | res <- Registry.evict_stale(t) 30 | for r <- res.results par: 31 | k <- Op.string_to_b58(r.key.id) 32 | nodes <- Kademlia.neighborhood(k, nil, nil) 33 | for n <- nodes par: 34 | on n: 35 | tt <- Peer.timestamp_sec() 36 | key_weight <- TrustGraph.get_weight(r.key.owner_peer_id, tt) 37 | Registry.republish_key(r.key, key_weight, tt) 38 | 39 | records_weights: *WeightResult 40 | for record <- r.records: 41 | records_weights <- TrustGraph.get_weight(record.metadata.issued_by, tt) 42 | Registry.republish_records(r.records, records_weights, tt) 43 | -------------------------------------------------------------------------------- /aqua/registry-service.aqua: -------------------------------------------------------------------------------- 1 | aqua Registry declares * 2 | 3 | data ClearExpiredResult: 4 | success: bool 5 | error: string 6 | count_keys: u64 7 | count_records: u64 8 | count_tombstones: u64 9 | 10 | data Key: 11 | id: string 12 | label: string 13 | owner_peer_id: string 14 | timestamp_created: u64 15 | challenge: []u8 16 | challenge_type: string 17 | signature: []u8 18 | 19 | data RecordMetadata: 20 | key_id: string 21 | issued_by: string 22 | peer_id: string 23 | timestamp_issued: u64 24 | solution: []u8 25 | value: string 26 | relay_id: []string 27 | service_id: []string 28 | issuer_signature: []u8 29 | 30 | data Record: 31 | metadata: RecordMetadata 32 | timestamp_created: u64 33 | signature: []u8 34 | 35 | data Tombstone: 36 | key_id: string 37 | issued_by: string 38 | peer_id: string 39 | timestamp_issued: u64 40 | solution: []u8 41 | issuer_signature: []u8 42 | 43 | data EvictStaleItem: 44 | key: Key 45 | records: []Record 46 | tombstones: []Tombstone 47 | 48 | data EvictStaleResult: 49 | success: bool 50 | error: string 51 | results: []EvictStaleItem 52 | 53 | data GetKeyMetadataResult: 54 | success: bool 55 | error: string 56 | key: Key 57 | 58 | data GetRecordsResult: 59 | success: bool 60 | error: string 61 | result: []Record 62 | 63 | data GetTombstonesResult: 64 | success: bool 65 | error: string 66 | result: []Tombstone 67 | 68 | data MergeKeysResult: 69 | success: bool 70 | error: string 71 | key: Key 72 | 73 | data MergeResult: 74 | success: bool 75 | error: string 76 | result: []Record 77 | 78 | data RegisterKeyResult: 79 | success: bool 80 | error: string 81 | key_id: string 82 | 83 | data RegistryResult: 84 | success: bool 85 | error: string 86 | 87 | data RepublishRecordsResult: 88 | success: bool 89 | error: string 90 | updated: u64 91 | 92 | data WeightResult: 93 | success: bool 94 | weight: u32 95 | peer_id: string 96 | error: string 97 | 98 | service Registry("registry"): 99 | add_tombstone(key_id: string, issued_by: string, peer_id: string, timestamp_issued: u64, solution: []u8, signature: []u8, current_timestamp_sec: u64) -> RegistryResult 100 | clear_expired(current_timestamp_sec: u64) -> ClearExpiredResult 101 | create_record_metadata(key_id: string, issued_by: string, timestamp_issued: u64, value: string, peer_id: string, relay_id: []string, service_id: []string, solution: []u8, signature: []u8) -> RecordMetadata 102 | evict_stale(current_timestamp_sec: u64) -> EvictStaleResult 103 | get_key_bytes(label: string, owner_peer_id: []string, timestamp_created: u64, challenge: []u8, challenge_type: string) -> []u8 104 | get_key_id(label: string, peer_id: string) -> string 105 | get_key_metadata(key_id: string) -> GetKeyMetadataResult 106 | get_record_bytes(metadata: RecordMetadata, timestamp_created: u64) -> []u8 107 | get_record_metadata_bytes(key_id: string, issued_by: string, timestamp_issued: u64, value: string, peer_id: string, relay_id: []string, service_id: []string, solution: []u8) -> []u8 108 | get_records(key_id: string, current_timestamp_sec: u64) -> GetRecordsResult 109 | get_stale_local_records(current_timestamp_sec: u64) -> GetRecordsResult 110 | get_tombstone_bytes(key_id: string, issued_by: string, peer_id: string, timestamp_issued: u64, solution: []u8) -> []u8 111 | get_tombstones(key_id: string, current_timestamp_sec: u64) -> GetTombstonesResult 112 | merge(records: [][]Record) -> MergeResult 113 | merge_keys(keys: []Key) -> MergeKeysResult 114 | merge_two(a: []Record, b: []Record) -> MergeResult 115 | put_record(metadata: RecordMetadata, timestamp_created: u64, signature: []u8, weight: WeightResult, current_timestamp_sec: u64) -> RegistryResult 116 | register_key(label: string, owner_peer_id: []string, timestamp_created: u64, challenge: []u8, challenge_type: string, signature: []u8, weight: WeightResult, current_timestamp_sec: u64) -> RegisterKeyResult 117 | republish_key(key: Key, weight: WeightResult, current_timestamp_sec: u64) -> RegistryResult 118 | republish_records(records: []Record, weights: []WeightResult, current_timestamp_sec: u64) -> RepublishRecordsResult 119 | republish_tombstones(tombstones: []Tombstone, current_timestamp_sec: u64) -> RegistryResult 120 | set_expired_timeout(timeout_sec: u64) 121 | set_stale_timeout(timeout_sec: u64) 122 | -------------------------------------------------------------------------------- /aqua/resources-api.aqua: -------------------------------------------------------------------------------- 1 | aqua Registry.ResourcesAPI declares * 2 | 3 | import "registry-service.aqua" 4 | import "registry-api.aqua" 5 | import "misc.aqua" 6 | import "constants.aqua" 7 | import "@fluencelabs/aqua-lib/builtin.aqua" 8 | 9 | func getResource(resource_id: ResourceId) -> ?Resource, *Error: 10 | on HOST_PEER_ID: 11 | result, error <- getResourceHelper(resource_id) 12 | <- result, error 13 | 14 | func getResourceId(label: string, peer_id: string) -> ResourceId: 15 | on HOST_PEER_ID: 16 | resource_id <- Registry.get_key_id(label, peer_id) 17 | <- resource_id 18 | 19 | -- Create a resource: register it on the closest peers 20 | func createResource(label: string) -> ?ResourceId, *Error: 21 | t <- Peer.timestamp_sec() 22 | 23 | resource_id: *ResourceId 24 | error: *Error 25 | on HOST_PEER_ID: 26 | sig_result <- getKeySignature(label, t) 27 | if sig_result.success == false: 28 | error <<- sig_result.error! 29 | else: 30 | signature = sig_result.signature! 31 | id <- Registry.get_key_id(label, INIT_PEER_ID) 32 | nodes <- getNeighbors(id) 33 | 34 | successful: *bool 35 | for n <- nodes par: 36 | on n: 37 | try: 38 | res <- registerKey(label, t, signature) 39 | 40 | if res.success: 41 | successful <<- true 42 | else: 43 | error <<- res.error 44 | 45 | success <- wait(successful, INITIAL_REPLICATION_FACTOR, DEFAULT_TIMEOUT) 46 | 47 | if success == false: 48 | error <<- "resource wasn't created: timeout exceeded" 49 | else: 50 | resource_id <<- id 51 | 52 | <- resource_id, error 53 | 54 | -- Note: resource must be already created 55 | func registerService(resource_id: ResourceId, value: string, peer_id: PeerId, service_id: ?string) -> bool, *Error: 56 | relay_id: *string 57 | if peer_id == INIT_PEER_ID: 58 | relay_id <<- HOST_PEER_ID 59 | 60 | success: *bool 61 | error: *Error 62 | 63 | on HOST_PEER_ID: 64 | metadata, err <- getRecordMetadata(resource_id, value, peer_id, relay_id, service_id, nil) 65 | if metadata == nil: 66 | success <<- false 67 | error <<- err! 68 | else: 69 | t <- Peer.timestamp_sec() 70 | sig_result = getRecordSignature(metadata!, t) 71 | if sig_result.success == false: 72 | error <<- sig_result.error! 73 | success <<- false 74 | else: 75 | key, error_get <- getResourceHelper(resource_id) 76 | if key == nil: 77 | appendErrors(error, error_get) 78 | success <<- false 79 | else: 80 | if peer_id != INIT_PEER_ID: 81 | on peer_id via HOST_PEER_ID: 82 | republish_result <- republishKey(key!) 83 | if republish_result.success == false: 84 | error <<- republish_result.error 85 | else: 86 | p_res <- putRecord(metadata!, t, sig_result.signature!) 87 | if p_res.success == false: 88 | error <<- p_res.error 89 | success <<- false 90 | 91 | nodes <- getNeighbors(resource_id) 92 | successful: *bool 93 | for n <- nodes par: 94 | on n: 95 | try: 96 | republish_res <- republishKey(key!) 97 | if republish_res.success == false: 98 | error <<- republish_res.error 99 | else: 100 | put_res <- putRecord(metadata!, t, sig_result.signature!) 101 | if put_res.success: 102 | successful <<- true 103 | else: 104 | error <<- put_res.error 105 | success <- wait(successful, INITIAL_REPLICATION_FACTOR, DEFAULT_TIMEOUT) 106 | 107 | succ = success! 108 | if succ == false: 109 | error <<- "service hasn't registered: timeout exceeded" 110 | 111 | <- succ, error 112 | 113 | 114 | func unregisterService(resource_id: ResourceId, peer_id: PeerId) -> bool, *Error: 115 | success: *bool 116 | error: *Error 117 | 118 | on HOST_PEER_ID: 119 | t <- Peer.timestamp_sec() 120 | sig_result = getTombstoneSignature(resource_id, peer_id, t, nil) 121 | if sig_result.success == false: 122 | error <<- sig_result.error! 123 | success <<- false 124 | else: 125 | key, error_get <- getResourceHelper(resource_id) 126 | if key == nil: 127 | appendErrors(error, error_get) 128 | success <<- false 129 | else: 130 | 131 | if peer_id != INIT_PEER_ID: 132 | on peer_id: 133 | republish_result <- republishKey(key!) 134 | if republish_result.success == false: 135 | error <<- republish_result.error 136 | else: 137 | res <- addTombstone(resource_id, peer_id, t, nil, sig_result.signature!) 138 | if res.success == false: 139 | error <<- res.error 140 | success <<- false 141 | 142 | nodes <- getNeighbors(resource_id) 143 | successful: *bool 144 | for n <- nodes par: 145 | on n: 146 | try: 147 | republish_res <- republishKey(key!) 148 | if republish_res.success == false: 149 | error <<- republish_res.error 150 | else: 151 | add_res <- addTombstone(resource_id, peer_id, t, nil, sig_result.signature!) 152 | if add_res.success: 153 | successful <<- true 154 | else: 155 | error <<- add_res.error 156 | success <- wait(successful, INITIAL_REPLICATION_FACTOR, DEFAULT_TIMEOUT) 157 | 158 | succ = success! 159 | if succ == false: 160 | error <<- "unregisterService failed: timeout exceeded" 161 | 162 | <- succ, error 163 | 164 | func resolveResource(resource_id: ResourceId, ack: i16) -> ?[]Record, *Error: 165 | on HOST_PEER_ID: 166 | nodes <- getNeighbors(resource_id) 167 | result: *[]Record 168 | records: *[]Record 169 | error: *Error 170 | successful: *bool 171 | for n <- nodes par: 172 | on n: 173 | try: 174 | t <- Peer.timestamp_sec() 175 | get_result <- Registry.get_records(resource_id, t) 176 | if get_result.success: 177 | records <<- get_result.result 178 | successful <<- true 179 | else: 180 | error <<- get_result.error 181 | 182 | success <- wait(successful, ack, DEFAULT_TIMEOUT) 183 | if success == false: 184 | error <<- "timeout exceeded" 185 | else: 186 | merged <- Registry.merge(records) 187 | if merged.success == false: 188 | error <<- merged.error 189 | else: 190 | result <<- merged.result 191 | <- result, error 192 | 193 | -- Execute the given call on providers 194 | -- Note that you can provide another Aqua function as an argument to this one 195 | func executeOnResource(resource_id: ResourceId, ack: i16, call: Record -> ()) -> bool, *Error: 196 | success: *bool 197 | result, error <- resolveResource(resource_id, ack) 198 | 199 | if result == nil: 200 | success <<- false 201 | else: 202 | for r <- result! par: 203 | on r.metadata.peer_id via r.metadata.relay_id: 204 | call(r) 205 | success <<- true 206 | <- success!, error 207 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit -o nounset -o pipefail 4 | set -x 5 | 6 | # set current working directory to script directory to run script from everywhere 7 | cd "$(dirname "$0")" 8 | 9 | # Build the service 10 | ./service/build.sh 11 | 12 | DISTRO_TARGET=distro/registry-service 13 | mkdir -p "$DISTRO_TARGET" 14 | 15 | cd ./aqua 16 | npm pack 17 | cd - 18 | 19 | packed_archive_file_name_pattern="fluencelabs-registry-" 20 | packed_archive_file_name=$(find "./aqua" -type f -name "${packed_archive_file_name_pattern}*") 21 | 22 | cd ./aqua-tests 23 | echo " '@fluencelabs/registry': file:.$packed_archive_file_name" >> "./fluence.yaml" 24 | fluence dep i 25 | fluence aqua -i ./spell/spell.aqua --no-relay --air -o "../$DISTRO_TARGET/air" 26 | cd - 27 | 28 | cp service/artifacts/registry.wasm service/artifacts/sqlite3.wasm distro/Config.toml "$DISTRO_TARGET" 29 | 30 | cd distro 31 | cargo build 32 | -------------------------------------------------------------------------------- /distro/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "built" 7 | version = "0.7.1" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "38d17f4d6e4dc36d1a02fbedc2753a096848e7c1b0772f7654eab8e2c927dd53" 10 | 11 | [[package]] 12 | name = "itoa" 13 | version = "1.0.6" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" 16 | 17 | [[package]] 18 | name = "maplit" 19 | version = "1.0.2" 20 | source = "registry+https://github.com/rust-lang/crates.io-index" 21 | checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" 22 | 23 | [[package]] 24 | name = "registry-distro" 25 | version = "0.9.4" 26 | dependencies = [ 27 | "built", 28 | "maplit", 29 | "serde_json", 30 | ] 31 | 32 | [[package]] 33 | name = "ryu" 34 | version = "1.0.13" 35 | source = "registry+https://github.com/rust-lang/crates.io-index" 36 | checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" 37 | 38 | [[package]] 39 | name = "serde" 40 | version = "1.0.160" 41 | source = "registry+https://github.com/rust-lang/crates.io-index" 42 | checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" 43 | 44 | [[package]] 45 | name = "serde_json" 46 | version = "1.0.96" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" 49 | dependencies = [ 50 | "itoa", 51 | "ryu", 52 | "serde", 53 | ] 54 | -------------------------------------------------------------------------------- /distro/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "registry-distro" 3 | version = "0.9.4" 4 | edition = "2021" 5 | build = "built.rs" 6 | include = [ "/src", "built.rs", "Cargo.toml", "registry-service"] 7 | description = "Distribution package for the registry service including scheduled scripts" 8 | license = "Apache-2.0" 9 | 10 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 11 | [dependencies] 12 | maplit = "1.0.2" 13 | serde_json = "1.0.96" 14 | 15 | [build-dependencies] 16 | built = "0.7.1" 17 | -------------------------------------------------------------------------------- /distro/Config.toml: -------------------------------------------------------------------------------- 1 | modules_dir = "." 2 | total_memory_limit = "Infinity" 3 | 4 | [[module]] 5 | name = "sqlite3" 6 | mem_pages_count = 100 7 | logger_enabled = false 8 | 9 | [module.wasi] 10 | preopened_files = ["./tmp"] 11 | mapped_dirs = { "tmp" = "./tmp" } 12 | 13 | [[module]] 14 | name = "registry" 15 | mem_pages_count = 1 16 | logger_enabled = false 17 | -------------------------------------------------------------------------------- /distro/built.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | built::write_built_file().expect("Failed to acquire build-time information") 3 | } 4 | -------------------------------------------------------------------------------- /distro/src/lib.rs: -------------------------------------------------------------------------------- 1 | use maplit::hashmap; 2 | use std::collections::HashMap; 3 | use serde_json::{json, Value as JValue}; 4 | 5 | pub const REGISTRY_WASM: &'static [u8] = include_bytes!("../registry-service/registry.wasm"); 6 | pub const SQLITE_WASM: &'static [u8] = include_bytes!("../registry-service/sqlite3.wasm"); 7 | pub const CONFIG: &'static [u8] = include_bytes!("../registry-service/Config.toml"); 8 | 9 | pub const REGISTRY_SPELL: &'static str = 10 | include_str!("../registry-service/air/spell.spell.air"); 11 | 12 | pub mod build_info { 13 | include!(concat!(env!("OUT_DIR"), "/built.rs")); 14 | } 15 | 16 | pub use build_info::PKG_VERSION as VERSION; 17 | 18 | pub fn modules() -> std::collections::HashMap<&'static str, &'static [u8]> { 19 | maplit::hashmap! { 20 | "sqlite3" => SQLITE_WASM, 21 | "registry" => REGISTRY_WASM, 22 | } 23 | } 24 | 25 | pub struct DistrSpell { 26 | /// AIR script of the spell 27 | pub air: &'static str, 28 | /// Initial key-value records for spells KV storage 29 | pub init_data: HashMap<&'static str, JValue>, 30 | } 31 | 32 | 33 | #[derive(Debug)] 34 | pub struct RegistryConfig { 35 | pub expired_interval: u32, 36 | pub renew_interval: u32, 37 | pub replicate_interval: u32 38 | } 39 | 40 | pub fn registry_spell(config: RegistryConfig) -> DistrSpell { 41 | DistrSpell { 42 | air: REGISTRY_SPELL, 43 | init_data: hashmap!{ 44 | "config" => json!( { 45 | "expired_interval": config.expired_interval, 46 | "renew_interval": config.renew_interval, 47 | "replicate_interval": config.replicate_interval, 48 | }), 49 | }, 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /examples/archived/1-registry/.fluence/aqua/deals.aqua: -------------------------------------------------------------------------------- 1 | aqua Deals declares * 2 | 3 | data Deal: 4 | definition: string 5 | timestamp: string 6 | dealIdOriginal: string 7 | dealId: string 8 | chainNetwork: string 9 | chainNetworkId: u64 10 | 11 | data Deals: 12 | dealName: ?Deal 13 | 14 | func get() -> Deals: 15 | <- Deals( 16 | dealName=?[Deal( 17 | definition="bafkreidqtqpmmferdscg4bqrs74cl6ckib3vyhvejhrc4watln5xxcrj2i", 18 | timestamp="2023-12-19T20:01:24.334Z", 19 | dealIdOriginal="0xEb92A1B5c10AD7BFdcaf23Cb7DDA9ea062CD07E8", 20 | dealId="eb92a1b5c10ad7bfdcaf23cb7dda9ea062cd07e8", 21 | chainNetwork="local", 22 | chainNetworkId=31337 23 | )] 24 | ) 25 | -------------------------------------------------------------------------------- /examples/archived/1-registry/.fluence/aqua/hosts.aqua: -------------------------------------------------------------------------------- 1 | aqua Hosts declares * 2 | 3 | func get() -> ?u8: 4 | <- nil 5 | -------------------------------------------------------------------------------- /examples/archived/1-registry/.fluence/aqua/services.aqua: -------------------------------------------------------------------------------- 1 | service EchoService("echo_service"): 2 | echo(msg: string) -> string 3 | -------------------------------------------------------------------------------- /examples/archived/1-registry/.fluence/configs/nox-0_Config.toml: -------------------------------------------------------------------------------- 1 | aquavm_pool_size = 2 2 | tcp_port = 7_771 3 | websocket_port = 9_991 4 | http_port = 18_080 5 | 6 | [system_services] 7 | enable = [ "registry", "decider" ] 8 | 9 | [system_services.aqua_ipfs] 10 | external_api_multiaddr = "/ip4/127.0.0.1/tcp/5001" 11 | local_api_multiaddr = "/dns4/ipfs/tcp/5001" 12 | 13 | [system_services.decider] 14 | decider_period_sec = 10 15 | worker_ipfs_multiaddr = "/dns4/ipfs/tcp/5001" 16 | network_api_endpoint = "http://chain:8545" 17 | network_id = 31_337 18 | start_block = "earliest" 19 | matcher_address = "0x0e1F3B362E22B2Dc82C9E35d6e62998C7E8e2349" 20 | wallet_key = "0x3cc23e0227bd17ea5d6ea9d42b5eaa53ad41b1974de4755c79fe236d361a6fd5" 21 | -------------------------------------------------------------------------------- /examples/archived/1-registry/.fluence/configs/nox-1_Config.toml: -------------------------------------------------------------------------------- 1 | aquavm_pool_size = 2 2 | tcp_port = 7_772 3 | websocket_port = 9_992 4 | http_port = 18_081 5 | 6 | [system_services] 7 | enable = [ "registry", "decider" ] 8 | 9 | [system_services.aqua_ipfs] 10 | external_api_multiaddr = "/ip4/127.0.0.1/tcp/5001" 11 | local_api_multiaddr = "/dns4/ipfs/tcp/5001" 12 | 13 | [system_services.decider] 14 | decider_period_sec = 10 15 | worker_ipfs_multiaddr = "/dns4/ipfs/tcp/5001" 16 | network_api_endpoint = "http://chain:8545" 17 | network_id = 31_337 18 | start_block = "earliest" 19 | matcher_address = "0x0e1F3B362E22B2Dc82C9E35d6e62998C7E8e2349" 20 | wallet_key = "0x089162470bcfc93192b95bff0a1860d063266875c782af9d882fcca125323b41" 21 | -------------------------------------------------------------------------------- /examples/archived/1-registry/.fluence/configs/nox-2_Config.toml: -------------------------------------------------------------------------------- 1 | aquavm_pool_size = 2 2 | tcp_port = 7_773 3 | websocket_port = 9_993 4 | http_port = 18_082 5 | 6 | [system_services] 7 | enable = [ "registry", "decider" ] 8 | 9 | [system_services.aqua_ipfs] 10 | external_api_multiaddr = "/ip4/127.0.0.1/tcp/5001" 11 | local_api_multiaddr = "/dns4/ipfs/tcp/5001" 12 | 13 | [system_services.decider] 14 | decider_period_sec = 10 15 | worker_ipfs_multiaddr = "/dns4/ipfs/tcp/5001" 16 | network_api_endpoint = "http://chain:8545" 17 | network_id = 31_337 18 | start_block = "earliest" 19 | matcher_address = "0x0e1F3B362E22B2Dc82C9E35d6e62998C7E8e2349" 20 | wallet_key = "0xdacd4b197ee7e9efdd5db1921c6c558d88e2c8b69902b8bafc812fb226a6b5e0" 21 | -------------------------------------------------------------------------------- /examples/archived/1-registry/.fluence/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=schemas/docker-compose.json 2 | 3 | # Defines a multi-containers based application. 4 | 5 | # Documentation: https://github.com/fluencelabs/cli/tree/main/docs/configs/docker-compose.md 6 | 7 | version: "3" 8 | 9 | services: 10 | chain: 11 | image: fluencelabs/chain-rpc:0.2.20 12 | ports: 13 | - 8545:8545 14 | ipfs: 15 | image: ipfs/go-ipfs 16 | ports: 17 | - 5001:5001 18 | - 4001:4001 19 | environment: 20 | IPFS_PROFILE: server 21 | volumes: 22 | - ./ipfs/:/container-init.d/ 23 | nox-0: 24 | image: fluencelabs/nox:0.16.3 25 | pull_policy: always 26 | ports: 27 | - 7771:7771 28 | - 9991:9991 29 | environment: 30 | WASM_LOG: info 31 | RUST_LOG: debug,particle_reap=debug,aquamarine=warn,aquamarine::particle_functions=debug,aquamarine::log=debug,aquamarine::aqua_runtime=error,ipfs_effector=off,ipfs_pure=off,system_services=debug,marine_core::module::marine_module=info,tokio_threadpool=info,tokio_reactor=info,mio=info,tokio_io=info,soketto=info,yamux=info,multistream_select=info,libp2p_secio=info,libp2p_websocket::framed=info,libp2p_ping=info,libp2p_core::upgrade::apply=info,libp2p_kad::kbucket=info,cranelift_codegen=info,wasmer_wasi=info,cranelift_codegen=info,wasmer_wasi=info,run-console=trace,wasmtime_cranelift=off,wasmtime_jit=off,libp2p_tcp=off,libp2p_swarm=off,particle_protocol::libp2p_protocol::upgrade=info,libp2p_mplex=off,particle_reap=off,netlink_proto=warn 32 | FLUENCE_MAX_SPELL_PARTICLE_TTL: 9s 33 | FLUENCE_ROOT_KEY_PAIR__PATH: /run/secrets/nox-0 34 | command: 35 | - --config=/run/configs/nox-0_Config.toml 36 | - --external-maddrs 37 | - /dns4/nox-0/tcp/7771 38 | - /dns4/nox-0/tcp/9991/ws 39 | - --allow-private-ips 40 | - --local 41 | depends_on: 42 | - ipfs 43 | volumes: 44 | - ./configs/nox-0_Config.toml:/run/configs/nox-0_Config.toml 45 | secrets: 46 | - nox-0 47 | nox-1: 48 | image: fluencelabs/nox:0.16.3 49 | pull_policy: always 50 | ports: 51 | - 7772:7772 52 | - 9992:9992 53 | environment: 54 | WASM_LOG: info 55 | RUST_LOG: debug,particle_reap=debug,aquamarine=warn,aquamarine::particle_functions=debug,aquamarine::log=debug,aquamarine::aqua_runtime=error,ipfs_effector=off,ipfs_pure=off,system_services=debug,marine_core::module::marine_module=info,tokio_threadpool=info,tokio_reactor=info,mio=info,tokio_io=info,soketto=info,yamux=info,multistream_select=info,libp2p_secio=info,libp2p_websocket::framed=info,libp2p_ping=info,libp2p_core::upgrade::apply=info,libp2p_kad::kbucket=info,cranelift_codegen=info,wasmer_wasi=info,cranelift_codegen=info,wasmer_wasi=info,run-console=trace,wasmtime_cranelift=off,wasmtime_jit=off,libp2p_tcp=off,libp2p_swarm=off,particle_protocol::libp2p_protocol::upgrade=info,libp2p_mplex=off,particle_reap=off,netlink_proto=warn 56 | FLUENCE_MAX_SPELL_PARTICLE_TTL: 9s 57 | FLUENCE_ROOT_KEY_PAIR__PATH: /run/secrets/nox-1 58 | command: 59 | - --config=/run/configs/nox-1_Config.toml 60 | - --external-maddrs 61 | - /dns4/nox-1/tcp/7772 62 | - /dns4/nox-1/tcp/9992/ws 63 | - --allow-private-ips 64 | - --bootstraps=/dns/nox-0/tcp/7771 65 | depends_on: 66 | - ipfs 67 | volumes: 68 | - ./configs/nox-1_Config.toml:/run/configs/nox-1_Config.toml 69 | secrets: 70 | - nox-1 71 | nox-2: 72 | image: fluencelabs/nox:0.16.3 73 | pull_policy: always 74 | ports: 75 | - 7773:7773 76 | - 9993:9993 77 | environment: 78 | WASM_LOG: info 79 | RUST_LOG: debug,particle_reap=debug,aquamarine=warn,aquamarine::particle_functions=debug,aquamarine::log=debug,aquamarine::aqua_runtime=error,ipfs_effector=off,ipfs_pure=off,system_services=debug,marine_core::module::marine_module=info,tokio_threadpool=info,tokio_reactor=info,mio=info,tokio_io=info,soketto=info,yamux=info,multistream_select=info,libp2p_secio=info,libp2p_websocket::framed=info,libp2p_ping=info,libp2p_core::upgrade::apply=info,libp2p_kad::kbucket=info,cranelift_codegen=info,wasmer_wasi=info,cranelift_codegen=info,wasmer_wasi=info,run-console=trace,wasmtime_cranelift=off,wasmtime_jit=off,libp2p_tcp=off,libp2p_swarm=off,particle_protocol::libp2p_protocol::upgrade=info,libp2p_mplex=off,particle_reap=off,netlink_proto=warn 80 | FLUENCE_MAX_SPELL_PARTICLE_TTL: 9s 81 | FLUENCE_ROOT_KEY_PAIR__PATH: /run/secrets/nox-2 82 | command: 83 | - --config=/run/configs/nox-2_Config.toml 84 | - --external-maddrs 85 | - /dns4/nox-2/tcp/7773 86 | - /dns4/nox-2/tcp/9993/ws 87 | - --allow-private-ips 88 | - --bootstraps=/dns/nox-0/tcp/7771 89 | depends_on: 90 | - ipfs 91 | volumes: 92 | - ./configs/nox-2_Config.toml:/run/configs/nox-2_Config.toml 93 | secrets: 94 | - nox-2 95 | 96 | secrets: 97 | nox-0: 98 | file: secrets/nox-0.txt 99 | nox-1: 100 | file: secrets/nox-1.txt 101 | nox-2: 102 | file: secrets/nox-2.txt 103 | -------------------------------------------------------------------------------- /examples/archived/1-registry/.fluence/workers.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=schemas/workers.json 2 | 3 | # A result of app deployment. This file is created automatically after successful deployment using `fluence workers deploy` command 4 | 5 | # Documentation: https://github.com/fluencelabs/cli/tree/main/docs/configs/workers.md 6 | 7 | version: 1 8 | 9 | deals: 10 | local: 11 | dealName: 12 | definition: bafkreidqtqpmmferdscg4bqrs74cl6ckib3vyhvejhrc4watln5xxcrj2i 13 | timestamp: 2023-12-19T20:01:24.334Z 14 | dealIdOriginal: "0xEb92A1B5c10AD7BFdcaf23Cb7DDA9ea062CD07E8" 15 | dealId: eb92a1b5c10ad7bfdcaf23cb7dda9ea062cd07e8 16 | chainNetwork: local 17 | chainNetworkId: 31337 18 | -------------------------------------------------------------------------------- /examples/archived/1-registry/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .DS_Store 3 | /.fluence/secrets 4 | /.fluence/env.yaml 5 | /.fluence/schemas 6 | /.fluence/tmp 7 | **/node_modules 8 | **/target/ 9 | .repl_history 10 | /.vscode/settings.json 11 | /src/ts/src/aqua 12 | /src/js/src/aqua -------------------------------------------------------------------------------- /examples/archived/1-registry/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "redhat.vscode-yaml", 4 | "FluenceLabs.aqua" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /examples/archived/1-registry/Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ "src/services/echo_service/modules/echo_service" ] 3 | -------------------------------------------------------------------------------- /examples/archived/1-registry/README.md: -------------------------------------------------------------------------------- 1 | # Services advertisement and discovery 2 | 3 | ## Overview 4 | 5 | This example shows how to use Registry to discover and call fluence services without having their exact peer and service ids. 6 | 7 | ## Table of contents: 8 | 9 | - [Services advertisement and discovery](#services-advertisement-and-discovery) 10 | - [Overview](#overview) 11 | - [Table of contents:](#table-of-contents) 12 | - [Set up the environment](#set-up-the-environment) 13 | - [Deploy echo service written in Rust](#deploy-echo-service-written-in-rust) 14 | - [Run echo service written in JS/TS](#run-echo-service-written-in-jsts) 15 | - [Register both services using Registry](#register-both-services-using-registry) 16 | - [Call both services using resourceId](#call-both-services-using-resourceid) 17 | - [Remove service record](#remove-service-record) 18 | 19 | ## Set up the environment 20 | 21 | 1. [Install the latest version of Fluence CLI](https://github.com/fluencelabs/cli#installation-and-usage) 22 | 2. Install Fluence project dependencies. It may take a while: 23 | ```sh 24 | fluence dep i 25 | ``` 26 | 3. Install JS dependencies: 27 | ```sh 28 | npm i 29 | ``` 30 | You can also use VSCode with [Aqua extension](https://marketplace.visualstudio.com/items?itemName=FluenceLabs.aqua) for [Aqua language](https://fluence.dev/docs/aqua-book/getting-started/) syntax highlighting and better developer experience. 31 | 32 | ## Deploy echo service written in Rust 33 | 34 | To deploy the Fluence application execute 35 | ```sh 36 | fluence deploy 37 | ``` 38 | Press Enter when prompted `? Do you want to deploy all of these services? (Y/n)` 39 | 40 | This Fluence application, described in [fluence.yaml](fluence.yaml), consists of just one [echo service](./echo_service) which has only one [module](./echo_service/modules/echo_service/) written in Rust. [The module code](echo_service/modules/echo_service/src/main.rs) has only one function [echo](echo_service/modules/echo_service/src/main.rs#L9), which returns your `msg` along with peerId of the host: 41 | 42 | To call [echo](src/aqua/main.aqua#L8) aqua function execute: 43 | ```sh 44 | fluence run -f 'echo("hi")' 45 | ``` 46 | The function uses `peerId` and `serviceId`, which Fluence CLI stored in `./.fluence/app.yaml` when you deployed the Fluence application in the previous step. 47 | 48 | You should see output similar to this: 49 | ``` 50 | "12D3KooWFEwNWcHqi9rtsmDhsYcDbRUCDXH84RC4FW6UfsFWaoHi: hi" 51 | ``` 52 | 53 | It means we successfully deployed our echo service, and anyone can call it if they have `peerId` and `serviceId` 54 | 55 | ## Run echo service written in JS/TS 56 | 57 | Execute 58 | ```sh 59 | npm run start 60 | ``` 61 | 62 | First, aqua code in [src/aqua/export.aqua](src/aqua/export.aqua) will be compiled to typescript and you will see it in [src/generated/export.ts](src/generated/export.ts). 63 | 64 | Then you possibly will have to confirm ts-node installation and [src/echo.ts](src/echo.ts) will be executed. It registers local js service with serviceId "echo", so anyone who has `relayId`, `peerId` and `serviceId` ("echo") will be able to call it. Copy the command from the terminal, which will look similar to this: 65 | ```sh 66 | fluence run -f 'echoJS("12D3KooWCmnhnGvKTqEXpVLzdrYu3TkQ3HcLyArGJpLPooJQ69dN", "12D3KooWSD5PToNiLQwKDXsu8JSysCwUt8BVUJEqCHcDe7P5h45e", "echo", "hi")' 67 | ``` 68 | This command executes [echoJS](src/aqua/main.aqua#L16) aqua function with arguments: relayId, peerId, serviceId and msg 69 | 70 | Open another terminal in the same directory, paste the command and run it. 71 | 72 | You should see output similar to this: 73 | ``` 74 | "12D3KooWCmnhnGvKTqEXpVLzdrYu3TkQ3HcLyArGJpLPooJQ69dN: hi" 75 | ``` 76 | 77 | It means anyone can call our `echo` service, written in TS/JS, if they have `relayId`, `peerId` and `serviceId`. 78 | ## Register both services using Registry 79 | 80 | We can register our services in Registry if we want anyone to be able to call our services without specifying the exact relay, peer, and service IDs. 81 | 82 | First, we need to create the Resource. The Resource represents a group of services and has a corresponding `resourceId` which we can use for service discovery. 83 | 84 | To call [createRes](src/aqua/main.aqua#L22) aqua function, execute 85 | ```sh 86 | fluence run -f 'createRes()' 87 | ``` 88 | It uses `createResource` function from Resources API to register the Resource with the label `echo`. 89 | You should see output similar to this: 90 | 91 | ``` 92 | 5pYpWB3ozi6fi1EjNs9X5kE156aA6iLECxTuVdJgUaLB 93 | ``` 94 | 95 | It is `resourceId`, which we will use to register our services, and then we will be able to use the same `resourceId` to discover and call our services 96 | 97 | To register the `echo` service written in Rust, replace `RESOURCE_ID` and execute 98 | ```sh 99 | fluence run -f 'registerEchoService("RESOURCE_ID")' 100 | ``` 101 | This command calls [registerEchoService](src/aqua/main.aqua#L26) aqua function, which uses `registerService` function from Resources API to register the rust service on this `resourceId` 102 | 103 | You should see this output: 104 | ``` 105 | [ 106 | true, 107 | [] 108 | ] 109 | ``` 110 | It means the service is registered in Registry and should be accessible by anyone who only has the `resourceId` of this service. 111 | 112 | Then please stop fluence js peer in the previous terminal that you ran. 113 | 114 | To register echo service written in JS/TS on the Resource, replace `RESOURCE_ID` and execute 115 | ```sh 116 | npm run start -- 'RESOURCE_ID' 117 | ``` 118 | ## Call both services using resourceId 119 | Go to a different terminal in the same directory, replace `RESOURCE_ID` and execute this command to call [echoAll](src/aqua/main.aqua#L33) aqua function 120 | ```sh 121 | fluence run -f 'echoAll("RESOURCE_ID", "hi")' 122 | ``` 123 | It uses `resourceId` to resolve a minimum of two records with peer and service ids and then uses them to call our services 124 | 125 | You should see output similar to this: 126 | ``` 127 | [ 128 | [ 129 | "12D3KooWFEwNWcHqi9rtsmDhsYcDbRUCDXH84RC4FW6UfsFWaoHi: hi", 130 | "12D3KooWCmnhnGvKTqEXpVLzdrYu3TkQ3HcLyArGJpLPooJQ69dN: hi" 131 | ] 132 | ] 133 | ``` 134 | It means we successfully registered our services using Registry, and now anyone can call these services using only `resourceId`. 135 | 136 | ## Remove service record 137 | Replace `RESOURCE_ID` and execute 138 | ```sh 139 | fluence run -f 'unregisterEchoService("RESOURCE_ID")' 140 | ``` 141 | to call [unregisterEchoService](src/aqua/main.aqua#L43) function that uses `unregisterService` function from Resources API to unregister only our echo services written in Rust 142 | 143 | The output should look like this: 144 | ``` 145 | [ 146 | [ 147 | true 148 | ] 149 | ] 150 | ``` 151 | Let's make sure we've removed the service record. Once again, replace `RESOURCE_ID` and execute 152 | ```sh 153 | fluence run -f 'echoAll("RESOURCE_ID", "hi")' 154 | ``` 155 | 156 | You should see output similar to this: 157 | ``` 158 | [ 159 | [ 160 | "12D3KooWCmnhnGvKTqEXpVLzdrYu3TkQ3HcLyArGJpLPooJQ69dN: hi" 161 | ] 162 | ] 163 | ``` 164 | You can notice that only one result is left instead of two. It means we successfully removed the service record from our Resource 165 | -------------------------------------------------------------------------------- /examples/archived/1-registry/fluence.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=.fluence/schemas/fluence.json 2 | 3 | # Defines Fluence Project, most importantly - what exactly you want to deploy and how. You can use `fluence init` command to generate a template for new Fluence project 4 | 5 | # Documentation: https://github.com/fluencelabs/cli/tree/main/docs/configs/fluence.md 6 | 7 | version: 5 8 | 9 | aquaInputPath: src/aqua/main.aqua 10 | 11 | deals: 12 | dealName: 13 | minWorkers: 1 14 | targetWorkers: 3 15 | services: [ echo_service ] 16 | spells: [] 17 | 18 | services: 19 | echo_service: 20 | get: src/services/echo_service 21 | 22 | relaysPath: src/frontend/src 23 | 24 | aquaOutputTSPath: src/frontend/src/compiled-aqua 25 | -------------------------------------------------------------------------------- /examples/archived/1-registry/provider.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=.fluence/schemas/provider.json 2 | 3 | # Defines config used for provider set up 4 | 5 | # Documentation: https://github.com/fluencelabs/cli/tree/main/docs/configs/provider.md 6 | 7 | version: 0 8 | 9 | env: local 10 | 11 | nox: 12 | systemServices: 13 | enable: 14 | - registry 15 | - decider 16 | 17 | computePeers: 18 | nox-0: 19 | computeUnits: 1 20 | nox-1: 21 | computeUnits: 1 22 | nox-2: 23 | computeUnits: 1 24 | 25 | offers: 26 | offer-0: 27 | maxCollateralPerWorker: 1 28 | minPricePerWorkerEpoch: 0.1 29 | computePeers: 30 | - nox-0 31 | - nox-1 32 | - nox-2 33 | -------------------------------------------------------------------------------- /examples/archived/1-registry/src/aqua/main.aqua: -------------------------------------------------------------------------------- 1 | import "@fluencelabs/aqua-lib/builtin.aqua" 2 | import "@fluencelabs/aqua-lib/subnet.aqua" 3 | import createResource, registerService, resolveResource from "@fluencelabs/registry/resources-api.aqua" 4 | 5 | use "deals.aqua" 6 | use "hosts.aqua" 7 | import "services.aqua" 8 | 9 | service EchoJSService: 10 | echo(msg: string) -> string 11 | 12 | func echo(msg: string) -> string: 13 | deals <- Deals.get() 14 | dealId = deals.dealName!.dealIdOriginal 15 | 16 | on HOST_PEER_ID: 17 | subnet <- Subnet.resolve(dealId) 18 | 19 | if subnet.success == false: 20 | Console.print(["Failed to resolve subnet: ", subnet.error]) 21 | 22 | w = subnet.workers! 23 | 24 | on w.worker_id! via w.host_id: 25 | res <- EchoService.echo(msg) 26 | <- res 27 | 28 | 29 | func echoJS(peerId: string, relayId: string, serviceId: string, msg: string) -> string: 30 | on peerId via relayId: 31 | EchoService serviceId 32 | res <- EchoService.echo(msg) 33 | <- res 34 | 35 | func createRes(label: string) -> ?string: 36 | resourceId, error <- createResource(label) 37 | <- error 38 | 39 | func registerResourceService(resourceId: string, serviceId: string) -> bool, *string: 40 | on HOST_PEER_ID: 41 | -- TODO: change hardcoded local peer to resolve 42 | res, message <- registerService(resourceId, "" , "12D3KooWBM3SdXWqGaawQDGQ6JprtwswEg3FWGvGhmgmMez1vRbR", [serviceId]) 43 | <- res, message 44 | 45 | 46 | func echoAll(resourceId: string, msg: string) -> *string: 47 | -- 2 is the min number of peers we want to ask 48 | records <- resolveResource(resourceId, 2) 49 | results: *string 50 | for r <- records!: 51 | on HOST_PEER_ID: 52 | EchoService r.metadata.service_id! 53 | results <- EchoService.echo(msg) 54 | <- results 55 | 56 | func showSubnets() -> *string: 57 | deals <- Deals.get() 58 | dealId = deals.dealName!.dealIdOriginal 59 | 60 | on HOST_PEER_ID: 61 | results: *string 62 | subnet <- Subnet.resolve(dealId) 63 | 64 | if subnet.success == false: 65 | Console.print(["Failed to resolve subnet: ", subnet.error]) 66 | 67 | for w <- subnet.workers: 68 | results <<- w.host_id 69 | 70 | <- results 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /examples/archived/1-registry/src/frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "module", 3 | "private": true, 4 | "name": "echo", 5 | "version": "0.0.0", 6 | "description": "Fluence Peer with echo service", 7 | "scripts": { 8 | "start": "node --loader ts-node/esm src/echo.ts" 9 | }, 10 | "keywords": [ 11 | "aqua", 12 | "dht", 13 | "p2p" 14 | ], 15 | "author": "Fluence Labs", 16 | "dependencies": { 17 | "@fluencelabs/js-client": "0.5.3" 18 | }, 19 | "devDependencies": { 20 | "@fluencelabs/registry": "^0.9.2", 21 | "ts-node": "10.9.2", 22 | "typescript": "5.0.2" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /examples/archived/1-registry/src/frontend/src/echo.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2022 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | import { Fluence, KeyPair } from '@fluencelabs/js-client'; 17 | import { registerEchoJSService } from './compiled-aqua/main.ts'; 18 | 19 | // don't store your secret key in the code. This is just for the example 20 | const secretKey = "Iz3HUmNIB78lkNNVmMkDKrju0nCivtkJNyObrFAr774="; 21 | 22 | async function main() { 23 | const keyPair = await KeyPair.fromEd25519SK(Buffer.from(secretKey, "base64")); 24 | 25 | await Fluence.connect({ 26 | multiaddr: 27 | "/ip4/127.0.0.1/tcp/9991/ws/p2p/12D3KooWBM3SdXWqGaawQDGQ6JprtwswEg3FWGvGhmgmMez1vRbR", 28 | peerId: "12D3KooWBM3SdXWqGaawQDGQ6JprtwswEg3FWGvGhmgmMez1vRbR", 29 | }, { keyPair: { 30 | type: 'Ed25519', 31 | source: keyPair.toEd25519PrivateKey() 32 | }}); 33 | 34 | const peerId = Fluence.getClient().getPeerId(); 35 | const relayId = Fluence.getClient().getRelayPeerId(); 36 | 37 | console.log(`📗 created a fluence peer ${peerId} with relay ${relayId}`); 38 | 39 | const serviceId = "echo"; 40 | 41 | // register local service with serviceId "echo" 42 | registerEchoJSService(serviceId, { 43 | echo(msg) { 44 | console.log(`Received message: ${msg}`); 45 | return `${peerId}: ${msg}`; 46 | }, 47 | }); 48 | 49 | const resourceId = process.argv[2]; 50 | 51 | // don't register if resource id isn't passed 52 | if (resourceId === undefined) { 53 | console.log( 54 | ` 55 | Copy this code to call this service: 56 | 57 | fluence run -f 'echoJS("${peerId}", "${relayId}", "${serviceId}", "hi")'` 58 | ); 59 | } else { 60 | // const [success, error] = await registerService( 61 | // resourceId, 62 | // "echo", 63 | // peerId, 64 | // serviceId 65 | // ); 66 | // console.log(`Registration result: ${success || error}`); 67 | } 68 | 69 | console.log("\nPress any key to stop fluence js peer"); 70 | 71 | // this code keeps fluence client running till any key pressed 72 | process.stdin.setRawMode(true); 73 | process.stdin.resume(); 74 | process.stdin.on("data", async () => { 75 | await Fluence.disconnect(); 76 | process.exit(0); 77 | }); 78 | } 79 | 80 | main().catch((error) => { 81 | console.error(error); 82 | process.exit(1); 83 | }); 84 | -------------------------------------------------------------------------------- /examples/archived/1-registry/src/frontend/src/relays.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "multiaddr": "/ip4/127.0.0.1/tcp/9991/ws/p2p/12D3KooWJTYHn4U8jJtL1XZvTonAgv2Tn6EEbZSauw56dhr3SNKg", 4 | "peerId": "12D3KooWJTYHn4U8jJtL1XZvTonAgv2Tn6EEbZSauw56dhr3SNKg" 5 | }, 6 | { 7 | "multiaddr": "/ip4/127.0.0.1/tcp/9992/ws/p2p/12D3KooWQrMQg2Ksqag5465Tnu8VQH3c4Z4NSosdS854bAsHEcwo", 8 | "peerId": "12D3KooWQrMQg2Ksqag5465Tnu8VQH3c4Z4NSosdS854bAsHEcwo" 9 | }, 10 | { 11 | "multiaddr": "/ip4/127.0.0.1/tcp/9993/ws/p2p/12D3KooWQCYhkDv4jPe7ymEo8AwRNMzLZRmfyrbV53vKpVS7fZA7", 12 | "peerId": "12D3KooWQCYhkDv4jPe7ymEo8AwRNMzLZRmfyrbV53vKpVS7fZA7" 13 | } 14 | ] -------------------------------------------------------------------------------- /examples/archived/1-registry/src/frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "useDefineForClassFields": true, 5 | "module": "ESNext", 6 | "lib": ["ES2020", "DOM", "DOM.Iterable"], 7 | "skipLibCheck": true, 8 | 9 | /* Bundler mode */ 10 | "moduleResolution": "bundler", 11 | "allowImportingTsExtensions": true, 12 | "resolveJsonModule": true, 13 | "isolatedModules": true, 14 | "noEmit": true, 15 | 16 | /* Linting */ 17 | "strict": true, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": true, 20 | "noFallthroughCasesInSwitch": true 21 | }, 22 | "include": ["src"] 23 | } 24 | -------------------------------------------------------------------------------- /examples/archived/1-registry/src/services/echo_service/modules/echo_service/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "echo_service" 3 | version = "0.1.0" 4 | edition = "2018" 5 | 6 | [[bin]] 7 | name = "echo_service" 8 | path = "src/main.rs" 9 | 10 | [dependencies] 11 | marine-rs-sdk = "0.10.2" 12 | 13 | [dev-dependencies] 14 | marine-rs-sdk-test = "=0.12.0" 15 | -------------------------------------------------------------------------------- /examples/archived/1-registry/src/services/echo_service/modules/echo_service/module.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=../../../../../.fluence/schemas/module.json 2 | 3 | # Defines [Marine Module](https://fluence.dev/docs/build/concepts/#modules). You can use `fluence module new` command to generate a template for new module 4 | 5 | # Documentation: https://github.com/fluencelabs/cli/tree/main/docs/configs/module.md 6 | 7 | version: 0 8 | 9 | type: rust 10 | 11 | name: echo_service 12 | -------------------------------------------------------------------------------- /examples/archived/1-registry/src/services/echo_service/modules/echo_service/src/main.rs: -------------------------------------------------------------------------------- 1 | use marine_rs_sdk::marine; 2 | use marine_rs_sdk::module_manifest; 3 | 4 | module_manifest!(); 5 | 6 | pub fn main() {} 7 | 8 | #[marine] 9 | pub fn echo(msg: String) -> String { 10 | format!("{}: {}", marine_rs_sdk::get_call_parameters().host_id, msg) 11 | } -------------------------------------------------------------------------------- /examples/archived/1-registry/src/services/echo_service/service.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=../../../.fluence/schemas/service.json 2 | 3 | # Defines a [Marine service](https://fluence.dev/docs/build/concepts/#services), most importantly the modules that the service consists of. You can use `fluence service new` command to generate a template for new service 4 | 5 | # Documentation: https://github.com/fluencelabs/cli/tree/main/docs/configs/service.md 6 | 7 | version: 0 8 | 9 | name: echo_service 10 | 11 | modules: 12 | facade: 13 | get: modules/echo_service 14 | -------------------------------------------------------------------------------- /images/availability.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluencelabs/registry/41e9b5ac8756900a26cd221c9aa89abd142f866b/images/availability.png -------------------------------------------------------------------------------- /images/decentralized.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluencelabs/registry/41e9b5ac8756900a26cd221c9aa89abd142f866b/images/decentralized.png -------------------------------------------------------------------------------- /images/discovery.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluencelabs/registry/41e9b5ac8756900a26cd221c9aa89abd142f866b/images/discovery.png -------------------------------------------------------------------------------- /images/mapping.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluencelabs/registry/41e9b5ac8756900a26cd221c9aa89abd142f866b/images/mapping.png -------------------------------------------------------------------------------- /images/registry.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluencelabs/registry/41e9b5ac8756900a26cd221c9aa89abd142f866b/images/registry.png -------------------------------------------------------------------------------- /images/subnetwork.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fluencelabs/registry/41e9b5ac8756900a26cd221c9aa89abd142f866b/images/subnetwork.png -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "nightly-2023-08-27" 3 | components = [ "rustfmt", "clippy" ] 4 | targets = [ "x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "wasm32-wasi" ] 5 | profile = "minimal" 6 | -------------------------------------------------------------------------------- /service/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "registry" 3 | version = "0.9.4" 4 | authors = ["Fluence Labs"] 5 | edition = "2018" 6 | publish = false 7 | 8 | [[bin]] 9 | name = "registry" 10 | path = "src/main.rs" 11 | 12 | [dependencies] 13 | bs58 = "=0.5.0" 14 | marine-rs-sdk = "=0.10.3" 15 | marine-sqlite-connector = "=0.9.3" 16 | fstrings = "=0.2.3" 17 | boolinator = "=2.4.0" 18 | toml = "=0.8.8" 19 | serde = { version = "1.0.188", features = ["derive"] } 20 | thiserror = "=1.0.50" 21 | sha2 = "=0.10.8" 22 | fluence-keypair = "=0.10.4" 23 | libp2p-identity = "=0.2.8" 24 | 25 | [dev-dependencies] 26 | marine-rs-sdk-test = "=0.12.1" 27 | rusqlite = "=0.30.0" 28 | 29 | [build-dependencies] 30 | marine-rs-sdk-test = "=0.12.1" 31 | -------------------------------------------------------------------------------- /service/Config.toml: -------------------------------------------------------------------------------- 1 | modules_dir = "artifacts" 2 | total_memory_limit = "Infinity" 3 | 4 | [[module]] 5 | name = "sqlite3" 6 | mem_pages_count = 100 7 | logger_enabled = false 8 | 9 | [[module]] 10 | name = "registry" 11 | mem_pages_count = 1 12 | logger_enabled = false 13 | -------------------------------------------------------------------------------- /service/README.md: -------------------------------------------------------------------------------- 1 | # Aqua DHT service 2 | 3 | Rust implementation of the AquaDHT service. 4 | 5 | ## How to build .wasm 6 | * Install dependencies 7 | 8 | ```bash 9 | rustup toolchain install nightly-2021-03-24-x86_64-unknown-linux-gnu 10 | rustup default nightly-2021-03-24-x86_64-unknown-linux-gnu 11 | rustup target add wasm32-wasi 12 | cargo install +nightly marine 13 | ``` 14 | 15 | * Compile compile .wasm and generate aqua file 16 | 17 | ```bash 18 | ./build.sh 19 | ``` 20 | 21 | ## How to run tests 22 | ```bash 23 | cargo test --release -- --test-threads=1 24 | ``` 25 | -------------------------------------------------------------------------------- /service/build.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | use marine_rs_sdk_test::generate_marine_test_env; 17 | use marine_rs_sdk_test::ServiceDescription; 18 | fn main() { 19 | let services = vec![( 20 | "registry".to_string(), 21 | ServiceDescription { 22 | config_path: "Config.toml".to_string(), 23 | modules_dir: None, 24 | }, 25 | )]; 26 | 27 | let target = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap(); 28 | if target != "wasm32" { 29 | generate_marine_test_env(services, "marine_test_env.rs", file!()); 30 | } 31 | 32 | println!("cargo:rerun-if-changed=src/key_api.rs"); 33 | println!("cargo:rerun-if-changed=src/record_api.rs"); 34 | println!("cargo:rerun-if-changed=src/tombstone_api.rs"); 35 | println!("cargo:rerun-if-changed=src/main.rs"); 36 | } 37 | -------------------------------------------------------------------------------- /service/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit -o nounset -o pipefail 3 | 4 | # set current working directory to script directory to run script from everywhere 5 | cd "$(dirname "$0")" 6 | 7 | # build registry.wasm 8 | marine build --release 9 | 10 | # copy .wasm to artifacts 11 | rm -f artifacts/* 12 | mkdir -p artifacts 13 | cp target/wasm32-wasi/release/registry.wasm artifacts/ 14 | 15 | # download SQLite 3 to use in tests 16 | curl -L https://github.com/fluencelabs/sqlite/releases/download/sqlite-wasm-v0.18.2/sqlite3.wasm -o artifacts/sqlite3.wasm 17 | 18 | # generate Aqua bindings 19 | marine aqua artifacts/registry.wasm -s Registry -i registry >../aqua/registry-service.aqua 20 | 21 | mkdir -p ../distro/registry-service 22 | cp artifacts/registry.wasm artifacts/sqlite3.wasm ../distro/registry-service 23 | -------------------------------------------------------------------------------- /service/clippy.toml: -------------------------------------------------------------------------------- 1 | too-many-arguments-threshold = 9 -------------------------------------------------------------------------------- /service/src/config.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use serde::{Deserialize, Serialize}; 18 | use std::fs; 19 | 20 | use crate::defaults::{CONFIG_FILE, DEFAULT_EXPIRED_AGE, DEFAULT_STALE_AGE}; 21 | 22 | #[derive(Deserialize, Serialize)] 23 | pub struct Config { 24 | pub expired_timeout: u64, 25 | pub stale_timeout: u64, 26 | } 27 | 28 | pub fn write_config(config: Config) { 29 | fs::write(CONFIG_FILE, toml::to_string(&config).unwrap()).unwrap(); 30 | } 31 | 32 | pub fn load_config() -> Config { 33 | let file_content = fs::read_to_string(CONFIG_FILE).unwrap(); 34 | let config: Config = toml::from_str(&file_content).unwrap(); 35 | config 36 | } 37 | 38 | pub fn create_config() { 39 | if fs::metadata(CONFIG_FILE).is_err() { 40 | write_config(Config { 41 | expired_timeout: DEFAULT_EXPIRED_AGE, 42 | stale_timeout: DEFAULT_STALE_AGE, 43 | }); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /service/src/defaults.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | pub static KEYS_TABLE_NAME: &str = "keys_table"; 18 | pub static RECORDS_TABLE_NAME: &str = "records_table"; 19 | pub static CONFIG_FILE: &str = "/tmp/Config.toml"; 20 | pub static DB_PATH: &str = "/tmp/registry.db"; 21 | pub static DEFAULT_STALE_AGE: u64 = 60 * 60; 22 | pub static DEFAULT_EXPIRED_AGE: u64 = 24 * 60 * 60; 23 | pub static RECORDS_LIMIT: usize = 32; 24 | 25 | pub static TRUSTED_TIMESTAMP_SERVICE_ID: &str = "peer"; 26 | pub static TRUSTED_TIMESTAMP_FUNCTION_NAME: &str = "timestamp_sec"; 27 | pub static TRUSTED_WEIGHT_SERVICE_ID: &str = "trust-graph"; 28 | pub static TRUSTED_WEIGHT_FUNCTION_NAME: &str = "get_weight"; 29 | -------------------------------------------------------------------------------- /service/src/error.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | use fluence_keypair::error::DecodingError; 17 | use marine_sqlite_connector::Error as SqliteError; 18 | use thiserror::Error as ThisError; 19 | 20 | #[derive(ThisError, Debug)] 21 | pub enum ServiceError { 22 | #[error("Internal Sqlite error: {0}")] 23 | SqliteError( 24 | #[from] 25 | #[source] 26 | SqliteError, 27 | ), 28 | #[error("Requested key {0} does not exist")] 29 | KeyNotExists(String), 30 | #[error("Key {0} for {1} peer_id already exists with newer timestamp")] 31 | KeyAlreadyExistsNewerTimestamp(String, String), 32 | #[error("Values limit for key_d {0} is exceeded")] 33 | ValuesLimitExceeded(String), 34 | #[error("Host value for key_id {0} not found ")] 35 | HostValueNotFound(String), 36 | #[error("Invalid set_host_value result: success is false or value is missing")] 37 | InvalidSetHostValueResult, 38 | #[error("Internal error: {0}")] 39 | InternalError(String), 40 | #[error( 41 | "Invalid timestamp tetraplet: you should use host peer.timestamp_sec to pass timestamp: {0}" 42 | )] 43 | InvalidTimestampTetraplet(String), 44 | #[error( 45 | "Invalid set_host_value tetraplet: you should use put_host_value to pass set_host_value: {0}" 46 | )] 47 | InvalidSetHostValueTetraplet(String), 48 | #[error( 49 | "Invalid weight tetraplet: you should use host trust-graph.get_weight to pass weight: {0}" 50 | )] 51 | InvalidWeightTetraplet(String), 52 | #[error("Invalid weight peer_id: expected {0}, found {1}")] 53 | InvalidWeightPeerId(String, String), 54 | #[error("Invalid signature for key_id {0}; label {1} and peer_id {2}: {3}")] 55 | InvalidKeySignature( 56 | String, 57 | String, 58 | String, 59 | #[source] fluence_keypair::error::VerificationError, 60 | ), 61 | #[error("Invalid record metadata signature for key_id {0} and issued by {1}: {2}")] 62 | InvalidRecordMetadataSignature( 63 | String, 64 | String, 65 | #[source] fluence_keypair::error::VerificationError, 66 | ), 67 | #[error("Invalid record signature for key_id {0} and issued by {1}: {2}")] 68 | InvalidRecordSignature( 69 | String, 70 | String, 71 | #[source] fluence_keypair::error::VerificationError, 72 | ), 73 | #[error("Invalid tombstone signature for key_id {0} and issued by {1}: {2}")] 74 | InvalidTombstoneSignature( 75 | String, 76 | String, 77 | #[source] fluence_keypair::error::VerificationError, 78 | ), 79 | #[error("Key can't be registered in the future")] 80 | InvalidKeyTimestamp, 81 | #[error("Record metadata can't be issued in the future")] 82 | InvalidRecordMetadataTimestamp, 83 | #[error("Record can't be registered in the future")] 84 | InvalidRecordTimestamp, 85 | #[error("Record is already expired")] 86 | RecordAlreadyExpired, 87 | #[error("Tombstone can't be issued in the future")] 88 | InvalidTombstoneTimestamp, 89 | #[error("Records to publish should belong to one key id")] 90 | RecordsPublishingError, 91 | #[error("Tombstones to publish should belong to one key id")] 92 | TombstonesPublishingError, 93 | #[error("peer id parse error: {0}")] 94 | PeerIdParseError(String), 95 | #[error("public key extraction from peer id failed: {0}")] 96 | PublicKeyExtractionError(String), 97 | #[error("{0}")] 98 | PublicKeyDecodeError( 99 | #[from] 100 | #[source] 101 | DecodingError, 102 | ), 103 | #[error("Weight for record with peer_id {0} and set_by {1} is missing ")] 104 | MissingRecordWeight(String, String), 105 | #[error("merge_keys: keys argument is empty")] 106 | KeysArgumentEmpty, 107 | #[error( 108 | "Newer record or tombstone for key_id: {0}, issued_by: {1}, peer_id: {2} already exists" 109 | )] 110 | NewerRecordOrTombstoneExists(String, String, String), 111 | } 112 | -------------------------------------------------------------------------------- /service/src/key.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use crate::error::ServiceError; 18 | use crate::misc::extract_public_key; 19 | use fluence_keypair::Signature; 20 | use marine_rs_sdk::marine; 21 | use sha2::{Digest, Sha256}; 22 | 23 | #[marine] 24 | #[derive(Default, Clone)] 25 | pub struct Key { 26 | /// base58-encoded sha256(concat(label, owner_peer_id)) 27 | pub id: String, 28 | /// any unique string defined by the owner 29 | pub label: String, 30 | /// peer id in base58 31 | pub owner_peer_id: String, 32 | /// timestamp of creation in seconds 33 | pub timestamp_created: u64, 34 | /// challenge in bytes, will be used for permissions 35 | pub challenge: Vec, 36 | /// challenge type, will be used for permissions 37 | pub challenge_type: String, 38 | /// encoded and hashed previous fields signed by `owner_peer_id` 39 | pub signature: Vec, 40 | } 41 | 42 | #[derive(Default, Clone)] 43 | pub struct KeyInternal { 44 | pub key: Key, 45 | /// timestamp of last publishing in seconds 46 | pub timestamp_published: u64, 47 | /// weight of key.owner_peer_id in local TrustGraph 48 | pub weight: u32, 49 | } 50 | 51 | impl Key { 52 | pub fn new( 53 | label: String, 54 | owner_peer_id: String, 55 | timestamp_created: u64, 56 | challenge: Vec, 57 | challenge_type: String, 58 | signature: Vec, 59 | ) -> Self { 60 | let id = Self::get_id(&label, &owner_peer_id); 61 | 62 | Self { 63 | id, 64 | label, 65 | owner_peer_id, 66 | timestamp_created, 67 | challenge, 68 | challenge_type, 69 | signature, 70 | } 71 | } 72 | 73 | pub fn get_id(label: &str, owner_peer_id: &str) -> String { 74 | let mut hasher = Sha256::new(); 75 | hasher.update(format!("{}{}", label, owner_peer_id).as_bytes()); 76 | bs58::encode(hasher.finalize()).into_string() 77 | } 78 | 79 | pub fn signature_bytes(&self) -> Vec { 80 | let mut bytes = Vec::new(); 81 | 82 | bytes.push(self.label.len() as u8); 83 | bytes.extend(self.label.as_bytes()); 84 | 85 | bytes.push(self.owner_peer_id.len() as u8); 86 | bytes.extend(self.owner_peer_id.as_bytes()); 87 | 88 | bytes.extend(self.timestamp_created.to_le_bytes()); 89 | 90 | bytes.push(self.challenge.len() as u8); 91 | bytes.extend(&self.challenge); 92 | 93 | bytes.push(self.challenge_type.len() as u8); 94 | bytes.extend(self.challenge_type.as_bytes()); 95 | 96 | let mut hasher = Sha256::new(); 97 | hasher.update(bytes); 98 | hasher.finalize().to_vec() 99 | } 100 | 101 | pub fn verify(&self, current_timestamp_sec: u64) -> Result<(), ServiceError> { 102 | if self.timestamp_created > current_timestamp_sec { 103 | return Err(ServiceError::InvalidKeyTimestamp); 104 | } 105 | 106 | self.verify_signature() 107 | } 108 | 109 | pub fn verify_signature(&self) -> Result<(), ServiceError> { 110 | let pk = extract_public_key(self.owner_peer_id.clone())?; 111 | let bytes = self.signature_bytes(); 112 | let signature = Signature::from_bytes(pk.get_key_format(), self.signature.clone()); 113 | pk.verify(&bytes, &signature).map_err(|e| { 114 | ServiceError::InvalidKeySignature( 115 | self.id.clone(), 116 | self.label.clone(), 117 | self.owner_peer_id.clone(), 118 | e, 119 | ) 120 | }) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /service/src/key_api.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | use crate::error::ServiceError; 17 | use crate::key::{Key, KeyInternal}; 18 | use crate::misc::check_weight_result; 19 | use crate::results::{GetKeyMetadataResult, MergeKeysResult, RegisterKeyResult, RegistryResult}; 20 | use crate::storage_impl::get_storage; 21 | use crate::tetraplets_checkers::{check_timestamp_tetraplets, check_weight_tetraplets}; 22 | use crate::{wrapped_try, WeightResult}; 23 | use marine_rs_sdk::marine; 24 | 25 | #[marine] 26 | pub fn get_key_bytes( 27 | label: String, 28 | mut owner_peer_id: Vec, 29 | timestamp_created: u64, 30 | challenge: Vec, 31 | challenge_type: String, 32 | ) -> Vec { 33 | Key { 34 | label, 35 | owner_peer_id: owner_peer_id 36 | .pop() 37 | .unwrap_or(marine_rs_sdk::get_call_parameters().init_peer_id), 38 | timestamp_created, 39 | challenge, 40 | challenge_type, 41 | ..Default::default() 42 | } 43 | .signature_bytes() 44 | } 45 | 46 | #[marine] 47 | pub fn get_key_id(label: String, peer_id: String) -> String { 48 | Key::get_id(&label, &peer_id) 49 | } 50 | 51 | /// register new key if not exists with caller peer_id, update if exists with same peer_id or return error 52 | #[marine] 53 | pub fn register_key( 54 | label: String, 55 | owner_peer_id: Vec, 56 | timestamp_created: u64, 57 | challenge: Vec, 58 | challenge_type: String, 59 | signature: Vec, 60 | weight: WeightResult, 61 | current_timestamp_sec: u64, 62 | ) -> RegisterKeyResult { 63 | wrapped_try(|| { 64 | let call_parameters = marine_rs_sdk::get_call_parameters(); 65 | check_weight_tetraplets(&call_parameters, 6, 0)?; 66 | check_timestamp_tetraplets(&call_parameters, 7)?; 67 | let owner_peer_id = owner_peer_id 68 | .get(0) 69 | .unwrap_or(&call_parameters.init_peer_id) 70 | .clone(); 71 | check_weight_result(&owner_peer_id, &weight)?; 72 | let key = Key::new( 73 | label, 74 | owner_peer_id, 75 | timestamp_created, 76 | challenge, 77 | challenge_type, 78 | signature, 79 | ); 80 | key.verify(current_timestamp_sec)?; 81 | 82 | let key_id = key.id.clone(); 83 | let weight = weight.weight; 84 | let storage = get_storage()?; 85 | storage.update_key(KeyInternal { 86 | key, 87 | timestamp_published: 0, 88 | weight, 89 | })?; 90 | 91 | Ok(key_id) 92 | }) 93 | .into() 94 | } 95 | 96 | #[marine] 97 | pub fn get_key_metadata(key_id: String) -> GetKeyMetadataResult { 98 | wrapped_try(|| get_storage()?.get_key(key_id)).into() 99 | } 100 | 101 | /// Used for replication, same as register_key, updates timestamp_accessed 102 | #[marine] 103 | pub fn republish_key( 104 | mut key: Key, 105 | weight: WeightResult, 106 | current_timestamp_sec: u64, 107 | ) -> RegistryResult { 108 | wrapped_try(|| { 109 | let call_parameters = marine_rs_sdk::get_call_parameters(); 110 | check_weight_tetraplets(&call_parameters, 1, 0)?; 111 | check_weight_result(&key.owner_peer_id, &weight)?; 112 | check_timestamp_tetraplets(&call_parameters, 2)?; 113 | key.verify(current_timestamp_sec)?; 114 | 115 | // just to be sure 116 | key.id = Key::get_id(&key.label, &key.owner_peer_id); 117 | 118 | let storage = get_storage()?; 119 | match storage.update_key(KeyInternal { 120 | key, 121 | timestamp_published: current_timestamp_sec, 122 | weight: weight.weight, 123 | }) { 124 | // we should ignore this error for republish 125 | Err(ServiceError::KeyAlreadyExistsNewerTimestamp(_, _)) => Ok(()), 126 | other => other, 127 | } 128 | }) 129 | .into() 130 | } 131 | 132 | /// merge key and return the latest 133 | #[marine] 134 | pub fn merge_keys(keys: Vec) -> MergeKeysResult { 135 | keys.into_iter() 136 | .max_by(|l, r| l.timestamp_created.cmp(&r.timestamp_created)) 137 | .ok_or(ServiceError::KeysArgumentEmpty) 138 | .into() 139 | } 140 | -------------------------------------------------------------------------------- /service/src/key_storage_impl.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use crate::defaults::KEYS_TABLE_NAME; 18 | 19 | use crate::error::ServiceError; 20 | use crate::error::ServiceError::{InternalError, KeyNotExists}; 21 | use crate::key::{Key, KeyInternal}; 22 | use crate::storage_impl::Storage; 23 | use marine_sqlite_connector::{State, Statement, Value}; 24 | 25 | impl Storage { 26 | pub fn create_key_tables(&self) { 27 | let table_schema = f!(" 28 | CREATE TABLE IF NOT EXISTS {KEYS_TABLE_NAME} ( 29 | key_id TEXT PRIMARY KEY, 30 | label TEXT, 31 | owner_peer_id TEXT, 32 | timestamp_created INTEGER, 33 | challenge BLOB, 34 | challenge_type TEXT, 35 | signature BLOB NOT NULL, 36 | timestamp_published INTEGER, 37 | weight INTEGER 38 | ); 39 | "); 40 | let current_table_schema = self 41 | .get_table_schema(KEYS_TABLE_NAME.to_string()) 42 | .expect(f!("failed to get {KEYS_TABLE_NAME} table schema").as_str()); 43 | if !current_table_schema.is_empty() && table_schema != current_table_schema { 44 | self.delete_table(KEYS_TABLE_NAME.to_string()) 45 | .expect(f!("failed to delete {KEYS_TABLE_NAME} table").as_str()) 46 | } 47 | 48 | let result = self.connection.execute(table_schema); 49 | 50 | if let Err(error) = result { 51 | println!("create_keys_table error: {}", error); 52 | } 53 | } 54 | 55 | pub fn get_key(&self, key_id: String) -> Result { 56 | let mut statement = self.connection.prepare(f!( 57 | "SELECT key_id, label, owner_peer_id, timestamp_created, challenge, challenge_type, signature \ 58 | FROM {KEYS_TABLE_NAME} WHERE key_id = ?" 59 | ))?; 60 | statement.bind(1, &Value::String(key_id.clone()))?; 61 | 62 | if let State::Row = statement.next()? { 63 | read_key(&statement) 64 | } else { 65 | Err(KeyNotExists(key_id)) 66 | } 67 | } 68 | 69 | pub fn write_key(&self, key: KeyInternal) -> Result<(), ServiceError> { 70 | let mut statement = self.connection.prepare(f!(" 71 | INSERT OR REPLACE INTO {KEYS_TABLE_NAME} VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?); 72 | "))?; 73 | 74 | statement.bind(1, &Value::String(key.key.id))?; 75 | statement.bind(2, &Value::String(key.key.label))?; 76 | statement.bind(3, &Value::String(key.key.owner_peer_id))?; 77 | statement.bind(4, &Value::Integer(key.key.timestamp_created as i64))?; 78 | statement.bind(5, &Value::Binary(key.key.challenge))?; 79 | statement.bind(6, &Value::String(key.key.challenge_type))?; 80 | statement.bind(7, &Value::Binary(key.key.signature))?; 81 | statement.bind(8, &Value::Integer(key.timestamp_published as i64))?; 82 | statement.bind(9, &Value::Integer(key.weight as i64))?; 83 | statement.next()?; 84 | Ok(()) 85 | } 86 | 87 | pub fn update_key(&self, key: KeyInternal) -> Result<(), ServiceError> { 88 | if let Ok(existing_key) = self.get_key(key.key.id.clone()) { 89 | if existing_key.timestamp_created > key.key.timestamp_created { 90 | return Err(ServiceError::KeyAlreadyExistsNewerTimestamp( 91 | key.key.label, 92 | key.key.owner_peer_id, 93 | )); 94 | } 95 | } 96 | 97 | self.write_key(key) 98 | } 99 | 100 | pub fn check_key_existence(&self, key_id: &str) -> Result<(), ServiceError> { 101 | let mut statement = self.connection.prepare(f!( 102 | "SELECT EXISTS(SELECT 1 FROM {KEYS_TABLE_NAME} WHERE key_id = ? LIMIT 1)" 103 | ))?; 104 | statement.bind(1, &Value::String(key_id.to_string()))?; 105 | 106 | if let State::Row = statement.next()? { 107 | let exists = statement.read::(0)?; 108 | if exists == 1 { 109 | Ok(()) 110 | } else { 111 | Err(KeyNotExists(key_id.to_string())) 112 | } 113 | } else { 114 | Err(InternalError( 115 | "EXISTS should always return something".to_string(), 116 | )) 117 | } 118 | } 119 | 120 | pub fn get_stale_keys(&self, stale_timestamp: u64) -> Result, ServiceError> { 121 | let mut statement = self.connection.prepare(f!( 122 | "SELECT key_id, label, owner_peer_id, timestamp_created, challenge, challenge_type, signature, timestamp_published, weight \ 123 | FROM {KEYS_TABLE_NAME} WHERE timestamp_published <= ?" 124 | ))?; 125 | statement.bind(1, &Value::Integer(stale_timestamp as i64))?; 126 | 127 | let mut stale_keys: Vec = vec![]; 128 | while let State::Row = statement.next()? { 129 | stale_keys.push(read_internal_key(&statement)?); 130 | } 131 | 132 | Ok(stale_keys) 133 | } 134 | 135 | pub fn delete_key(&self, key_id: String) -> Result<(), ServiceError> { 136 | let mut statement = self 137 | .connection 138 | .prepare(f!("DELETE FROM {KEYS_TABLE_NAME} WHERE key_id=?"))?; 139 | statement.bind(1, &Value::String(key_id.clone()))?; 140 | statement.next().map(drop)?; 141 | 142 | if self.connection.changes() == 1 { 143 | Ok(()) 144 | } else { 145 | Err(KeyNotExists(key_id)) 146 | } 147 | } 148 | 149 | pub fn clear_expired_keys(&self, expired_timestamp: u64) -> Result { 150 | let mut statement = self.connection.prepare(f!( 151 | "SELECT key_id FROM {KEYS_TABLE_NAME} WHERE timestamp_created <= ?" 152 | ))?; 153 | statement.bind(1, &Value::Integer(expired_timestamp as i64))?; 154 | 155 | let mut expired_keys: Vec = vec![]; 156 | while let State::Row = statement.next()? { 157 | let key_id = statement.read::(0)?; 158 | if self.get_records_count_by_key(&key_id)? == 0 { 159 | expired_keys.push(key_id); 160 | } 161 | } 162 | 163 | let removed_keys = expired_keys.len(); 164 | for id in expired_keys.into_iter() { 165 | self.delete_key(id)?; 166 | } 167 | 168 | Ok(removed_keys as u64) 169 | } 170 | } 171 | 172 | pub fn read_key(statement: &Statement) -> Result { 173 | Ok(Key { 174 | id: statement.read::(0)?, 175 | label: statement.read::(1)?, 176 | owner_peer_id: statement.read::(2)?, 177 | timestamp_created: statement.read::(3)? as u64, 178 | challenge: statement.read::>(4)?, 179 | challenge_type: statement.read::(5)?, 180 | signature: statement.read::>(6)?, 181 | }) 182 | } 183 | 184 | pub fn read_internal_key(statement: &Statement) -> Result { 185 | Ok(KeyInternal { 186 | key: read_key(statement)?, 187 | timestamp_published: statement.read::(7)? as u64, 188 | weight: statement.read::(8)? as u32, 189 | }) 190 | } 191 | -------------------------------------------------------------------------------- /service/src/main.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | #![allow(dead_code)] 17 | #![allow(clippy::result_large_err)] 18 | use marine_rs_sdk::marine; 19 | use marine_rs_sdk::module_manifest; 20 | 21 | use crate::config::{create_config, load_config, write_config}; 22 | use crate::results::{ClearExpiredResult, EvictStaleResult}; 23 | use crate::storage_impl::get_storage; 24 | use crate::tetraplets_checkers::check_timestamp_tetraplets; 25 | 26 | mod config; 27 | mod defaults; 28 | mod error; 29 | mod key; 30 | mod key_api; 31 | mod key_storage_impl; 32 | mod misc; 33 | mod record; 34 | mod record_api; 35 | mod record_storage_impl; 36 | mod results; 37 | mod storage_impl; 38 | mod tests; 39 | mod tetraplets_checkers; 40 | mod tombstone; 41 | mod tombstone_api; 42 | mod tombstone_storage_impl; 43 | 44 | #[macro_use] 45 | extern crate fstrings; 46 | 47 | module_manifest!(); 48 | 49 | pub fn wrapped_try(func: F) -> T 50 | where 51 | F: FnOnce() -> T, 52 | { 53 | func() 54 | } 55 | 56 | // TODO: ship tg results as crate, remove duplication 57 | #[marine] 58 | pub struct WeightResult { 59 | pub success: bool, 60 | pub weight: u32, 61 | pub peer_id: String, 62 | pub error: String, 63 | } 64 | 65 | fn main() { 66 | let storage = get_storage().unwrap(); 67 | storage.create_key_tables(); 68 | storage.create_records_table(); 69 | create_config(); 70 | } 71 | 72 | #[marine] 73 | pub fn clear_expired(current_timestamp_sec: u64) -> ClearExpiredResult { 74 | wrapped_try(|| { 75 | let call_parameters = marine_rs_sdk::get_call_parameters(); 76 | check_timestamp_tetraplets(&call_parameters, 0)?; 77 | get_storage()?.clear_expired(current_timestamp_sec) 78 | }) 79 | .into() 80 | } 81 | 82 | #[marine] 83 | pub fn evict_stale(current_timestamp_sec: u64) -> EvictStaleResult { 84 | wrapped_try(|| { 85 | let call_parameters = marine_rs_sdk::get_call_parameters(); 86 | check_timestamp_tetraplets(&call_parameters, 0)?; 87 | get_storage()?.evict_stale(current_timestamp_sec) 88 | }) 89 | .into() 90 | } 91 | 92 | #[marine] 93 | pub fn set_expired_timeout(timeout_sec: u64) { 94 | let mut config = load_config(); 95 | config.expired_timeout = timeout_sec; 96 | write_config(config); 97 | } 98 | 99 | #[marine] 100 | pub fn set_stale_timeout(timeout_sec: u64) { 101 | let mut config = load_config(); 102 | config.stale_timeout = timeout_sec; 103 | write_config(config); 104 | } 105 | -------------------------------------------------------------------------------- /service/src/misc.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use crate::error::ServiceError; 18 | use crate::WeightResult; 19 | use boolinator::Boolinator; 20 | use fluence_keypair::PublicKey; 21 | use libp2p_identity::PeerId; 22 | use std::convert::TryFrom; 23 | use std::str::FromStr; 24 | 25 | fn parse_peer_id(peer_id: String) -> Result { 26 | PeerId::from_str(&peer_id).map_err(|e| ServiceError::PeerIdParseError(format!("{:?}", e))) 27 | } 28 | 29 | pub fn extract_public_key(peer_id: String) -> Result { 30 | PublicKey::try_from( 31 | parse_peer_id(peer_id) 32 | .map_err(|e| ServiceError::PublicKeyExtractionError(e.to_string()))?, 33 | ) 34 | .map_err(ServiceError::PublicKeyDecodeError) 35 | } 36 | 37 | pub fn check_weight_result(peer_id: &str, weight: &WeightResult) -> Result<(), ServiceError> { 38 | (weight.success && weight.peer_id.eq(peer_id)).as_result( 39 | (), 40 | ServiceError::InvalidWeightPeerId(peer_id.to_string(), weight.peer_id.clone()), 41 | ) 42 | } 43 | -------------------------------------------------------------------------------- /service/src/record.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use crate::misc::extract_public_key; 18 | use crate::{defaults::DEFAULT_EXPIRED_AGE, error::ServiceError}; 19 | use fluence_keypair::Signature; 20 | use marine_rs_sdk::marine; 21 | use sha2::{Digest, Sha256}; 22 | 23 | #[marine] 24 | #[derive(Debug, Default, Clone)] 25 | pub struct RecordMetadata { 26 | /// base58-encoded key id 27 | pub key_id: String, 28 | /// peer id of the issuer in base58 29 | pub issued_by: String, 30 | /// peer_id of hoster 31 | pub peer_id: String, 32 | /// timestamp in seconds 33 | pub timestamp_issued: u64, 34 | /// will be used for permissions 35 | pub solution: Vec, 36 | /// any string 37 | pub value: String, 38 | /// optional (length is 0 or 1), base58 relay id 39 | pub relay_id: Vec, 40 | /// optional (length is 0 or 1), advertising service id 41 | pub service_id: Vec, 42 | /// encoded and hashed previous fields signed by `issued_by` 43 | pub issuer_signature: Vec, 44 | } 45 | 46 | impl RecordMetadata { 47 | pub fn signature_bytes(&self) -> Vec { 48 | let mut bytes = Vec::new(); 49 | bytes.push(self.key_id.len() as u8); 50 | bytes.extend(self.key_id.as_bytes()); 51 | 52 | bytes.push(self.issued_by.len() as u8); 53 | bytes.extend(self.issued_by.as_bytes()); 54 | 55 | bytes.push(self.peer_id.len() as u8); 56 | bytes.extend(self.peer_id.as_bytes()); 57 | 58 | bytes.extend(self.timestamp_issued.to_le_bytes()); 59 | 60 | bytes.push(self.solution.len() as u8); 61 | bytes.extend(&self.solution); 62 | 63 | bytes.push(self.value.len() as u8); 64 | bytes.extend(self.value.as_bytes()); 65 | 66 | bytes.extend(self.relay_id.len().to_le_bytes()); 67 | for id in &self.relay_id { 68 | bytes.push(id.len() as u8); 69 | bytes.extend(id.as_bytes()); 70 | } 71 | 72 | bytes.extend(self.service_id.len().to_le_bytes()); 73 | for id in &self.service_id { 74 | bytes.push(id.len() as u8); 75 | bytes.extend(id.as_bytes()); 76 | } 77 | 78 | let mut hasher = Sha256::new(); 79 | hasher.update(bytes); 80 | hasher.finalize().to_vec() 81 | } 82 | 83 | pub fn verify(&self, current_timestamp_sec: u64) -> Result<(), ServiceError> { 84 | if self.timestamp_issued > current_timestamp_sec { 85 | return Err(ServiceError::InvalidRecordMetadataTimestamp); 86 | } 87 | 88 | let pk = extract_public_key(self.issued_by.clone())?; 89 | let bytes = self.signature_bytes(); 90 | let signature = Signature::from_bytes(pk.get_key_format(), self.issuer_signature.clone()); 91 | pk.verify(&bytes, &signature).map_err(|e| { 92 | ServiceError::InvalidRecordMetadataSignature( 93 | self.key_id.clone(), 94 | self.issued_by.clone(), 95 | e, 96 | ) 97 | }) 98 | } 99 | } 100 | 101 | #[marine] 102 | #[derive(Debug, Default, Clone)] 103 | pub struct Record { 104 | /// record metadata 105 | pub metadata: RecordMetadata, 106 | /// timestamp in seconds 107 | pub timestamp_created: u64, 108 | /// encoded and hashed previous fields signed by `metadata.peer_id` 109 | pub signature: Vec, 110 | } 111 | 112 | #[derive(Default, Debug, Clone)] 113 | pub struct RecordInternal { 114 | pub record: Record, 115 | pub weight: u32, 116 | } 117 | 118 | impl Record { 119 | pub fn signature_bytes(&self) -> Vec { 120 | let mut bytes = Vec::new(); 121 | let mut metadata = self.metadata.signature_bytes(); 122 | metadata.push(self.metadata.issuer_signature.len() as u8); 123 | metadata.extend(&self.metadata.issuer_signature); 124 | 125 | bytes.push(metadata.len() as u8); 126 | bytes.append(&mut metadata); 127 | 128 | bytes.extend(self.timestamp_created.to_le_bytes()); 129 | let mut hasher = Sha256::new(); 130 | hasher.update(metadata); 131 | hasher.finalize().to_vec() 132 | } 133 | 134 | pub fn verify(&self, current_timestamp_sec: u64) -> Result<(), ServiceError> { 135 | if self.timestamp_created > current_timestamp_sec { 136 | return Err(ServiceError::InvalidRecordTimestamp); 137 | } 138 | 139 | if self.is_expired(current_timestamp_sec) { 140 | return Err(ServiceError::RecordAlreadyExpired); 141 | } 142 | 143 | self.metadata.verify(current_timestamp_sec)?; 144 | 145 | let pk = extract_public_key(self.metadata.peer_id.clone())?; 146 | let bytes = self.signature_bytes(); 147 | let signature = Signature::from_bytes(pk.get_key_format(), self.signature.clone()); 148 | pk.verify(&bytes, &signature).map_err(|e| { 149 | ServiceError::InvalidRecordSignature( 150 | self.metadata.key_id.clone(), 151 | self.metadata.peer_id.clone(), 152 | e, 153 | ) 154 | }) 155 | } 156 | 157 | pub fn is_expired(&self, current_timestamp_sec: u64) -> bool { 158 | (current_timestamp_sec - self.timestamp_created) > DEFAULT_EXPIRED_AGE 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /service/src/record_api.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | use crate::error::ServiceError; 17 | use crate::error::ServiceError::MissingRecordWeight; 18 | use crate::misc::check_weight_result; 19 | use crate::record::{Record, RecordInternal, RecordMetadata}; 20 | use crate::record_storage_impl::merge_records; 21 | use crate::results::{GetRecordsResult, MergeResult, RegistryResult, RepublishRecordsResult}; 22 | use crate::storage_impl::get_storage; 23 | use crate::tetraplets_checkers::{check_timestamp_tetraplets, check_weight_tetraplets}; 24 | use crate::{load_config, wrapped_try, WeightResult}; 25 | use marine_rs_sdk::marine; 26 | 27 | #[marine] 28 | pub fn get_record_metadata_bytes( 29 | key_id: String, 30 | issued_by: String, 31 | timestamp_issued: u64, 32 | value: String, 33 | peer_id: String, 34 | relay_id: Vec, 35 | service_id: Vec, 36 | solution: Vec, 37 | ) -> Vec { 38 | RecordMetadata { 39 | key_id, 40 | issued_by, 41 | timestamp_issued, 42 | value, 43 | peer_id, 44 | relay_id, 45 | service_id, 46 | solution, 47 | ..Default::default() 48 | } 49 | .signature_bytes() 50 | } 51 | 52 | #[marine] 53 | pub fn create_record_metadata( 54 | key_id: String, 55 | issued_by: String, 56 | timestamp_issued: u64, 57 | value: String, 58 | peer_id: String, 59 | relay_id: Vec, 60 | service_id: Vec, 61 | solution: Vec, 62 | signature: Vec, 63 | ) -> RecordMetadata { 64 | RecordMetadata { 65 | key_id, 66 | issued_by, 67 | timestamp_issued, 68 | value, 69 | peer_id, 70 | relay_id, 71 | service_id, 72 | solution, 73 | issuer_signature: signature, 74 | } 75 | } 76 | 77 | #[marine] 78 | pub fn get_record_bytes(metadata: RecordMetadata, timestamp_created: u64) -> Vec { 79 | Record { 80 | metadata, 81 | timestamp_created, 82 | ..Default::default() 83 | } 84 | .signature_bytes() 85 | } 86 | 87 | #[marine] 88 | pub fn put_record( 89 | metadata: RecordMetadata, 90 | timestamp_created: u64, 91 | signature: Vec, 92 | weight: WeightResult, 93 | current_timestamp_sec: u64, 94 | ) -> RegistryResult { 95 | wrapped_try(|| { 96 | let cp = marine_rs_sdk::get_call_parameters(); 97 | check_weight_tetraplets(&cp, 3, 0)?; 98 | check_timestamp_tetraplets(&cp, 4)?; 99 | check_weight_result(&cp.init_peer_id, &weight)?; 100 | let record = Record { 101 | metadata, 102 | timestamp_created, 103 | signature, 104 | }; 105 | record.verify(current_timestamp_sec)?; 106 | 107 | let storage = get_storage()?; 108 | storage.check_key_existence(&record.metadata.key_id)?; 109 | storage.update_record(RecordInternal { 110 | record, 111 | weight: weight.weight, 112 | }) 113 | }) 114 | .into() 115 | } 116 | 117 | /// Return all values by key 118 | #[marine] 119 | pub fn get_records(key_id: String, current_timestamp_sec: u64) -> GetRecordsResult { 120 | wrapped_try(|| { 121 | let call_parameters = marine_rs_sdk::get_call_parameters(); 122 | check_timestamp_tetraplets(&call_parameters, 1)?; 123 | let storage = get_storage()?; 124 | storage.check_key_existence(&key_id)?; 125 | storage 126 | .get_records(key_id, current_timestamp_sec) 127 | .map(|records| records.into_iter().map(|r| r.record).collect()) 128 | }) 129 | .into() 130 | } 131 | 132 | /// Return all values by key 133 | #[marine] 134 | pub fn get_stale_local_records(current_timestamp_sec: u64) -> GetRecordsResult { 135 | wrapped_try(|| { 136 | let call_parameters = marine_rs_sdk::get_call_parameters(); 137 | check_timestamp_tetraplets(&call_parameters, 0)?; 138 | let storage = get_storage()?; 139 | 140 | // TODO: add some meaningful constant for expiring local records 141 | let stale_timestamp_sec = current_timestamp_sec - load_config().expired_timeout + 100; 142 | storage 143 | .get_local_stale_records(stale_timestamp_sec) 144 | .map(|records| records.into_iter().map(|r| r.record).collect()) 145 | }) 146 | .into() 147 | } 148 | 149 | /// If the key exists, then merge new records with existing (last-write-wins) and put 150 | #[marine] 151 | pub fn republish_records( 152 | records: Vec, 153 | weights: Vec, 154 | current_timestamp_sec: u64, 155 | ) -> RepublishRecordsResult { 156 | wrapped_try(|| { 157 | if records.is_empty() { 158 | return Ok(0); 159 | } 160 | 161 | let key_id = records[0].metadata.key_id.clone(); 162 | let call_parameters = marine_rs_sdk::get_call_parameters(); 163 | check_timestamp_tetraplets(&call_parameters, 2)?; 164 | let mut records_to_merge = vec![]; 165 | 166 | for (i, record) in records.into_iter().enumerate() { 167 | record.verify(current_timestamp_sec)?; 168 | check_weight_tetraplets(&call_parameters, 1, i)?; 169 | let weight_result = weights.get(i).ok_or_else(|| { 170 | MissingRecordWeight( 171 | record.metadata.peer_id.clone(), 172 | record.metadata.issued_by.clone(), 173 | ) 174 | })?; 175 | check_weight_result(&record.metadata.issued_by, weight_result)?; 176 | if record.metadata.key_id != key_id { 177 | return Err(ServiceError::RecordsPublishingError); 178 | } 179 | 180 | records_to_merge.push(RecordInternal { 181 | record, 182 | weight: weight_result.weight, 183 | }); 184 | } 185 | 186 | let storage = get_storage()?; 187 | storage.check_key_existence(&key_id)?; 188 | storage.merge_and_update_records(key_id, records_to_merge, current_timestamp_sec) 189 | }) 190 | .into() 191 | } 192 | 193 | #[marine] 194 | pub fn merge_two(a: Vec, b: Vec) -> MergeResult { 195 | merge_records( 196 | a.into_iter() 197 | .chain(b) 198 | .map(|record| RecordInternal { 199 | record, 200 | ..Default::default() 201 | }) 202 | .collect(), 203 | ) 204 | .map(|recs| recs.into_iter().map(|r| r.record).collect()) 205 | .into() 206 | } 207 | 208 | #[marine] 209 | pub fn merge(records: Vec>) -> MergeResult { 210 | merge_records( 211 | records 212 | .into_iter() 213 | .flatten() 214 | .map(|record| RecordInternal { 215 | record, 216 | ..Default::default() 217 | }) 218 | .collect(), 219 | ) 220 | .map(|recs| recs.into_iter().map(|r| r.record).collect()) 221 | .into() 222 | } 223 | -------------------------------------------------------------------------------- /service/src/results.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use crate::error::ServiceError; 18 | use crate::key::Key; 19 | use crate::record::Record; 20 | use crate::tombstone::Tombstone; 21 | use marine_rs_sdk::marine; 22 | 23 | #[marine] 24 | #[derive(Debug)] 25 | pub struct RegistryResult { 26 | pub success: bool, 27 | pub error: String, 28 | } 29 | 30 | impl From> for RegistryResult { 31 | fn from(result: Result<(), ServiceError>) -> Self { 32 | match result { 33 | Ok(_) => Self { 34 | success: true, 35 | error: "".to_string(), 36 | }, 37 | Err(err) => Self { 38 | success: false, 39 | error: err.to_string(), 40 | }, 41 | } 42 | } 43 | } 44 | 45 | #[marine] 46 | #[derive(Debug)] 47 | pub struct RegisterKeyResult { 48 | pub success: bool, 49 | pub error: String, 50 | pub key_id: String, 51 | } 52 | 53 | impl From> for RegisterKeyResult { 54 | fn from(result: Result) -> Self { 55 | match result { 56 | Ok(key_id) => Self { 57 | success: true, 58 | error: "".to_string(), 59 | key_id, 60 | }, 61 | Err(err) => Self { 62 | success: false, 63 | error: err.to_string(), 64 | key_id: "".to_string(), 65 | }, 66 | } 67 | } 68 | } 69 | 70 | #[marine] 71 | #[derive(Debug)] 72 | pub struct GetRecordsResult { 73 | pub success: bool, 74 | pub error: String, 75 | pub result: Vec, 76 | } 77 | 78 | impl From, ServiceError>> for GetRecordsResult { 79 | fn from(result: Result, ServiceError>) -> Self { 80 | match result { 81 | Ok(result) => Self { 82 | success: true, 83 | error: "".to_string(), 84 | result, 85 | }, 86 | Err(err) => Self { 87 | success: false, 88 | error: err.to_string(), 89 | result: vec![], 90 | }, 91 | } 92 | } 93 | } 94 | 95 | #[marine] 96 | #[derive(Debug)] 97 | pub struct GetTombstonesResult { 98 | pub success: bool, 99 | pub error: String, 100 | pub result: Vec, 101 | } 102 | 103 | impl From, ServiceError>> for GetTombstonesResult { 104 | fn from(result: Result, ServiceError>) -> Self { 105 | match result { 106 | Ok(result) => Self { 107 | success: true, 108 | error: "".to_string(), 109 | result, 110 | }, 111 | Err(err) => Self { 112 | success: false, 113 | error: err.to_string(), 114 | result: vec![], 115 | }, 116 | } 117 | } 118 | } 119 | 120 | #[marine] 121 | #[derive(Debug)] 122 | pub struct ClearExpiredResult { 123 | pub success: bool, 124 | pub error: String, 125 | pub count_keys: u64, 126 | pub count_records: u64, 127 | pub count_tombstones: u64, 128 | } 129 | 130 | impl From> for ClearExpiredResult { 131 | fn from(result: Result<(u64, u64, u64), ServiceError>) -> Self { 132 | match result { 133 | Ok((count_keys, count_records, count_tombstones)) => Self { 134 | success: true, 135 | error: "".to_string(), 136 | count_keys, 137 | count_records, 138 | count_tombstones, 139 | }, 140 | Err(err) => Self { 141 | success: false, 142 | error: err.to_string(), 143 | count_keys: 0, 144 | count_records: 0, 145 | count_tombstones: 0, 146 | }, 147 | } 148 | } 149 | } 150 | 151 | #[marine] 152 | #[derive(Debug)] 153 | pub struct GetStaleRecordsResult { 154 | pub success: bool, 155 | pub error: String, 156 | pub result: Vec, 157 | } 158 | 159 | impl From, ServiceError>> for GetStaleRecordsResult { 160 | fn from(result: Result, ServiceError>) -> Self { 161 | match result { 162 | Ok(result) => Self { 163 | success: true, 164 | error: "".to_string(), 165 | result, 166 | }, 167 | Err(err) => Self { 168 | success: false, 169 | error: err.to_string(), 170 | result: vec![], 171 | }, 172 | } 173 | } 174 | } 175 | 176 | #[marine] 177 | pub struct GetKeyMetadataResult { 178 | pub success: bool, 179 | pub error: String, 180 | pub key: Key, 181 | } 182 | 183 | impl From> for GetKeyMetadataResult { 184 | fn from(result: Result) -> Self { 185 | match result { 186 | Ok(key) => Self { 187 | success: true, 188 | error: "".to_string(), 189 | key, 190 | }, 191 | Err(err) => Self { 192 | success: false, 193 | error: err.to_string(), 194 | key: Key::default(), 195 | }, 196 | } 197 | } 198 | } 199 | 200 | #[marine] 201 | pub struct RepublishRecordsResult { 202 | pub success: bool, 203 | pub error: String, 204 | pub updated: u64, 205 | } 206 | 207 | impl From> for RepublishRecordsResult { 208 | fn from(result: Result) -> Self { 209 | match result { 210 | Ok(count) => Self { 211 | success: true, 212 | error: "".to_string(), 213 | updated: count, 214 | }, 215 | Err(err) => Self { 216 | success: false, 217 | error: err.to_string(), 218 | updated: 0, 219 | }, 220 | } 221 | } 222 | } 223 | 224 | #[marine] 225 | pub struct EvictStaleItem { 226 | pub key: Key, 227 | pub records: Vec, 228 | pub tombstones: Vec, 229 | } 230 | 231 | #[marine] 232 | pub struct EvictStaleResult { 233 | pub success: bool, 234 | pub error: String, 235 | pub results: Vec, 236 | } 237 | 238 | impl From, ServiceError>> for EvictStaleResult { 239 | fn from(result: Result, ServiceError>) -> Self { 240 | match result { 241 | Ok(results) => Self { 242 | success: true, 243 | error: "".to_string(), 244 | results, 245 | }, 246 | Err(err) => Self { 247 | success: false, 248 | error: err.to_string(), 249 | results: vec![], 250 | }, 251 | } 252 | } 253 | } 254 | 255 | #[marine] 256 | #[derive(Debug)] 257 | pub struct MergeResult { 258 | pub success: bool, 259 | pub error: String, 260 | pub result: Vec, 261 | } 262 | 263 | impl From, ServiceError>> for MergeResult { 264 | fn from(result: Result, ServiceError>) -> Self { 265 | match result { 266 | Ok(result) => Self { 267 | success: true, 268 | error: "".to_string(), 269 | result, 270 | }, 271 | Err(err) => Self { 272 | success: false, 273 | error: err.to_string(), 274 | result: vec![], 275 | }, 276 | } 277 | } 278 | } 279 | 280 | #[marine] 281 | pub struct MergeKeysResult { 282 | pub success: bool, 283 | pub error: String, 284 | pub key: Key, 285 | } 286 | 287 | impl From> for MergeKeysResult { 288 | fn from(result: Result) -> Self { 289 | match result { 290 | Ok(key) => Self { 291 | success: true, 292 | error: "".to_string(), 293 | key, 294 | }, 295 | Err(err) => Self { 296 | success: false, 297 | error: err.to_string(), 298 | key: Key::default(), 299 | }, 300 | } 301 | } 302 | } 303 | -------------------------------------------------------------------------------- /service/src/storage_impl.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use crate::config::load_config; 18 | use crate::defaults::DB_PATH; 19 | use crate::error::ServiceError; 20 | use crate::record::Record; 21 | use crate::results::EvictStaleItem; 22 | use marine_sqlite_connector::{Connection, Result as SqliteResult, State, Value}; 23 | 24 | pub struct Storage { 25 | pub(crate) connection: Connection, 26 | } 27 | 28 | #[inline] 29 | pub(crate) fn get_storage() -> SqliteResult { 30 | marine_sqlite_connector::open(DB_PATH).map(|c| Storage { connection: c }) 31 | } 32 | 33 | pub fn get_custom_option(value: String) -> Vec { 34 | if value.is_empty() { 35 | vec![] 36 | } else { 37 | vec![value] 38 | } 39 | } 40 | 41 | pub fn from_custom_option(value: Vec) -> String { 42 | if value.is_empty() { 43 | "".to_string() 44 | } else { 45 | value[0].clone() 46 | } 47 | } 48 | 49 | impl Storage { 50 | pub fn get_table_schema(&self, table_name: String) -> Result { 51 | let mut statement = self 52 | .connection 53 | .prepare(f!("SELECT sql FROM sqlite_master WHERE name=?;"))?; 54 | 55 | statement.bind(1, &Value::String(table_name))?; 56 | 57 | if let State::Row = statement.next()? { 58 | let schema = statement.read::(0)?; 59 | Ok(schema) 60 | } else { 61 | Ok("".to_string()) 62 | } 63 | } 64 | 65 | pub fn delete_table(&self, table_name: String) -> Result<(), ServiceError> { 66 | self.connection 67 | .execute(f!("DROP TABLE IF EXISTS {table_name};"))?; 68 | Ok(()) 69 | } 70 | 71 | /// Remove expired records (based on `timestamp_created`), expired tombstones (based on `timestamp_issued`) 72 | /// and then expired keys without actual records 73 | pub fn clear_expired( 74 | &self, 75 | current_timestamp_sec: u64, 76 | ) -> Result<(u64, u64, u64), ServiceError> { 77 | let config = load_config(); 78 | 79 | let expired_timestamp = current_timestamp_sec - config.expired_timeout; 80 | let deleted_tombstones = self.clear_expired_tombstones(expired_timestamp)?; 81 | let deleted_records = self.clear_expired_records(expired_timestamp)?; 82 | let deleted_keys = self.clear_expired_keys(expired_timestamp)?; 83 | 84 | Ok((deleted_keys, deleted_records, deleted_tombstones)) 85 | } 86 | 87 | pub fn evict_stale( 88 | &self, 89 | current_timestamp_sec: u64, 90 | ) -> Result, ServiceError> { 91 | let stale_timestamp = current_timestamp_sec - load_config().stale_timeout; 92 | 93 | let stale_keys = self.get_stale_keys(stale_timestamp)?; 94 | let mut results: Vec = vec![]; 95 | for key in stale_keys.into_iter() { 96 | let records: Vec = self 97 | .get_records(key.key.id.clone(), current_timestamp_sec)? 98 | .into_iter() 99 | .map(|r| r.record) 100 | .collect(); 101 | 102 | let tombstones = self.get_tombstones(key.key.id.clone(), current_timestamp_sec)?; 103 | results.push(EvictStaleItem { 104 | key: key.key, 105 | records, 106 | tombstones, 107 | }); 108 | } 109 | 110 | Ok(results) 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /service/src/tetraplets_checkers.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use crate::defaults::{ 18 | TRUSTED_TIMESTAMP_FUNCTION_NAME, TRUSTED_TIMESTAMP_SERVICE_ID, TRUSTED_WEIGHT_FUNCTION_NAME, 19 | TRUSTED_WEIGHT_SERVICE_ID, 20 | }; 21 | use crate::error::ServiceError; 22 | use crate::error::ServiceError::{InvalidTimestampTetraplet, InvalidWeightTetraplet}; 23 | use marine_rs_sdk::CallParameters; 24 | 25 | /// Check timestamps are generated on the current host with builtin ("peer" "timestamp_sec") 26 | pub(crate) fn check_timestamp_tetraplets( 27 | call_parameters: &CallParameters, 28 | arg_number: usize, 29 | ) -> Result<(), ServiceError> { 30 | let tetraplets = call_parameters 31 | .tetraplets 32 | .get(arg_number) 33 | .ok_or_else(|| InvalidTimestampTetraplet(format!("{:?}", call_parameters.tetraplets)))?; 34 | let tetraplet = tetraplets 35 | .get(0) 36 | .ok_or_else(|| InvalidTimestampTetraplet(format!("{:?}", call_parameters.tetraplets)))?; 37 | (tetraplet.service_id == TRUSTED_TIMESTAMP_SERVICE_ID 38 | && tetraplet.function_name == TRUSTED_TIMESTAMP_FUNCTION_NAME 39 | && tetraplet.peer_pk == call_parameters.host_id) 40 | .then_some(()) 41 | .ok_or_else(|| InvalidTimestampTetraplet(format!("{:?}", tetraplet))) 42 | } 43 | 44 | pub(crate) fn check_weight_tetraplets( 45 | call_parameters: &CallParameters, 46 | arg_number: usize, 47 | index: usize, 48 | ) -> Result<(), ServiceError> { 49 | let tetraplets = call_parameters 50 | .tetraplets 51 | .get(arg_number) 52 | .ok_or_else(|| InvalidWeightTetraplet(format!("{:?}", call_parameters.tetraplets)))?; 53 | let tetraplet = tetraplets 54 | .get(index) 55 | .ok_or_else(|| InvalidWeightTetraplet(format!("{:?}", call_parameters.tetraplets)))?; 56 | (tetraplet.service_id == TRUSTED_WEIGHT_SERVICE_ID 57 | && tetraplet.function_name == TRUSTED_WEIGHT_FUNCTION_NAME 58 | && tetraplet.peer_pk == call_parameters.host_id) 59 | .then_some(()) 60 | .ok_or_else(|| InvalidWeightTetraplet(format!("{:?}", tetraplet))) 61 | } 62 | -------------------------------------------------------------------------------- /service/src/tombstone.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use crate::error::ServiceError; 18 | use crate::misc::extract_public_key; 19 | use fluence_keypair::Signature; 20 | use marine_rs_sdk::marine; 21 | use sha2::{Digest, Sha256}; 22 | 23 | #[marine] 24 | #[derive(Debug, Default, Clone)] 25 | pub struct Tombstone { 26 | /// base58-encoded key id 27 | pub key_id: String, 28 | /// peer id of the issuer in base58 29 | pub issued_by: String, 30 | /// peer_id of hoster 31 | pub peer_id: String, 32 | /// timestamp in seconds 33 | pub timestamp_issued: u64, 34 | /// will be used for permissions 35 | pub solution: Vec, 36 | /// encoded and hashed previous fields signed by `issued_by` 37 | pub issuer_signature: Vec, 38 | } 39 | 40 | impl Tombstone { 41 | pub fn signature_bytes(&self) -> Vec { 42 | let mut bytes = Vec::new(); 43 | bytes.push(self.key_id.len() as u8); 44 | bytes.extend(self.key_id.as_bytes()); 45 | 46 | bytes.push(self.issued_by.len() as u8); 47 | bytes.extend(self.issued_by.as_bytes()); 48 | 49 | bytes.push(self.peer_id.len() as u8); 50 | bytes.extend(self.peer_id.as_bytes()); 51 | 52 | bytes.extend(self.timestamp_issued.to_le_bytes()); 53 | 54 | bytes.push(self.solution.len() as u8); 55 | bytes.extend(&self.solution); 56 | 57 | let mut hasher = Sha256::new(); 58 | hasher.update(bytes); 59 | hasher.finalize().to_vec() 60 | } 61 | 62 | pub fn verify(&self, current_timestamp_sec: u64) -> Result<(), ServiceError> { 63 | if self.timestamp_issued > current_timestamp_sec { 64 | return Err(ServiceError::InvalidTombstoneTimestamp); 65 | } 66 | 67 | let pk = extract_public_key(self.issued_by.clone())?; 68 | let bytes = self.signature_bytes(); 69 | let signature = Signature::from_bytes(pk.get_key_format(), self.issuer_signature.clone()); 70 | pk.verify(&bytes, &signature).map_err(|e| { 71 | ServiceError::InvalidTombstoneSignature(self.key_id.clone(), self.issued_by.clone(), e) 72 | }) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /service/src/tombstone_api.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | use crate::error::ServiceError; 17 | use crate::results::{GetTombstonesResult, RegistryResult}; 18 | use crate::storage_impl::get_storage; 19 | use crate::tetraplets_checkers::check_timestamp_tetraplets; 20 | use crate::tombstone::Tombstone; 21 | use crate::wrapped_try; 22 | use marine_rs_sdk::marine; 23 | 24 | #[marine] 25 | pub fn get_tombstone_bytes( 26 | key_id: String, 27 | issued_by: String, 28 | peer_id: String, 29 | timestamp_issued: u64, 30 | solution: Vec, 31 | ) -> Vec { 32 | Tombstone { 33 | key_id, 34 | issued_by, 35 | peer_id, 36 | timestamp_issued, 37 | solution, 38 | ..Default::default() 39 | } 40 | .signature_bytes() 41 | } 42 | 43 | #[marine] 44 | pub fn add_tombstone( 45 | key_id: String, 46 | issued_by: String, 47 | peer_id: String, 48 | timestamp_issued: u64, 49 | solution: Vec, 50 | signature: Vec, 51 | current_timestamp_sec: u64, 52 | ) -> RegistryResult { 53 | wrapped_try(|| { 54 | let cp = marine_rs_sdk::get_call_parameters(); 55 | check_timestamp_tetraplets(&cp, 6)?; 56 | let tombstone = Tombstone { 57 | key_id, 58 | issued_by, 59 | peer_id, 60 | timestamp_issued, 61 | solution, 62 | issuer_signature: signature, 63 | }; 64 | tombstone.verify(current_timestamp_sec)?; 65 | 66 | let storage = get_storage()?; 67 | storage.check_key_existence(&tombstone.key_id)?; 68 | storage.write_tombstone(tombstone) 69 | }) 70 | .into() 71 | } 72 | 73 | /// Return all tombstones by key id 74 | #[marine] 75 | pub fn get_tombstones(key_id: String, current_timestamp_sec: u64) -> GetTombstonesResult { 76 | wrapped_try(|| { 77 | let call_parameters = marine_rs_sdk::get_call_parameters(); 78 | check_timestamp_tetraplets(&call_parameters, 1)?; 79 | let storage = get_storage()?; 80 | storage.check_key_existence(&key_id)?; 81 | storage.get_tombstones(key_id, current_timestamp_sec) 82 | }) 83 | .into() 84 | } 85 | 86 | /// If the key exists, then merge tombstones with existing (last-write-wins) 87 | #[marine] 88 | pub fn republish_tombstones( 89 | tombstones: Vec, 90 | current_timestamp_sec: u64, 91 | ) -> RegistryResult { 92 | wrapped_try(|| { 93 | if tombstones.is_empty() { 94 | return Ok(()); 95 | } 96 | 97 | let key_id = tombstones[0].key_id.clone(); 98 | let call_parameters = marine_rs_sdk::get_call_parameters(); 99 | check_timestamp_tetraplets(&call_parameters, 1)?; 100 | 101 | for tombstone in tombstones.iter() { 102 | tombstone.verify(current_timestamp_sec)?; 103 | 104 | if tombstone.key_id != key_id { 105 | return Err(ServiceError::TombstonesPublishingError); 106 | } 107 | } 108 | 109 | let storage = get_storage()?; 110 | storage.check_key_existence(&key_id)?; 111 | for tombstone in tombstones.into_iter() { 112 | storage.write_tombstone(tombstone)?; 113 | } 114 | 115 | Ok(()) 116 | }) 117 | .into() 118 | } 119 | -------------------------------------------------------------------------------- /service/src/tombstone_storage_impl.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Fluence Labs Limited 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | use crate::defaults::RECORDS_TABLE_NAME; 17 | use crate::error::ServiceError; 18 | use crate::load_config; 19 | use crate::storage_impl::Storage; 20 | use crate::tombstone::Tombstone; 21 | use marine_sqlite_connector::{State, Statement, Value}; 22 | 23 | pub fn read_tombstone(statement: &Statement) -> Result { 24 | Ok(Tombstone { 25 | key_id: statement.read::(0)?, 26 | issued_by: statement.read::(1)?, 27 | peer_id: statement.read::(2)?, 28 | timestamp_issued: statement.read::(3)? as u64, 29 | solution: statement.read::>(4)?, 30 | issuer_signature: statement.read::>(5)?, 31 | }) 32 | } 33 | 34 | impl Storage { 35 | /// insert tombstone if a record or tombstone with `(key_id, issued_by, peer_id)` does not exist 36 | /// or replace if it has lower `timestamp_issued` 37 | pub fn write_tombstone(&self, tombstone: Tombstone) -> Result<(), ServiceError> { 38 | self.check_row( 39 | tombstone.key_id.clone(), 40 | tombstone.issued_by.clone(), 41 | tombstone.peer_id.clone(), 42 | tombstone.timestamp_issued, 43 | )?; 44 | let mut statement = self.connection.prepare(f!( 45 | "INSERT OR REPLACE INTO {RECORDS_TABLE_NAME} VALUES (?, ?, ?, ?, ?, ?, ?, \ 46 | NULL, NULL, NULL, NULL, NULL, NULL);" 47 | ))?; 48 | 49 | let is_tombstoned = 1; 50 | statement.bind(1, &Value::String(tombstone.key_id.clone()))?; 51 | statement.bind(2, &Value::String(tombstone.issued_by.clone()))?; 52 | statement.bind(3, &Value::String(tombstone.peer_id.clone()))?; 53 | statement.bind(4, &Value::Integer(tombstone.timestamp_issued as i64))?; 54 | statement.bind(5, &Value::Binary(tombstone.solution))?; 55 | statement.bind(6, &Value::Binary(tombstone.issuer_signature))?; 56 | statement.bind(7, &Value::Integer(is_tombstoned))?; 57 | 58 | statement.next().map(drop)?; 59 | 60 | Ok(()) 61 | } 62 | 63 | pub fn get_tombstones( 64 | &self, 65 | key_id: String, 66 | current_timestamp_sec: u64, 67 | ) -> Result, ServiceError> { 68 | let mut statement = self.connection.prepare(f!( 69 | "SELECT key_id, issued_by, peer_id, timestamp_issued, solution, issuer_signature \ 70 | FROM {RECORDS_TABLE_NAME} WHERE key_id = ? AND is_tombstoned = 1 and timestamp_issued > ?" 71 | ))?; 72 | 73 | let expired_timestamp = current_timestamp_sec - load_config().expired_timeout; 74 | statement.bind(1, &Value::String(key_id))?; 75 | statement.bind(2, &Value::Integer(expired_timestamp as i64))?; 76 | 77 | let mut result: Vec = vec![]; 78 | 79 | while let State::Row = statement.next()? { 80 | result.push(read_tombstone(&statement)?) 81 | } 82 | 83 | Ok(result) 84 | } 85 | 86 | /// Remove expired tombstones 87 | pub fn clear_expired_tombstones(&self, expired_timestamp: u64) -> Result { 88 | self.connection.execute(f!( 89 | "DELETE FROM {RECORDS_TABLE_NAME} WHERE timestamp_created <= {expired_timestamp} AND is_tombstoned = 1" 90 | ))?; 91 | Ok(self.connection.changes() as u64) 92 | } 93 | } 94 | --------------------------------------------------------------------------------