├── .codeclimate.yml ├── .codecov.yml ├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ └── feature_request.md ├── config.yml ├── dependabot.yml └── workflows │ ├── codeql-analysis.yml │ ├── docker-image.yml │ ├── stale.yml │ └── tests.yml ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── COPYRIGHT ├── Dockerfile ├── Dockerfile-test ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT ├── Makefile ├── README.md ├── add_test.go ├── adder ├── adder.go ├── adder_test.go ├── adderutils │ └── adderutils.go ├── ipfsadd │ └── add.go ├── sharding │ ├── dag.go │ ├── dag_service.go │ ├── dag_service_test.go │ ├── shard.go │ └── verify.go ├── single │ ├── dag_service.go │ └── dag_service_test.go └── util.go ├── allocate.go ├── allocator └── balanced │ ├── balanced.go │ ├── balanced_test.go │ ├── config.go │ └── config_test.go ├── api ├── add.go ├── add_test.go ├── common │ ├── api.go │ ├── api_test.go │ ├── config.go │ ├── config_test.go │ └── test │ │ ├── helpers.go │ │ ├── server.crt │ │ └── server.key ├── ipfsproxy │ ├── config.go │ ├── config_test.go │ ├── headers.go │ ├── ipfsproxy.go │ ├── ipfsproxy_test.go │ └── util.go ├── pb │ ├── generate.go │ ├── types.pb.go │ └── types.proto ├── pinsvcapi │ ├── config.go │ ├── pinsvc │ │ └── pinsvc.go │ ├── pinsvcapi.go │ └── pinsvcapi_test.go ├── rest │ ├── client │ │ ├── .travis.yml │ │ ├── README.md │ │ ├── client.go │ │ ├── client_test.go │ │ ├── lbclient.go │ │ ├── lbclient_test.go │ │ ├── methods.go │ │ ├── methods_test.go │ │ ├── request.go │ │ └── transports.go │ ├── config.go │ ├── restapi.go │ └── restapi_test.go ├── types.go ├── types_test.go └── util.go ├── cluster.go ├── cluster_config.go ├── cluster_config_test.go ├── cluster_test.go ├── clusterhost.go ├── cmd ├── ipfs-cluster-ctl │ ├── Makefile │ ├── dist │ │ ├── LICENSE │ │ ├── LICENSE-APACHE │ │ ├── LICENSE-MIT │ │ └── README.md │ ├── formatters.go │ ├── graph.go │ ├── graph_test.go │ └── main.go ├── ipfs-cluster-follow │ ├── Makefile │ ├── commands.go │ ├── dist │ │ ├── LICENSE │ │ ├── LICENSE-APACHE │ │ ├── LICENSE-MIT │ │ └── README.md │ └── main.go └── ipfs-cluster-service │ ├── Makefile │ ├── daemon.go │ ├── dist │ ├── LICENSE │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ └── README.md │ ├── export.json │ ├── lock.go │ ├── main.go │ └── main_test.go ├── cmdutils ├── cmdutils.go ├── configs.go └── state.go ├── config ├── config.go ├── config_test.go ├── identity.go ├── identity_test.go └── util.go ├── config_test.go ├── connect_graph.go ├── consensus ├── crdt │ ├── config.go │ ├── config_test.go │ ├── consensus.go │ └── consensus_test.go └── raft │ ├── config.go │ ├── config_test.go │ ├── consensus.go │ ├── consensus_test.go │ ├── data_helper.go │ ├── data_helper_test.go │ ├── log_op.go │ ├── log_op_test.go │ └── raft.go ├── datastore ├── badger │ ├── badger.go │ ├── config.go │ └── config_test.go ├── badger3 │ ├── badger.go │ ├── config.go │ └── config_test.go ├── inmem │ └── inmem.go ├── leveldb │ ├── config.go │ ├── config_test.go │ └── leveldb.go └── pebble │ ├── config.go │ ├── config_test.go │ └── pebble.go ├── docker-compose.yml ├── docker ├── cluster-restart.sh ├── entrypoint.sh ├── get-docker-tags.sh ├── random-killer.sh ├── random-stopper.sh ├── start-daemons.sh ├── test-entrypoint.sh └── wait-killer-stopper.sh ├── go.mod ├── go.sum ├── informer ├── disk │ ├── config.go │ ├── config_test.go │ ├── disk.go │ └── disk_test.go ├── numpin │ ├── config.go │ ├── config_test.go │ ├── numpin.go │ └── numpin_test.go ├── pinqueue │ ├── config.go │ ├── config_test.go │ ├── pinqueue.go │ └── pinqueue_test.go └── tags │ ├── config.go │ ├── config_test.go │ ├── tags.go │ └── tags_test.go ├── internal └── fd │ ├── fd.go │ ├── sys_not_unix.go │ └── sys_unix.go ├── ipfs-cluster.fundring ├── ipfscluster.go ├── ipfscluster_test.go ├── ipfsconn └── ipfshttp │ ├── config.go │ ├── config_test.go │ ├── ipfshttp.go │ └── ipfshttp_test.go ├── logging.go ├── monitor ├── metrics │ ├── checker.go │ ├── checker_test.go │ ├── store.go │ ├── store_test.go │ ├── util.go │ ├── window.go │ └── window_test.go └── pubsubmon │ ├── config.go │ ├── config_test.go │ ├── pubsubmon.go │ └── pubsubmon_test.go ├── observations ├── config.go ├── config_test.go ├── metrics.go └── setup.go ├── peer_manager_test.go ├── pintracker ├── optracker │ ├── operation.go │ ├── operation_test.go │ ├── operationtracker.go │ ├── operationtracker_test.go │ ├── operationtype_string.go │ └── phase_string.go ├── pintracker_test.go └── stateless │ ├── config.go │ ├── config_test.go │ ├── stateless.go │ └── stateless_test.go ├── pnet_test.go ├── pstoremgr ├── pstoremgr.go └── pstoremgr_test.go ├── release.sh ├── rpc_api.go ├── rpc_policy.go ├── rpcutil ├── policygen │ └── policygen.go └── rpcutil.go ├── sharness ├── config │ ├── basic_auth │ │ ├── identity.json │ │ └── service.json │ ├── ssl-basic_auth │ │ ├── identity.json │ │ ├── server.crt │ │ ├── server.key │ │ └── service.json │ └── ssl │ │ ├── identity.json │ │ ├── server.crt │ │ ├── server.key │ │ └── service.json ├── lib │ └── test-lib.sh ├── run-sharness-tests.sh ├── t0010-ctl-basic-commands.sh ├── t0020-service-basic-commands.sh ├── t0021-service-init.sh ├── t0025-ctl-status-report-commands.sh ├── t0030-ctl-pin.sh ├── t0031-ctl-add.sh ├── t0032-ctl-health.sh ├── t0040-ssl-simple-exchange.sh ├── t0041-ssl-enforcement.sh ├── t0042-basic-auth.sh ├── t0043-ssl-basic-auth.sh ├── t0052-service-state-export.sh ├── t0053-service-state-import.sh ├── t0054-service-state-clean.sh └── test_data │ ├── importState │ ├── small_file │ └── v1Crc ├── state ├── dsstate │ ├── datastore.go │ └── datastore_test.go ├── empty.go └── interface.go ├── test ├── cids.go ├── ipfs_mock.go ├── rpc_api_mock.go ├── sharding.go ├── test.go └── test_test.go ├── util.go └── version └── version.go /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | ratings: 2 | paths: 3 | - "**/*.go" 4 | 5 | checks: 6 | file-lines: 7 | config: 8 | threshold: 500 9 | method-complexity: 10 | config: 11 | threshold: 15 12 | method-lines: 13 | config: 14 | threshold: 80 15 | similar-code: 16 | enabled: false 17 | return-statements: 18 | config: 19 | threshold: 10 20 | argument-count: 21 | config: 22 | threshold: 6 23 | 24 | engines: 25 | fixme: 26 | enabled: true 27 | config: 28 | strings: 29 | - FIXME 30 | - HACK 31 | - XXX 32 | - BUG 33 | golint: 34 | enabled: true 35 | govet: 36 | enabled: true 37 | gofmt: 38 | enabled: true -------------------------------------------------------------------------------- /.codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | # basic 6 | target: auto 7 | threshold: 50 8 | base: auto 9 | # advanced 10 | branches: null 11 | if_no_uploads: error 12 | if_not_found: success 13 | if_ci_failed: error 14 | only_pulls: false 15 | flags: null 16 | paths: null 17 | patch: 18 | default: 19 | # basic 20 | target: auto 21 | threshold: 50 22 | base: auto 23 | # advanced 24 | branches: null 25 | if_no_uploads: error 26 | if_not_found: success 27 | if_ci_failed: error 28 | only_pulls: false 29 | flags: null 30 | paths: null 31 | comment: false 32 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile* 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a bug report for IPFS Cluster. 4 | title: '' 5 | labels: kind/bug, need/triage 6 | assignees: '' 7 | 8 | --- 9 | 10 | 19 | 20 | **Additional information:** 21 | - OS: [e.g. Linux] 22 | - IPFS Cluster version: [e.g. 0.10.1, master] 23 | - Installation method: [e.g. built from source, dist.ipfs.io, docker] 24 | 25 | **Describe the bug:** 26 | 27 | A clear and concise description of what the bug is and how the behavior was different from expected. 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: IPFS Cluster Website/Documentation issues 4 | url: https://github.com/ipfs/ipfs-cluster-website 5 | about: Report issues with the Docs or anything in the IPFS Cluster website. 6 | - name: Official Documentation 7 | url: https://cluster.ipfs.io/documentation 8 | about: Extensive documentation is available at the IPFS Cluster website. 9 | - name: IPFS Official Forums 10 | url: https://discuss.ipfs.io 11 | about: Please post general questions, support requests, and discussions here. 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature/Enhancement request 3 | about: Suggest an idea to improve IPFS Cluster. 4 | title: '' 5 | labels: kind/feature, need/triage 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the feature you are proposing** 11 | 12 | Please describe in detail the feature you want to propose, your use-case and how you would benefit from it. 13 | 14 | **Additional context** 15 | 16 | Add any other context here, if relevant, that may help better understand your request. 17 | -------------------------------------------------------------------------------- /.github/config.yml: -------------------------------------------------------------------------------- 1 | # Configuration for welcome - https://github.com/behaviorbot/welcome 2 | 3 | # Configuration for new-issue-welcome - https://github.com/behaviorbot/new-issue-welcome 4 | # Comment to be posted to on first time issues 5 | newIssueWelcomeComment: > 6 | Thank you for submitting your first issue to this repository! A maintainer 7 | will be here shortly to triage and review. 8 | 9 | In the meantime, please double-check that you have provided all the 10 | necessary information to make this process easy! Any information that can 11 | help save additional round trips is useful! We currently aim to give 12 | initial feedback within **two business days**. If this does not happen, feel 13 | free to leave a comment. 14 | 15 | Please keep an eye on how this issue will be labeled, as labels give an 16 | overview of priorities, assignments and additional actions requested by the 17 | maintainers: 18 | 19 | - "Priority" labels will show how urgent this is for the team. 20 | - "Status" labels will show if this is ready to be worked on, blocked, or in progress. 21 | - "Need" labels will indicate if additional input or analysis is required. 22 | 23 | Finally, remember to use https://discuss.ipfs.io if you just need general 24 | support. 25 | 26 | # Configuration for new-pr-welcome - https://github.com/behaviorbot/new-pr-welcome 27 | # Comment to be posted to on PRs from first time contributors in your repository 28 | newPRWelcomeComment: > 29 | Thank you for submitting this PR! 30 | 31 | A maintainer will be here shortly to review it. 32 | 33 | We are super grateful, but we are also overloaded! Help us by making sure 34 | that: 35 | 36 | * The context for this PR is clear, with relevant discussion, decisions 37 | and stakeholders linked/mentioned. 38 | 39 | * Your contribution itself is clear (code comments, self-review for the 40 | rest) and in its best form. Follow the [code contribution 41 | guidelines](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md#code-contribution-guidelines) 42 | if they apply. 43 | 44 | Getting other community members to do a review would be great help too on 45 | complex PRs (you can ask in the chats/forums). If you are unsure about 46 | something, just leave us a comment. 47 | 48 | Next steps: 49 | 50 | * A maintainer will triage and assign priority to this PR, commenting on 51 | any missing things and potentially assigning a reviewer for high 52 | priority items. 53 | 54 | * The PR gets reviews, discussed and approvals as needed. 55 | 56 | * The PR is merged by maintainers when it has been approved and comments addressed. 57 | 58 | We currently aim to provide initial feedback/triaging within **two business 59 | days**. Please keep an eye on any labeling actions, as these will indicate 60 | priorities and status of your contribution. 61 | 62 | We are very grateful for your contribution! 63 | 64 | 65 | # Configuration for first-pr-merge - https://github.com/behaviorbot/first-pr-merge 66 | # Comment to be posted to on pull requests merged by a first time user 67 | # Currently disabled 68 | #firstPRMergeComment: "" 69 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: weekly 7 | time: "11:00" 8 | open-pull-requests-limit: 20 9 | target-branch: dependency-upgrades 10 | labels: 11 | - topic/dependencies 12 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "master" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "master" ] 20 | schedule: 21 | - cron: '20 2 * * 5' 22 | 23 | env: 24 | GO: 1.23.3 25 | 26 | jobs: 27 | analyze: 28 | name: Analyze 29 | runs-on: ubuntu-latest 30 | permissions: 31 | actions: read 32 | contents: read 33 | security-events: write 34 | 35 | strategy: 36 | fail-fast: false 37 | matrix: 38 | language: [ 'go' ] 39 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 40 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 41 | 42 | steps: 43 | - name: Checkout repository 44 | uses: actions/checkout@v3 45 | 46 | - name: Set up Go 47 | uses: actions/setup-go@v2 48 | with: 49 | go-version: ${{ env.GO }} 50 | 51 | # Initializes the CodeQL tools for scanning. 52 | - name: Initialize CodeQL 53 | uses: github/codeql-action/init@v2 54 | with: 55 | languages: ${{ matrix.language }} 56 | # If you wish to specify custom queries, you can do so here or in a config file. 57 | # By default, queries listed here will override any specified in a config file. 58 | # Prefix the list here with "+" to use these queries and those in the config file. 59 | 60 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 61 | # queries: security-extended,security-and-quality 62 | 63 | 64 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 65 | # If this step fails, then you should remove it and run the build manually (see below) 66 | - name: Autobuild 67 | uses: github/codeql-action/autobuild@v2 68 | 69 | # ℹ️ Command-line programs to run using the OS shell. 70 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 71 | 72 | # If the Autobuild fails above, remove it and uncomment the following three lines. 73 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 74 | 75 | # - run: | 76 | # echo "Run, Build Application using script" 77 | # ./location_of_script_within_repo/buildscript.sh 78 | 79 | - name: Perform CodeQL Analysis 80 | uses: github/codeql-action/analyze@v2 81 | -------------------------------------------------------------------------------- /.github/workflows/docker-image.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker image 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - 'master' 8 | tags: 9 | - 'v*' 10 | 11 | env: 12 | REGISTRY: "" 13 | IMAGE_NAME: ipfs/ipfs-cluster 14 | 15 | jobs: 16 | build-and-push-image: 17 | runs-on: ubuntu-latest 18 | permissions: 19 | contents: read 20 | packages: write 21 | steps: 22 | - name: Checkout repository 23 | uses: actions/checkout@v4 24 | 25 | - name: Set up QEMU 26 | uses: docker/setup-qemu-action@v3 27 | 28 | - name: Set up Docker Buildx 29 | uses: docker/setup-buildx-action@v3 30 | 31 | - name: Cache Docker layers 32 | uses: actions/cache@v3 33 | with: 34 | path: /tmp/.buildx-cache 35 | key: ${{ runner.os }}-buildx-${{ github.sha }} 36 | restore-keys: | 37 | ${{ runner.os }}-buildx- 38 | 39 | - name: Log in to the Container registry 40 | uses: docker/login-action@v3 41 | with: 42 | # registry: ${{ env.REGISTRY }} 43 | username: ${{ secrets.DOCKER_USERNAME }} 44 | password: ${{ secrets.DOCKER_PASSWORD }} 45 | 46 | - name: Get tags 47 | id: tags 48 | run: | 49 | echo "value<> $GITHUB_OUTPUT 50 | ./docker/get-docker-tags.sh "$(date -u +%F)" >> $GITHUB_OUTPUT 51 | echo "EOF" >> $GITHUB_OUTPUT 52 | shell: bash 53 | 54 | - name: Build Docker image and publish to Docker Hub 55 | uses: docker/build-push-action@v4 56 | with: 57 | platforms: linux/amd64,linux/arm/v7,linux/arm64/v8 58 | context: . 59 | push: true 60 | file: ./Dockerfile 61 | tags: "${{ steps.tags.outputs.value }}" 62 | cache-from: type=local,src=/tmp/.buildx-cache 63 | cache-to: type=local,dest=/tmp/.buildx-cache-new 64 | 65 | # https://github.com/docker/build-push-action/issues/252 66 | # https://github.com/moby/buildkit/issues/1896 67 | - name: Move cache to limit growth 68 | run: | 69 | rm -rf /tmp/.buildx-cache 70 | mv /tmp/.buildx-cache-new /tmp/.buildx-cache 71 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Close and mark stale issue 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' 6 | 7 | jobs: 8 | stale: 9 | 10 | runs-on: ubuntu-latest 11 | permissions: 12 | issues: write 13 | pull-requests: write 14 | 15 | steps: 16 | - uses: actions/stale@v3 17 | with: 18 | repo-token: ${{ secrets.GITHUB_TOKEN }} 19 | stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' 20 | close-issue-message: 'This issue was closed because it is missing author input.' 21 | stale-issue-label: 'kind/stale' 22 | any-of-labels: 'need/author-input' 23 | exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis' 24 | days-before-issue-stale: 6 25 | days-before-issue-close: 7 26 | enable-statistics: true 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tag_annotation 2 | coverage.out 3 | cmd/ipfs-cluster-service/ipfs-cluster-service 4 | cmd/ipfs-cluster-ctl/ipfs-cluster-ctl 5 | cmd/ipfs-cluster-follow/ipfs-cluster-follow 6 | sharness/lib/sharness 7 | sharness/test-results 8 | sharness/trash* 9 | vendor/ 10 | 11 | 12 | raftFolderFromTest* 13 | peerstore 14 | shardTesting 15 | compose 16 | 17 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 18 | *.o 19 | *.a 20 | *.so 21 | 22 | # Folders 23 | _obj 24 | _test 25 | test/sharness/test-results 26 | test/sharness/trash* 27 | test/sharness/lib/sharness 28 | test/sharness/.test_config 29 | test/sharness/.test_ipfs 30 | 31 | # Architecture specific extensions/prefixes 32 | *.[568vq] 33 | [568vq].out 34 | 35 | *.cgo1.go 36 | *.cgo2.c 37 | _cgo_defun.c 38 | _cgo_gotypes.go 39 | _cgo_export.* 40 | 41 | _testmain.go 42 | 43 | *.exe 44 | *.test 45 | *.prof 46 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Guidelines for contributing 2 | 3 | Please see https://ipfscluster.io/developer/contribute . 4 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Copyright 2019. Protocol Labs, Inc. 2 | 3 | This library is dual-licensed under Apache 2.0 and MIT terms. 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.24-bullseye AS builder 2 | MAINTAINER Hector Sanjuan 3 | 4 | # This dockerfile builds and runs ipfs-cluster-service. 5 | ARG TARGETPLATFORM TARGETOS TARGETARCH 6 | 7 | ENV GOPATH /go 8 | ENV SRC_PATH $GOPATH/src/github.com/ipfs-cluster/ipfs-cluster 9 | ENV GOPROXY https://proxy.golang.org 10 | 11 | COPY --chown=1000:users go.* $SRC_PATH/ 12 | WORKDIR $SRC_PATH 13 | RUN go mod download -x 14 | 15 | COPY --chown=1000:users . $SRC_PATH 16 | RUN git config --global --add safe.directory /go/src/github.com/ipfs-cluster/ipfs-cluster 17 | 18 | ENV CGO_ENABLED 0 19 | RUN make install 20 | 21 | 22 | #------------------------------------------------------ 23 | FROM alpine:3.18 24 | MAINTAINER Hector Sanjuan 25 | 26 | LABEL org.opencontainers.image.source=https://github.com/ipfs-cluster/ipfs-cluster 27 | LABEL org.opencontainers.image.description="Pinset orchestration for IPFS" 28 | LABEL org.opencontainers.image.licenses=MIT+APACHE_2.0 29 | 30 | # Install binaries for $TARGETARCH 31 | RUN apk add --no-cache tini su-exec ca-certificates 32 | 33 | ENV GOPATH /go 34 | ENV SRC_PATH /go/src/github.com/ipfs-cluster/ipfs-cluster 35 | ENV IPFS_CLUSTER_PATH /data/ipfs-cluster 36 | ENV IPFS_CLUSTER_CONSENSUS crdt 37 | 38 | EXPOSE 9094 39 | EXPOSE 9095 40 | EXPOSE 9096 41 | 42 | COPY --from=builder $GOPATH/bin/ipfs-cluster-service /usr/local/bin/ipfs-cluster-service 43 | COPY --from=builder $GOPATH/bin/ipfs-cluster-ctl /usr/local/bin/ipfs-cluster-ctl 44 | COPY --from=builder $GOPATH/bin/ipfs-cluster-follow /usr/local/bin/ipfs-cluster-follow 45 | COPY --from=builder $SRC_PATH/docker/entrypoint.sh /usr/local/bin/entrypoint.sh 46 | 47 | RUN mkdir -p $IPFS_CLUSTER_PATH && \ 48 | adduser -D -h $IPFS_CLUSTER_PATH -u 1000 -G users ipfs && \ 49 | chown ipfs:users $IPFS_CLUSTER_PATH 50 | 51 | VOLUME $IPFS_CLUSTER_PATH 52 | ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/entrypoint.sh"] 53 | 54 | # Defaults for ipfs-cluster-service go here 55 | CMD ["daemon"] 56 | -------------------------------------------------------------------------------- /Dockerfile-test: -------------------------------------------------------------------------------- 1 | FROM golang:1.24-bullseye AS builder 2 | MAINTAINER Hector Sanjuan 3 | 4 | # This build state just builds the cluster binaries 5 | 6 | ENV GOPATH /go 7 | ENV SRC_PATH $GOPATH/src/github.com/ipfs-cluster/ipfs-cluster 8 | ENV GO111MODULE on 9 | ENV GOPROXY https://proxy.golang.org 10 | 11 | RUN cd /tmp && \ 12 | wget https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && \ 13 | chmod +x jq-linux64 14 | 15 | COPY --chown=1000:users go.* $SRC_PATH/ 16 | WORKDIR $SRC_PATH 17 | RUN go mod download 18 | 19 | COPY --chown=1000:users . $SRC_PATH 20 | RUN git config --global --add safe.directory /go/src/github.com/ipfs-cluster/ipfs-cluster 21 | RUN make install 22 | 23 | #------------------------------------------------------ 24 | FROM ipfs/kubo:master-latest 25 | MAINTAINER Hector Sanjuan 26 | 27 | # This is the container which just puts the previously 28 | # built binaries on the kubo-container. 29 | 30 | ENV GOPATH /go 31 | ENV SRC_PATH /go/src/github.com/ipfs-cluster/ipfs-cluster 32 | ENV IPFS_CLUSTER_PATH /data/ipfs-cluster 33 | ENV IPFS_CLUSTER_CONSENSUS crdt 34 | ENV IPFS_CLUSTER_DATASTORE pebble 35 | ENV IPFS_CLUSTER_RESTAPI_HTTPLISTENMULTIADDRESS /ip4/0.0.0.0/tcp/9094 36 | ENV IPFS_CLUSTER_IPFSPROXY_LISTENMULTIADDRESS /ip4/0.0.0.0/tcp/9095 37 | 38 | EXPOSE 9094 39 | EXPOSE 9095 40 | EXPOSE 9096 41 | 42 | COPY --from=builder $GOPATH/bin/ipfs-cluster-service /usr/local/bin/ipfs-cluster-service 43 | COPY --from=builder $GOPATH/bin/ipfs-cluster-ctl /usr/local/bin/ipfs-cluster-ctl 44 | COPY --from=builder $GOPATH/bin/ipfs-cluster-follow /usr/local/bin/ipfs-cluster-follow 45 | COPY --from=builder $SRC_PATH/docker/test-entrypoint.sh /usr/local/bin/test-entrypoint.sh 46 | COPY --from=builder $SRC_PATH/docker/random-stopper.sh /usr/local/bin/random-stopper.sh 47 | COPY --from=builder $SRC_PATH/docker/random-killer.sh /usr/local/bin/random-killer.sh 48 | COPY --from=builder $SRC_PATH/docker/wait-killer-stopper.sh /usr/local/bin/wait-killer-stopper.sh 49 | COPY --from=builder $SRC_PATH/docker/cluster-restart.sh /usr/local/bin/cluster-restart.sh 50 | 51 | # Add jq 52 | COPY --from=builder /tmp/jq-linux64 /usr/local/bin/jq 53 | 54 | # Add bash 55 | COPY --from=builder /bin/bash /bin/bash 56 | COPY --from=builder /lib/*-linux-gnu*/libtinfo.so* /lib64/ 57 | 58 | USER root 59 | 60 | RUN mkdir -p $IPFS_CLUSTER_PATH && \ 61 | chown 1000:100 $IPFS_CLUSTER_PATH 62 | 63 | USER ipfs 64 | 65 | VOLUME $IPFS_CLUSTER_PATH 66 | ENTRYPOINT ["/usr/local/bin/test-entrypoint.sh"] 67 | 68 | # Defaults would go here 69 | CMD ["daemon"] 70 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Dual-licensed under MIT and ASLv2, by way of the [Permissive License 2 | Stack](https://protocol.ai/blog/announcing-the-permissive-license-stack/). 3 | 4 | Apache-2.0: https://www.apache.org/licenses/license-2.0 5 | MIT: https://www.opensource.org/licenses/mit 6 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Copyright 2020. Protocol Labs, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright 2020. Protocol Labs, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | sharness = sharness/lib/sharness 2 | 3 | export GO111MODULE := on 4 | 5 | all: build 6 | clean: rwundo clean_sharness 7 | $(MAKE) -C cmd/ipfs-cluster-service clean 8 | $(MAKE) -C cmd/ipfs-cluster-ctl clean 9 | $(MAKE) -C cmd/ipfs-cluster-follow clean 10 | @rm -rf ./test/testingData 11 | @rm -rf ./compose 12 | 13 | install: 14 | $(MAKE) -C cmd/ipfs-cluster-service install 15 | $(MAKE) -C cmd/ipfs-cluster-ctl install 16 | $(MAKE) -C cmd/ipfs-cluster-follow install 17 | 18 | build: 19 | $(MAKE) -C cmd/ipfs-cluster-service build 20 | $(MAKE) -C cmd/ipfs-cluster-ctl build 21 | $(MAKE) -C cmd/ipfs-cluster-follow build 22 | 23 | service: 24 | $(MAKE) -C cmd/ipfs-cluster-service ipfs-cluster-service 25 | ctl: 26 | $(MAKE) -C cmd/ipfs-cluster-ctl ipfs-cluster-ctl 27 | follow: 28 | $(MAKE) -C cmd/ipfs-cluster-follow ipfs-cluster-follow 29 | 30 | check: 31 | go vet ./... 32 | staticcheck --checks all ./... 33 | misspell -error -locale US . 34 | 35 | test: 36 | go test -v ./... 37 | 38 | test_sharness: $(sharness) 39 | @sh sharness/run-sharness-tests.sh 40 | 41 | test_problem: 42 | go test -timeout 20m -loglevel "DEBUG" -v -run $(problematic_test) 43 | 44 | $(sharness): 45 | @echo "Downloading sharness" 46 | @curl -L -s -o sharness/lib/sharness.tar.gz http://github.com/chriscool/sharness/archive/28c7490f5cdf1e95a8ebebf8b06ed5588db13875.tar.gz 47 | @cd sharness/lib; tar -zxf sharness.tar.gz; cd ../.. 48 | @mv sharness/lib/sharness-28c7490f5cdf1e95a8ebebf8b06ed5588db13875 sharness/lib/sharness 49 | @rm sharness/lib/sharness.tar.gz 50 | 51 | clean_sharness: 52 | @rm -rf ./sharness/test-results 53 | @rm -rf ./sharness/lib/sharness 54 | @rm -rf sharness/trash\ directory* 55 | 56 | docker: 57 | docker build -t cluster-image -f Dockerfile . 58 | docker run --name tmp-make-cluster -d --rm cluster-image && sleep 4 59 | docker exec tmp-make-cluster sh -c "ipfs-cluster-ctl version" 60 | docker exec tmp-make-cluster sh -c "ipfs-cluster-service -v" 61 | docker kill tmp-make-cluster 62 | 63 | docker build -t cluster-image-test -f Dockerfile-test . 64 | docker run --name tmp-make-cluster-test -d --rm cluster-image && sleep 4 65 | docker exec tmp-make-cluster-test sh -c "ipfs-cluster-ctl version" 66 | docker exec tmp-make-cluster-test sh -c "ipfs-cluster-service -v" 67 | docker kill tmp-make-cluster-test 68 | 69 | docker-compose: 70 | mkdir -p compose/ipfs0 compose/ipfs1 compose/cluster0 compose/cluster1 71 | chmod -R 0777 compose 72 | CLUSTER_SECRET=$(shell od -vN 32 -An -tx1 /dev/urandom | tr -d ' \n') docker compose up -d 73 | sleep 35 74 | docker exec cluster0 ipfs-cluster-ctl peers ls 75 | docker exec cluster1 ipfs-cluster-ctl peers ls 76 | docker exec cluster0 ipfs-cluster-ctl peers ls | grep -o "Sees 2 other peers" | uniq -c | grep 3 77 | docker exec cluster1 ipfs-cluster-ctl peers ls | grep -o "Sees 2 other peers" | uniq -c | grep 3 78 | docker compose down 79 | 80 | prcheck: check service ctl follow test 81 | 82 | .PHONY: all test test_sharness clean_sharness rw rwundo publish service ctl install clean docker 83 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # IPFS Cluster 2 | 3 | [![Made by](https://img.shields.io/badge/By-IP%20Shipyard-000000.svg?style=flat-square)](https://ipshipyard.com) 4 | [![Main project](https://img.shields.io/badge/project-ipfs--cluster-ef5c43.svg?style=flat-square)](http://github.com/ipfs-cluster) 5 | [![Discord](https://img.shields.io/badge/forum-discuss.ipfs.io-f9a035.svg?style=flat-square)](https://discuss.ipfs.io/c/help/help-ipfs-cluster/24) 6 | [![Matrix channel](https://img.shields.io/badge/matrix-%23ipfs--cluster-3c8da0.svg?style=flat-square)](https://app.element.io/#/room/#ipfs-cluster:ipfs.io) 7 | [![pkg.go.dev](https://pkg.go.dev/badge/github.com/ipfs-cluster/ipfs-cluster)](https://pkg.go.dev/github.com/ipfs-cluster/ipfs-cluster) 8 | [![Go Report Card](https://goreportcard.com/badge/github.com/ipfs-cluster/ipfs-cluster)](https://goreportcard.com/report/github.com/ipfs-cluster/ipfs-cluster) 9 | [![codecov](https://codecov.io/gh/ipfs-cluster/ipfs-cluster/branch/master/graph/badge.svg)](https://codecov.io/gh/ipfs-cluster/ipfs-cluster) 10 | 11 | > Pinset orchestration for IPFS 12 | 13 |

14 | logo 15 |

16 | 17 | [IPFS Cluster](https://ipfscluster.io) provides data orchestration across a swarm of IPFS daemons by allocating, replicating and tracking a global pinset distributed among multiple peers. 18 | 19 | There are 3 different applications: 20 | 21 | * A cluster peer application: `ipfs-cluster-service`, to be run along with `kubo` (`go-ipfs`) as a sidecar. 22 | * A client CLI application: `ipfs-cluster-ctl`, which allows easily interacting with the peer's HTTP API. 23 | * An additional "follower" peer application: `ipfs-cluster-follow`, focused on simplifying the process of configuring and running follower peers. 24 | 25 | --- 26 | 27 | ## Table of Contents 28 | 29 | - [Documentation](#documentation) 30 | - [News & Roadmap](#news--roadmap) 31 | - [Install](#install) 32 | - [Usage](#usage) 33 | - [Contribute](#contribute) 34 | - [License](#license) 35 | 36 | 37 | ## Documentation 38 | 39 | Please visit https://ipfscluster.io/documentation/ to access user documentation, guides and any other resources, including detailed **download** and **usage** instructions. 40 | 41 | ## News & Roadmap 42 | 43 | We regularly post project updates to https://ipfscluster.io/news/ . 44 | 45 | The most up-to-date *Roadmap* is available at https://ipfscluster.io/roadmap/ . 46 | 47 | ## Install 48 | 49 | Instructions for different installation methods (including from source) are available at https://ipfscluster.io/download . 50 | 51 | ## Usage 52 | 53 | Extensive usage information is provided at https://ipfscluster.io/documentation/ , including: 54 | 55 | * [Docs for `ipfs-cluster-service`](https://ipfscluster.io/documentation/reference/service/) 56 | * [Docs for `ipfs-cluster-ctl`](https://ipfscluster.io/documentation/reference/ctl/) 57 | * [Docs for `ipfs-cluster-follow`](https://ipfscluster.io/documentation/reference/follow/) 58 | 59 | ## Contribute 60 | 61 | PRs accepted. As part of the IPFS project, we have some [contribution guidelines](https://ipfscluster.io/support/#contribution-guidelines). 62 | 63 | ## License 64 | 65 | This library is dual-licensed under Apache 2.0 and MIT terms. 66 | -------------------------------------------------------------------------------- /adder/sharding/verify.go: -------------------------------------------------------------------------------- 1 | package sharding 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "testing" 8 | 9 | "github.com/ipfs-cluster/ipfs-cluster/api" 10 | ) 11 | 12 | // MockPinStore is used in VerifyShards 13 | type MockPinStore interface { 14 | // Gets a pin 15 | PinGet(context.Context, api.Cid) (api.Pin, error) 16 | } 17 | 18 | // MockBlockStore is used in VerifyShards 19 | type MockBlockStore interface { 20 | // Gets a block 21 | BlockGet(context.Context, api.Cid) ([]byte, error) 22 | } 23 | 24 | // VerifyShards checks that a sharded CID has been correctly formed and stored. 25 | // This is a helper function for testing. It returns a map with all the blocks 26 | // from all shards. 27 | func VerifyShards(t *testing.T, rootCid api.Cid, pins MockPinStore, ipfs MockBlockStore, expectedShards int) (map[string]struct{}, error) { 28 | ctx := context.Background() 29 | metaPin, err := pins.PinGet(ctx, rootCid) 30 | if err != nil { 31 | return nil, fmt.Errorf("meta pin was not pinned: %s", err) 32 | } 33 | 34 | if api.PinType(metaPin.Type) != api.MetaType { 35 | return nil, fmt.Errorf("bad MetaPin type") 36 | } 37 | 38 | if metaPin.Reference == nil { 39 | return nil, errors.New("metaPin.Reference is unset") 40 | } 41 | 42 | clusterPin, err := pins.PinGet(ctx, *metaPin.Reference) 43 | if err != nil { 44 | return nil, fmt.Errorf("cluster pin was not pinned: %s", err) 45 | } 46 | if api.PinType(clusterPin.Type) != api.ClusterDAGType { 47 | return nil, fmt.Errorf("bad ClusterDAGPin type") 48 | } 49 | 50 | if !clusterPin.Reference.Equals(metaPin.Cid) { 51 | return nil, fmt.Errorf("clusterDAG should reference the MetaPin") 52 | } 53 | 54 | clusterDAGBlock, err := ipfs.BlockGet(ctx, clusterPin.Cid) 55 | if err != nil { 56 | return nil, fmt.Errorf("cluster pin was not stored: %s", err) 57 | } 58 | 59 | clusterDAGNode, err := CborDataToNode(clusterDAGBlock, "cbor") 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | shards := clusterDAGNode.Links() 65 | if len(shards) != expectedShards { 66 | return nil, fmt.Errorf("bad number of shards") 67 | } 68 | 69 | shardBlocks := make(map[string]struct{}) 70 | var ref api.Cid 71 | // traverse shards in order 72 | for i := 0; i < len(shards); i++ { 73 | sh, _, err := clusterDAGNode.ResolveLink([]string{fmt.Sprintf("%d", i)}) 74 | if err != nil { 75 | return nil, err 76 | } 77 | 78 | shardPin, err := pins.PinGet(ctx, api.NewCid(sh.Cid)) 79 | if err != nil { 80 | return nil, fmt.Errorf("shard was not pinned: %s %s", sh.Cid, err) 81 | } 82 | 83 | if ref != api.CidUndef && !shardPin.Reference.Equals(ref) { 84 | t.Errorf("Ref (%s) should point to previous shard (%s)", ref, shardPin.Reference) 85 | } 86 | ref = shardPin.Cid 87 | 88 | shardBlock, err := ipfs.BlockGet(ctx, shardPin.Cid) 89 | if err != nil { 90 | return nil, fmt.Errorf("shard block was not stored: %s", err) 91 | } 92 | shardNode, err := CborDataToNode(shardBlock, "cbor") 93 | if err != nil { 94 | return nil, err 95 | } 96 | for _, l := range shardNode.Links() { 97 | ci := l.Cid.String() 98 | _, ok := shardBlocks[ci] 99 | if ok { 100 | return nil, fmt.Errorf("block belongs to two shards: %s", ci) 101 | } 102 | shardBlocks[ci] = struct{}{} 103 | } 104 | } 105 | return shardBlocks, nil 106 | } 107 | -------------------------------------------------------------------------------- /allocator/balanced/config.go: -------------------------------------------------------------------------------- 1 | package balanced 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | 7 | "github.com/ipfs-cluster/ipfs-cluster/config" 8 | "github.com/kelseyhightower/envconfig" 9 | ) 10 | 11 | const configKey = "balanced" 12 | const envConfigKey = "cluster_balanced" 13 | 14 | // These are the default values for a Config. 15 | var ( 16 | DefaultAllocateBy = []string{"tag:group", "freespace"} 17 | ) 18 | 19 | // Config allows to initialize the Allocator. 20 | type Config struct { 21 | config.Saver 22 | 23 | AllocateBy []string 24 | } 25 | 26 | type jsonConfig struct { 27 | AllocateBy []string `json:"allocate_by"` 28 | } 29 | 30 | // ConfigKey returns a human-friendly identifier for this 31 | // Config's type. 32 | func (cfg *Config) ConfigKey() string { 33 | return configKey 34 | } 35 | 36 | // Default initializes this Config with sensible values. 37 | func (cfg *Config) Default() error { 38 | cfg.AllocateBy = DefaultAllocateBy 39 | return nil 40 | } 41 | 42 | // ApplyEnvVars fills in any Config fields found 43 | // as environment variables. 44 | func (cfg *Config) ApplyEnvVars() error { 45 | jcfg := cfg.toJSONConfig() 46 | 47 | err := envconfig.Process(envConfigKey, jcfg) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | return cfg.applyJSONConfig(jcfg) 53 | } 54 | 55 | // Validate checks that the fields of this configuration have 56 | // sensible values. 57 | func (cfg *Config) Validate() error { 58 | if len(cfg.AllocateBy) <= 0 { 59 | return errors.New("metricalloc.allocate_by is invalid") 60 | } 61 | 62 | return nil 63 | } 64 | 65 | // LoadJSON parses a raw JSON byte-slice as generated by ToJSON(). 66 | func (cfg *Config) LoadJSON(raw []byte) error { 67 | jcfg := &jsonConfig{} 68 | err := json.Unmarshal(raw, jcfg) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | cfg.Default() 74 | 75 | return cfg.applyJSONConfig(jcfg) 76 | } 77 | 78 | func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error { 79 | // When unset, leave default 80 | if len(jcfg.AllocateBy) > 0 { 81 | cfg.AllocateBy = jcfg.AllocateBy 82 | } 83 | 84 | return cfg.Validate() 85 | } 86 | 87 | // ToJSON generates a human-friendly JSON representation of this Config. 88 | func (cfg *Config) ToJSON() ([]byte, error) { 89 | jcfg := cfg.toJSONConfig() 90 | 91 | return config.DefaultJSONMarshal(jcfg) 92 | } 93 | 94 | func (cfg *Config) toJSONConfig() *jsonConfig { 95 | return &jsonConfig{ 96 | AllocateBy: cfg.AllocateBy, 97 | } 98 | } 99 | 100 | // ToDisplayJSON returns JSON config as a string. 101 | func (cfg *Config) ToDisplayJSON() ([]byte, error) { 102 | return config.DisplayJSON(cfg.toJSONConfig()) 103 | } 104 | -------------------------------------------------------------------------------- /allocator/balanced/config_test.go: -------------------------------------------------------------------------------- 1 | package balanced 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | var cfgJSON = []byte(` 9 | { 10 | "allocate_by": ["tag", "disk"] 11 | } 12 | `) 13 | 14 | func TestLoadJSON(t *testing.T) { 15 | cfg := &Config{} 16 | err := cfg.LoadJSON(cfgJSON) 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | } 21 | 22 | func TestToJSON(t *testing.T) { 23 | cfg := &Config{} 24 | cfg.LoadJSON(cfgJSON) 25 | newjson, err := cfg.ToJSON() 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | cfg = &Config{} 30 | err = cfg.LoadJSON(newjson) 31 | if err != nil { 32 | t.Fatal(err) 33 | } 34 | if len(cfg.AllocateBy) != 2 { 35 | t.Error("configuration was lost in serialization/deserialization") 36 | } 37 | } 38 | 39 | func TestDefault(t *testing.T) { 40 | cfg := &Config{} 41 | cfg.Default() 42 | if cfg.Validate() != nil { 43 | t.Fatal("error validating") 44 | } 45 | 46 | cfg.AllocateBy = nil 47 | if cfg.Validate() == nil { 48 | t.Fatal("expected error validating") 49 | } 50 | 51 | } 52 | 53 | func TestApplyEnvVars(t *testing.T) { 54 | os.Setenv("CLUSTER_BALANCED_ALLOCATEBY", "a,b,c") 55 | cfg := &Config{} 56 | cfg.ApplyEnvVars() 57 | 58 | if len(cfg.AllocateBy) != 3 { 59 | t.Fatal("failed to override allocate_by with env var") 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /api/add_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "net/url" 5 | "testing" 6 | ) 7 | 8 | func TestAddParams_FromQuery(t *testing.T) { 9 | qStr := "layout=balanced&chunker=size-262144&name=test&raw-leaves=true&hidden=true&shard=true&replication-min=2&replication-max=4&shard-size=1" 10 | 11 | q, err := url.ParseQuery(qStr) 12 | if err != nil { 13 | t.Fatal(err) 14 | } 15 | 16 | p, err := AddParamsFromQuery(q) 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | if p.Layout != "balanced" || 21 | p.Chunker != "size-262144" || 22 | p.Name != "test" || 23 | !p.RawLeaves || !p.Hidden || !p.Shard || 24 | p.ReplicationFactorMin != 2 || 25 | p.ReplicationFactorMax != 4 || 26 | p.ShardSize != 1 { 27 | t.Fatal("did not parse the query correctly") 28 | } 29 | } 30 | 31 | func TestAddParams_FromQueryRawLeaves(t *testing.T) { 32 | qStr := "cid-version=1" 33 | 34 | q, err := url.ParseQuery(qStr) 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | 39 | p, err := AddParamsFromQuery(q) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | if !p.RawLeaves { 44 | t.Error("RawLeaves should be true with cid-version=1") 45 | } 46 | 47 | qStr = "cid-version=1&raw-leaves=false" 48 | 49 | q, err = url.ParseQuery(qStr) 50 | if err != nil { 51 | t.Fatal(err) 52 | } 53 | 54 | p, err = AddParamsFromQuery(q) 55 | if err != nil { 56 | t.Fatal(err) 57 | } 58 | if p.RawLeaves { 59 | t.Error("RawLeaves should be false when explicitally set") 60 | } 61 | 62 | qStr = "cid-version=0&raw-leaves=true" 63 | 64 | q, err = url.ParseQuery(qStr) 65 | if err != nil { 66 | t.Fatal(err) 67 | } 68 | 69 | p, err = AddParamsFromQuery(q) 70 | if err != nil { 71 | t.Fatal(err) 72 | } 73 | if !p.RawLeaves { 74 | t.Error("RawLeaves should be true when explicitly set") 75 | } 76 | } 77 | 78 | func TestAddParams_ToQueryString(t *testing.T) { 79 | p := DefaultAddParams() 80 | p.ReplicationFactorMin = 3 81 | p.ReplicationFactorMax = 6 82 | p.Name = "something" 83 | p.RawLeaves = true 84 | p.ShardSize = 1020 85 | qstr, err := p.ToQueryString() 86 | if err != nil { 87 | t.Fatal(err) 88 | } 89 | q, err := url.ParseQuery(qstr) 90 | if err != nil { 91 | t.Fatal(err) 92 | } 93 | 94 | p2, err := AddParamsFromQuery(q) 95 | if err != nil { 96 | t.Fatal(err) 97 | } 98 | 99 | if !p.Equals(p2) { 100 | t.Error("generated and parsed params should be equal") 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /api/common/test/server.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID7TCCAtWgAwIBAgIJAMqpHdKRMzMLMA0GCSqGSIb3DQEBCwUAMIGCMQswCQYD 3 | VQQGEwJVUzERMA8GA1UECAwIQ29sb3JhZG8xDzANBgNVBAcMBmdvbGRlbjEMMAoG 4 | A1UECgwDQ1NNMREwDwYDVQQLDAhTZWN0b3IgNzEMMAoGA1UEAwwDQm9iMSAwHgYJ 5 | KoZIhvcNAQkBFhFtaW5pc3RlckBtb3N3Lm9yZzAeFw0xNzA3MjExNjA5NTlaFw0y 6 | NzA3MTkxNjA5NTlaMIGCMQswCQYDVQQGEwJVUzERMA8GA1UECAwIQ29sb3JhZG8x 7 | DzANBgNVBAcMBmdvbGRlbjEMMAoGA1UECgwDQ1NNMREwDwYDVQQLDAhTZWN0b3Ig 8 | NzEMMAoGA1UEAwwDQm9iMSAwHgYJKoZIhvcNAQkBFhFtaW5pc3RlckBtb3N3Lm9y 9 | ZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALuoP8PehGItmKPi3+8S 10 | IV1qz8C3FiK85X/INxYLjyuzvpmDROtlkOvdmPCJrveKDZF7ECQpwIGApFbnKCCW 11 | 3zdOPQmAVzm4N8bvnzFtM9mTm8qKb9SwRi6ZLZ/qXo98t8C7CV6FaNKUkIw0lUes 12 | ZiXEcmknrlPy3svaDQVoSOH8L38d0g4geqiNrMmZDaGe8FAYdpCoeYDIm/u0Ag9y 13 | G3+XAbETxWhkfTyH3XcQ/Izg0wG9zFY8y/fyYwC+C7+xF75x4gbIzHAY2iFS2ua7 14 | GTKa2GZhOXtMuzJ6cf+TZW460Z+O+PkA1aH01WrGL7iCW/6Cn9gPRKL+IP6iyDnh 15 | 9HMCAwEAAaNkMGIwDwYDVR0RBAgwBocEfwAAATAdBgNVHQ4EFgQU9mXv8mv/LlAa 16 | jwr8X9hzk52cBagwHwYDVR0jBBgwFoAU9mXv8mv/LlAajwr8X9hzk52cBagwDwYD 17 | VR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAIxqpKYzF6A9RlLso0lkF 18 | nYfcyeVAvi03IBdiTNnpOe6ROa4gNwKH/JUJMCRDPzm/x78+srCmrcCCAJJTcqgi 19 | b84vq3DegGPg2NXbn9qVUA1SdiXFelqMFwLitDn2KKizihEN4L5PEArHuDaNvLI+ 20 | kMr+yZSALWTdtfydj211c7hTBvFqO8l5MYDXCmfoS9sqniorlNHIaBim/SNfDsi6 21 | 8hAhvfRvk3e6dPjAPrIZYdQR5ROGewtD4F/anXgKY2BmBtWwd6gbGeMnnVi1SGRP 22 | 0UHc4O9aq9HrAOFL/72WVk/kyyPyJ/GtSaPYL1OFS12R/l0hNi+pER7xDtLOVHO2 23 | iw== 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /api/common/test/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpQIBAAKCAQEAu6g/w96EYi2Yo+Lf7xIhXWrPwLcWIrzlf8g3FguPK7O+mYNE 3 | 62WQ692Y8Imu94oNkXsQJCnAgYCkVucoIJbfN049CYBXObg3xu+fMW0z2ZObyopv 4 | 1LBGLpktn+pej3y3wLsJXoVo0pSQjDSVR6xmJcRyaSeuU/Ley9oNBWhI4fwvfx3S 5 | DiB6qI2syZkNoZ7wUBh2kKh5gMib+7QCD3Ibf5cBsRPFaGR9PIfddxD8jODTAb3M 6 | VjzL9/JjAL4Lv7EXvnHiBsjMcBjaIVLa5rsZMprYZmE5e0y7Mnpx/5NlbjrRn474 7 | +QDVofTVasYvuIJb/oKf2A9Eov4g/qLIOeH0cwIDAQABAoIBAAOYreArG45mIU7C 8 | wlfqmQkZSvH+kEYKKLvSMnwRrKTBxR1cDq4UPDrI/G1ftiK4Wpo3KZAH3NCejoe7 9 | 1mEJgy2kKjdMZl+M0ETXws1Hsn6w/YNcM9h3qGCsPtuZukY1ta/T5dIR7HhcsIh/ 10 | WX0OKMcAhNDPGeAx/2MYwrcf0IXELx0+eP1fuBllkajH14J8+ZkVrBMDhqppn8Iq 11 | f9poVNQliJtN7VkL6lJ60HwoVNGEhFaOYphn3CR/sCc6xl+/CzV4h6c5X/RIUfDs 12 | kjgl9mlPFuWq9S19Z+XVfLSE+sYd6LDrh0IZEx9s0OfOjucH2bUAuKNDnCq0wW70 13 | FzH6KoECgYEA4ZOcAMgujk8goL8nleNjuEq7d8pThAsuAy5vq9oyol8oe+p1pXHR 14 | SHP6wHyhXeTS5g1Ej+QV6f0v9gVFS2pFqTXymc9Gxald3trcnheodZXx63YbxHm2 15 | H7mYWyZvq05A0qRLmmqCoSRJHUOkH2wVqgj9KsVYP1anIhdykbycansCgYEA1Pdp 16 | uAfWt/GLZ7B0q3JPlVvusf97wBIUcoaxLHGKopvfsaFp0EY3NRxLSTaZ0NPOxTHh 17 | W6xaIlBmKllyt6q8W609A8hrXayV1yYnVE44b5UEMhVlfRFeEdf9Sp4YdQJ8r1J0 18 | QA89jHCjf8VocP5pSJz5tXvWHhmaotXBthFgWGkCgYEAiy7dwenCOBKAqk5n6Wb9 19 | X3fVBguzzjRrtpDPXHTsax1VyGeZIXUB0bemD2CW3G1U55dmJ3ZvQwnyrtT/tZGj 20 | 280qnFa1bz6aaegW2gD082CKfWNJrMgAZMDKTeuAWW2WN6Ih9+wiH7VY25Kh0LWL 21 | BHg5ZUuQsLwRscpP6bY7uMMCgYEAwY23hK2DJZyfEXcbIjL7R4jNMPM82nzUHp5x 22 | 6i2rTUyTitJj5Anc5SU4+2pnc5b9RtWltva22Jbvs6+mBm1jUYLqgESn5/QSHv8r 23 | IYER47+wl4BAw+GD+H2wVB/JpJbFEWbEBvCTBM/emSKmYIOo1njsrlfFa4fjtfjG 24 | XJ4ATXkCgYEAzeSrCCVrfPMLCmOijIYD1F7TMFthosW2JJie3bcHZMu2QEM8EIif 25 | YzkUvMaDAXJ4VniTHkDf3ubRoUi3DwLbvJIPnoOlx3jmzz6KYiEd+uXx40Yrebb0 26 | V9GB2S2q1RY7wsFoCqT/mq8usQkjr3ulYMJqeIWnCTWgajXWqAHH/Mw= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /api/ipfsproxy/util.go: -------------------------------------------------------------------------------- 1 | package ipfsproxy 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | // MultiError contains the results of multiple errors. 8 | type multiError struct { 9 | err strings.Builder 10 | } 11 | 12 | func (e *multiError) add(err string) { 13 | e.err.WriteString(err) 14 | e.err.WriteString("; ") 15 | } 16 | 17 | func (e *multiError) Error() string { 18 | return e.err.String() 19 | } 20 | -------------------------------------------------------------------------------- /api/pb/generate.go: -------------------------------------------------------------------------------- 1 | // Package pb provides protobuf definitions for serialized types in Cluster. 2 | //go:generate protoc -I=. --go_out=. types.proto 3 | package pb 4 | -------------------------------------------------------------------------------- /api/pb/types.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package api.pb; 3 | 4 | option go_package=".;pb"; 5 | 6 | message Pin { 7 | enum PinType { 8 | BadType = 0; // 1 << iota 9 | DataType = 1; // 2 << iota 10 | MetaType = 2; 11 | ClusterDAGType = 3; 12 | ShardType = 4; 13 | } 14 | 15 | bytes Cid = 1; 16 | PinType Type = 2; 17 | repeated bytes Allocations = 3; 18 | sint32 MaxDepth = 4; 19 | bytes Reference = 5; 20 | PinOptions Options = 6; 21 | uint64 Timestamp = 7; 22 | } 23 | 24 | message PinOptions { 25 | sint32 ReplicationFactorMin = 1; 26 | sint32 ReplicationFactorMax = 2; 27 | string Name = 3; 28 | uint64 ShardSize = 4; 29 | reserved 5; // reserved for UserAllocations 30 | map Metadata = 6 [deprecated = true]; 31 | bytes PinUpdate = 7; 32 | uint64 ExpireAt = 8; 33 | repeated bytes Origins = 9; 34 | repeated Metadata SortedMetadata = 10; 35 | } 36 | 37 | message Metadata { 38 | string Key = 1; 39 | string Value = 2; 40 | } -------------------------------------------------------------------------------- /api/rest/client/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - '1.9' 4 | - tip 5 | install: 6 | - go get golang.org/x/tools/cmd/cover 7 | - go get github.com/mattn/goveralls 8 | - make deps 9 | script: 10 | - make test 11 | - "$GOPATH/bin/goveralls -coverprofile=coverage.out -service=travis-ci -repotoken $COVERALLS_TOKEN" 12 | env: 13 | global: 14 | secure: Skjty77A/J/34pKFmHtxnpNejY2QAJw5PAacBnflo1yZfq4D2mEqVjyd0V2o/pSqm54b+eUouYp+9hNsBbVRHXlgi3PocVClBTV7McFMAoOn+OOEBrdt5wF57L0IPbt8yde+RpXcnCQ5rRvuSfCkEcTNhlxUdUjx4r9qhFsGWKvZVodcSO6xZTRwPYu7/MJWnJK/JV5CAWl7dWlWeAZhrASwXwS7662tu3SN9eor5+ZVF0t5BMhLP6juu6WPz9TFijQ/W4cRiXJ1REbg+M2RscAj9gOy7lIdKR5MEF1xj8naX2jtiZXcxIdV5cduLwSeBA8v5hahwV0H/1cN4Ypymix9vXfkZKyMbU7/TpO0pEzZOcoFne9edHRh6oUrCRBrf4veOiPbkObjmAs0HsdE1ZoeakgCQVHGqaMUlYW1ybeu04JJrXNAMC7s+RD9lxacwknrx333fSBmw+kQwJGmkYkdKcELo2toivrX+yXezISLf2+puqVPAZznY/OxHAuWDi047QLEBxW72ZuTCpT9QiOj3nl5chvmNV+edqgdLN3SlUNOB0jTOpyac/J1GicFkI7IgE2+PjeqpzVnrhZvpcAy4j8YLadGfISWVzbg4NaoUrBUIqA82rqwiZ1L+CcQKNW1h+vEXWp6cLnn2kcPSihM8RrsLuSiJMMgdIhMN3o= 15 | -------------------------------------------------------------------------------- /api/rest/client/README.md: -------------------------------------------------------------------------------- 1 | # ipfs-cluster client 2 | 3 | [![Made by](https://img.shields.io/badge/By-Protocol%20Labs-000000.svg?style=flat-square)](https://protocol.ai) 4 | [![Main project](https://img.shields.io/badge/project-ipfs--cluster-ef5c43.svg?style=flat-square)](http://github.com/ipfs-cluster) 5 | [![Discord](https://img.shields.io/badge/forum-discuss.ipfs.io-f9a035.svg?style=flat-square)](https://discuss.ipfs.io/c/help/help-ipfs-cluster/24) 6 | [![Matrix channel](https://img.shields.io/badge/matrix-%23ipfs--cluster-3c8da0.svg?style=flat-square)](https://app.element.io/#/room/#ipfs-cluster:ipfs.io) 7 | [![pkg.go.dev](https://pkg.go.dev/badge/github.com/ipfs-cluster/ipfs-cluster)](https://pkg.go.dev/github.com/ipfs-cluster/ipfs-cluster/api/rest/client) 8 | 9 | 10 | > Go client for the ipfs-cluster HTTP API. 11 | 12 | This is a Go client library to use the ipfs-cluster REST HTTP API. 13 | 14 | ## Table of Contents 15 | 16 | - [Install](#install) 17 | - [Usage](#usage) 18 | - [Contribute](#contribute) 19 | - [License](#license) 20 | 21 | ## Install 22 | 23 | You can import `github.com/ipfs-cluster/ipfs-cluster/api/rest/client` in your code. 24 | 25 | The code can be downloaded and tested with: 26 | 27 | ``` 28 | $ git clone https://github.com/ipfs-cluster/ipfs-cluster.git 29 | $ cd ipfs-cluster/ipfs-cluster/rest/api/client 30 | $ go test -v 31 | ``` 32 | 33 | ## Usage 34 | 35 | Documentation can be read at [pkg.go.dev](https://pkg.go.dev/github.com/ipfs-cluster/ipfs-cluster/api/rest/client). 36 | 37 | ## Contribute 38 | 39 | PRs accepted. 40 | 41 | ## License 42 | 43 | MIT © Protocol Labs 44 | -------------------------------------------------------------------------------- /api/rest/client/lbclient_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "testing" 8 | 9 | "github.com/ipfs-cluster/ipfs-cluster/api" 10 | ma "github.com/multiformats/go-multiaddr" 11 | ) 12 | 13 | func TestFailoverConcurrently(t *testing.T) { 14 | // Create a load balancing client with 5 empty clients and 5 clients with APIs 15 | // say we want to retry the request for at most 5 times 16 | cfgs := make([]*Config, 10) 17 | 18 | // 5 clients with an invalid api address 19 | for i := 0; i < 5; i++ { 20 | maddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0") 21 | cfgs[i] = &Config{ 22 | APIAddr: maddr, 23 | DisableKeepAlives: true, 24 | } 25 | } 26 | 27 | // 5 clients with APIs 28 | for i := 5; i < 10; i++ { 29 | cfgs[i] = &Config{ 30 | APIAddr: apiMAddr(testAPI(t)), 31 | DisableKeepAlives: true, 32 | } 33 | } 34 | 35 | // Run many requests at the same time 36 | 37 | // With Failover strategy, it would go through first 5 empty clients 38 | // and then 6th working client. Thus, all requests should always succeed. 39 | testRunManyRequestsConcurrently(t, cfgs, &Failover{}, 200, 6, true) 40 | // First 5 clients are empty. Thus, all requests should fail. 41 | testRunManyRequestsConcurrently(t, cfgs, &Failover{}, 200, 5, false) 42 | } 43 | 44 | type dummyClient struct { 45 | defaultClient 46 | i int 47 | } 48 | 49 | // ID returns dummy client's serial number. 50 | func (d *dummyClient) ID(ctx context.Context) (api.ID, error) { 51 | return api.ID{ 52 | Peername: fmt.Sprintf("%d", d.i), 53 | }, nil 54 | } 55 | 56 | func TestRoundRobin(t *testing.T) { 57 | var clients []Client 58 | // number of clients 59 | n := 5 60 | // create n dummy clients 61 | for i := 0; i < n; i++ { 62 | c := &dummyClient{ 63 | i: i, 64 | } 65 | clients = append(clients, c) 66 | } 67 | 68 | roundRobin := loadBalancingClient{ 69 | strategy: &RoundRobin{ 70 | clients: clients, 71 | length: uint32(len(clients)), 72 | }, 73 | } 74 | 75 | // clients should be used in the sequence 1, 2,.., 4, 0. 76 | for i := 0; i < n; i++ { 77 | id, _ := roundRobin.ID(context.Background()) 78 | if id.Peername != fmt.Sprintf("%d", (i+1)%n) { 79 | t.Errorf("clients are not being tried in sequence, expected client: %d, but found: %s", i, id.Peername) 80 | } 81 | } 82 | 83 | } 84 | 85 | func testRunManyRequestsConcurrently(t *testing.T, cfgs []*Config, strategy LBStrategy, requests int, retries int, pass bool) { 86 | c, err := NewLBClient(strategy, cfgs, retries) 87 | if err != nil { 88 | t.Fatal(err) 89 | } 90 | 91 | var wg sync.WaitGroup 92 | for i := 0; i < requests; i++ { 93 | wg.Add(1) 94 | go func() { 95 | defer wg.Done() 96 | ctx := context.Background() 97 | _, err := c.ID(ctx) 98 | if err != nil && pass { 99 | t.Error(err) 100 | } 101 | if err == nil && !pass { 102 | t.Error("request should fail with connection refusal") 103 | } 104 | }() 105 | } 106 | wg.Wait() 107 | } 108 | -------------------------------------------------------------------------------- /api/util.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | peer "github.com/libp2p/go-libp2p/core/peer" 5 | ) 6 | 7 | // PeersToStrings Encodes a list of peers. 8 | func PeersToStrings(peers []peer.ID) []string { 9 | strs := make([]string, len(peers)) 10 | for i, p := range peers { 11 | if p != "" { 12 | strs[i] = p.String() 13 | } 14 | } 15 | return strs 16 | } 17 | 18 | // StringsToPeers decodes peer.IDs from strings. 19 | func StringsToPeers(strs []string) []peer.ID { 20 | peers := []peer.ID{} 21 | for _, p := range strs { 22 | pid, err := peer.Decode(p) 23 | if err != nil { 24 | continue 25 | } 26 | peers = append(peers, pid) 27 | } 28 | return peers 29 | } 30 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-ctl/Makefile: -------------------------------------------------------------------------------- 1 | # go source files 2 | SRC := $(shell find ../.. -type f -name '*.go') 3 | GOPATH := $(shell go env GOPATH) 4 | GOFLAGS := "-trimpath" 5 | 6 | all: ipfs-cluster-ctl 7 | 8 | ipfs-cluster-ctl: $(SRC) 9 | go build $(GOFLAGS) -mod=readonly 10 | 11 | build: ipfs-cluster-ctl 12 | 13 | install: 14 | go install $(GOFLAGS) 15 | 16 | clean: 17 | rm -f ipfs-cluster-ctl 18 | 19 | .PHONY: clean install build 20 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-ctl/dist/LICENSE: -------------------------------------------------------------------------------- 1 | Dual-licensed under MIT and ASLv2, by way of the [Permissive License 2 | Stack](https://protocol.ai/blog/announcing-the-permissive-license-stack/). 3 | 4 | Apache-2.0: https://www.apache.org/licenses/license-2.0 5 | MIT: https://www.opensource.org/licenses/mit 6 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-ctl/dist/LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Copyright 2020. Protocol Labs, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-ctl/dist/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright 2020. Protocol Labs, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-ctl/dist/README.md: -------------------------------------------------------------------------------- 1 | # `ipfs-cluster-ctl` 2 | 3 | > IPFS cluster management tool 4 | 5 | `ipfs-cluster-ctl` is the client application to manage the cluster nodes and perform actions. `ipfs-cluster-ctl` uses the HTTP API provided by the nodes and it is completely separate from the cluster service. 6 | 7 | ### Usage 8 | 9 | Usage information can be obtained by running: 10 | 11 | ``` 12 | $ ipfs-cluster-ctl --help 13 | ``` 14 | 15 | You can also obtain command-specific help with `ipfs-cluster-ctl help [cmd]`. The (`--host`) can be used to talk to any remote cluster peer (`localhost` is used by default). 16 | 17 | For more information, please check the [Documentation](https://ipfscluster.io/documentation), in particular the [`ipfs-cluster-ctl` section](https://ipfscluster.io/documentation/ipfs-cluster-ctl). 18 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-follow/Makefile: -------------------------------------------------------------------------------- 1 | # go source files 2 | SRC := $(shell find ../.. -type f -name '*.go') 3 | GOPATH := $(shell go env GOPATH) 4 | GOFLAGS := "-trimpath" 5 | 6 | all: ipfs-cluster-follow 7 | 8 | ipfs-cluster-follow: $(SRC) 9 | go build $(GOFLAGS) -mod=readonly -ldflags "-X main.commit=$(shell git rev-parse HEAD)" 10 | 11 | build: ipfs-cluster-follow 12 | 13 | install: 14 | go install $(GOFLAGS) -ldflags "-X main.commit=$(shell git rev-parse HEAD)" 15 | 16 | clean: 17 | rm -f ipfs-cluster-follow 18 | 19 | .PHONY: clean install build 20 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-follow/dist/LICENSE: -------------------------------------------------------------------------------- 1 | Dual-licensed under MIT and ASLv2, by way of the [Permissive License 2 | Stack](https://protocol.ai/blog/announcing-the-permissive-license-stack/). 3 | 4 | Apache-2.0: https://www.apache.org/licenses/license-2.0 5 | MIT: https://www.opensource.org/licenses/mit 6 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-follow/dist/LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Copyright 2020. Protocol Labs, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-follow/dist/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright 2020. Protocol Labs, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-follow/dist/README.md: -------------------------------------------------------------------------------- 1 | # `ipfs-cluster-follow` 2 | 3 | > A tool to run IPFS Cluster follower peers 4 | 5 | `ipfs-cluster-follow` allows to setup and run IPFS Cluster follower peers. 6 | 7 | Follower peers can join collaborative clusters to track content in the 8 | cluster. Follower peers do not have permissions to modify the cluster pinset 9 | or access endpoints from other follower peers. 10 | 11 | `ipfs-cluster-follow` allows to run several peers at the same time (each 12 | joining a different cluster) and it is intended to be a very easy to use 13 | application with a minimal feature set. In order to run a fully-featured peer 14 | (follower or not), use `ipfs-cluster-service`. 15 | 16 | ### Usage 17 | 18 | The `ipfs-cluster-follow` command is always followed by the cluster name 19 | that we wish to work with. Full usage information can be obtained by running: 20 | 21 | ``` 22 | $ ipfs-cluster-follow --help 23 | $ ipfs-cluster-follow --help 24 | $ ipfs-cluster-follow --help 25 | $ ipfs-cluster-follow info --help 26 | $ ipfs-cluster-follow init --help 27 | $ ipfs-cluster-follow run --help 28 | $ ipfs-cluster-follow list --help 29 | ``` 30 | 31 | For more information, please check the [Documentation](https://ipfscluster.io/documentation), in particular the [`ipfs-cluster-follow` section](https://ipfscluster.io/documentation/ipfs-cluster-follow). 32 | 33 | 34 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-service/Makefile: -------------------------------------------------------------------------------- 1 | # go source files 2 | SRC := $(shell find ../.. -type f -name '*.go') 3 | GOPATH := $(shell go env GOPATH) 4 | GOFLAGS := "-trimpath" 5 | 6 | all: ipfs-cluster-service 7 | 8 | ipfs-cluster-service: $(SRC) 9 | go build $(GOFLAGS) -mod=readonly -ldflags "-X main.commit=$(shell git rev-parse HEAD)" 10 | 11 | build: ipfs-cluster-service 12 | 13 | install: 14 | go install $(GOFLAGS) -ldflags "-X main.commit=$(shell git rev-parse HEAD)" 15 | 16 | clean: 17 | rm -f ipfs-cluster-service 18 | 19 | .PHONY: clean install build 20 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-service/dist/LICENSE: -------------------------------------------------------------------------------- 1 | Dual-licensed under MIT and ASLv2, by way of the [Permissive License 2 | Stack](https://protocol.ai/blog/announcing-the-permissive-license-stack/). 3 | 4 | Apache-2.0: https://www.apache.org/licenses/license-2.0 5 | MIT: https://www.opensource.org/licenses/mit 6 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-service/dist/LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Copyright 2020. Protocol Labs, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-service/dist/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright 2020. Protocol Labs, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-service/dist/README.md: -------------------------------------------------------------------------------- 1 | # `ipfs-cluster-service` 2 | 3 | > The IPFS cluster peer daemon 4 | 5 | `ipfs-cluster-service` runs a full IPFS Cluster peer. 6 | 7 | ### Usage 8 | 9 | Usage information can be obtained with: 10 | 11 | ``` 12 | $ ipfs-cluster-service --help 13 | ``` 14 | 15 | For more information, please check the [Documentation](https://ipfscluster.io/documentation), in particular the [`ipfs-cluster-service` section](https://ipfscluster.io/documentation/ipfs-cluster-service). 16 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-service/export.json: -------------------------------------------------------------------------------- 1 | {"replication_factor_min":-1,"replication_factor_max":-1,"name":"","mode":"direct","shard_size":0,"user_allocations":null,"expire_at":"0001-01-01T00:00:00Z","metadata":null,"pin_update":null,"cid":{"/":"QmUaFyXjZUNaUwYF8rBtbJc7fEJ46aJXvgV8z2HHs6jvmJ"},"type":2,"allocations":[],"max_depth":0,"reference":null} 2 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-service/lock.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "path" 8 | 9 | fslock "github.com/ipfs/go-fs-lock" 10 | "github.com/ipfs-cluster/ipfs-cluster/cmdutils" 11 | ) 12 | 13 | // lock logic heavily inspired by go-ipfs/repo/fsrepo/lock/lock.go 14 | 15 | // The name of the file used for locking 16 | const lockFileName = "cluster.lock" 17 | 18 | var locker *lock 19 | 20 | // lock helps to coordinate proceeds via a lock file 21 | type lock struct { 22 | lockCloser io.Closer 23 | path string 24 | } 25 | 26 | func (l *lock) lock() { 27 | if l.lockCloser != nil { 28 | checkErr("", errors.New("cannot acquire lock twice")) 29 | } 30 | 31 | // we should have a config folder whenever we try to lock 32 | cfgHelper := cmdutils.NewConfigHelper(configPath, identityPath, "", "") 33 | cfgHelper.MakeConfigFolder() 34 | 35 | // set the lock file within this function 36 | logger.Debug("checking lock") 37 | lk, err := fslock.Lock(l.path, lockFileName) 38 | if err != nil { 39 | logger.Debug(err) 40 | l.lockCloser = nil 41 | errStr := "%s. If no other " 42 | errStr += "%s process is running, remove %s, or make sure " 43 | errStr += "that the config folder is writable for the user " 44 | errStr += "running %s." 45 | errStr = fmt.Sprintf( 46 | errStr, 47 | err, 48 | programName, 49 | path.Join(l.path, lockFileName), 50 | programName, 51 | ) 52 | checkErr("obtaining execution lock", errors.New(errStr)) 53 | } 54 | logger.Debugf("%s execution lock acquired", programName) 55 | l.lockCloser = lk 56 | } 57 | 58 | func (l *lock) tryUnlock() error { 59 | // Noop in the uninitialized case 60 | if l.lockCloser == nil { 61 | logger.Debug("locking not initialized, unlock is noop") 62 | return nil 63 | } 64 | err := l.lockCloser.Close() 65 | if err != nil { 66 | return err 67 | } 68 | logger.Debug("successfully released execution lock") 69 | l.lockCloser = nil 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /cmd/ipfs-cluster-service/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ipfs-cluster/ipfs-cluster/cmdutils" 7 | 8 | ma "github.com/multiformats/go-multiaddr" 9 | ) 10 | 11 | func TestRandomPorts(t *testing.T) { 12 | port := "9096" 13 | m1, _ := ma.NewMultiaddr("/ip4/0.0.0.0/tcp/9096") 14 | m2, _ := ma.NewMultiaddr("/ip6/::/udp/9096") 15 | 16 | addresses, err := cmdutils.RandomizePorts([]ma.Multiaddr{m1, m2}) 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | 21 | v1, err := addresses[0].ValueForProtocol(ma.P_TCP) 22 | if err != nil { 23 | t.Fatal(addresses[0], err) 24 | } 25 | 26 | v2, err := addresses[1].ValueForProtocol(ma.P_UDP) 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | 31 | if v1 == port { 32 | t.Error("expected different ipv4 ports") 33 | } 34 | 35 | if v2 == port { 36 | t.Error("expected different ipv6 ports") 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /connect_graph.go: -------------------------------------------------------------------------------- 1 | package ipfscluster 2 | 3 | import ( 4 | "github.com/ipfs-cluster/ipfs-cluster/api" 5 | 6 | peer "github.com/libp2p/go-libp2p/core/peer" 7 | 8 | "go.opencensus.io/trace" 9 | ) 10 | 11 | // ConnectGraph returns a description of which cluster peers and ipfs 12 | // daemons are connected to each other. 13 | func (c *Cluster) ConnectGraph() (api.ConnectGraph, error) { 14 | ctx, span := trace.StartSpan(c.ctx, "cluster/ConnectGraph") 15 | defer span.End() 16 | 17 | cg := api.ConnectGraph{ 18 | ClusterID: c.host.ID(), 19 | IDtoPeername: make(map[string]string), 20 | IPFSLinks: make(map[string][]peer.ID), 21 | ClusterLinks: make(map[string][]peer.ID), 22 | ClusterTrustLinks: make(map[string]bool), 23 | ClustertoIPFS: make(map[string]peer.ID), 24 | } 25 | members, err := c.consensus.Peers(ctx) 26 | if err != nil { 27 | return cg, err 28 | } 29 | 30 | for _, member := range members { 31 | // one of the entries is for itself, but that shouldn't hurt 32 | cg.ClusterTrustLinks[member.String()] = c.consensus.IsTrustedPeer(ctx, member) 33 | } 34 | 35 | peers := make([][]api.ID, len(members)) 36 | errs := make([]error, len(members)) 37 | 38 | for i, member := range members { 39 | in := make(chan struct{}) 40 | close(in) 41 | out := make(chan api.ID, 1024) 42 | errCh := make(chan error, 1) 43 | go func(i int) { 44 | defer close(errCh) 45 | 46 | errCh <- c.rpcClient.Stream( 47 | ctx, 48 | member, 49 | "Cluster", 50 | "Peers", 51 | in, 52 | out, 53 | ) 54 | }(i) 55 | var ids []api.ID 56 | for id := range out { 57 | ids = append(ids, id) 58 | } 59 | peers[i] = ids 60 | errs[i] = <-errCh 61 | } 62 | 63 | for i, err := range errs { 64 | p := members[i].String() 65 | cg.ClusterLinks[p] = make([]peer.ID, 0) 66 | if err != nil { // Only setting cluster connections when no error occurs 67 | logger.Debugf("RPC error reaching cluster peer %s: %s", p, err.Error()) 68 | continue 69 | } 70 | 71 | selfConnection, pID := c.recordClusterLinks(&cg, p, peers[i]) 72 | cg.IDtoPeername[p] = pID.Peername 73 | // IPFS connections 74 | if !selfConnection { 75 | logger.Warnf("cluster peer %s not its own peer. No ipfs info ", p) 76 | continue 77 | } 78 | c.recordIPFSLinks(&cg, pID) 79 | } 80 | 81 | return cg, nil 82 | } 83 | 84 | func (c *Cluster) recordClusterLinks(cg *api.ConnectGraph, p string, peers []api.ID) (bool, api.ID) { 85 | selfConnection := false 86 | var pID api.ID 87 | for _, id := range peers { 88 | if id.Error != "" { 89 | logger.Debugf("Peer %s errored connecting to its peer %s", p, id.ID) 90 | continue 91 | } 92 | if id.ID.String() == p { 93 | selfConnection = true 94 | pID = id 95 | } else { 96 | cg.ClusterLinks[p] = append(cg.ClusterLinks[p], id.ID) 97 | } 98 | } 99 | return selfConnection, pID 100 | } 101 | 102 | func (c *Cluster) recordIPFSLinks(cg *api.ConnectGraph, pID api.ID) { 103 | ipfsID := pID.IPFS.ID 104 | if pID.IPFS.Error != "" { // Only setting ipfs connections when no error occurs 105 | logger.Warnf("ipfs id: %s has error: %s. Skipping swarm connections", ipfsID, pID.IPFS.Error) 106 | return 107 | } 108 | 109 | pid := pID.ID.String() 110 | ipfsPid := ipfsID.String() 111 | 112 | if _, ok := cg.IPFSLinks[pid]; ok { 113 | logger.Warnf("ipfs id: %s already recorded, one ipfs daemon in use by multiple cluster peers", ipfsID) 114 | } 115 | cg.ClustertoIPFS[pid] = ipfsID 116 | cg.IPFSLinks[ipfsPid] = make([]peer.ID, 0) 117 | var swarmPeers []peer.ID 118 | err := c.rpcClient.Call( 119 | pID.ID, 120 | "IPFSConnector", 121 | "SwarmPeers", 122 | struct{}{}, 123 | &swarmPeers, 124 | ) 125 | if err != nil { 126 | return 127 | } 128 | cg.IPFSLinks[ipfsPid] = swarmPeers 129 | } 130 | -------------------------------------------------------------------------------- /consensus/raft/config_test.go: -------------------------------------------------------------------------------- 1 | package raft 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | 8 | hraft "github.com/hashicorp/raft" 9 | ) 10 | 11 | var cfgJSON = []byte(` 12 | { 13 | "init_peerset": [], 14 | "wait_for_leader_timeout": "15s", 15 | "network_timeout": "1s", 16 | "commit_retries": 1, 17 | "commit_retry_delay": "200ms", 18 | "backups_rotate": 5, 19 | "heartbeat_timeout": "1s", 20 | "election_timeout": "1s", 21 | "commit_timeout": "50ms", 22 | "max_append_entries": 64, 23 | "trailing_logs": 10240, 24 | "snapshot_interval": "2m0s", 25 | "snapshot_threshold": 8192, 26 | "leader_lease_timeout": "500ms" 27 | } 28 | `) 29 | 30 | func TestLoadJSON(t *testing.T) { 31 | cfg := &Config{} 32 | err := cfg.LoadJSON(cfgJSON) 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | 37 | j := &jsonConfig{} 38 | json.Unmarshal(cfgJSON, j) 39 | j.HeartbeatTimeout = "1us" 40 | tst, _ := json.Marshal(j) 41 | err = cfg.LoadJSON(tst) 42 | if err == nil { 43 | t.Error("expected error decoding heartbeat_timeout") 44 | } 45 | 46 | json.Unmarshal(cfgJSON, j) 47 | j.LeaderLeaseTimeout = "abc" 48 | tst, _ = json.Marshal(j) 49 | err = cfg.LoadJSON(tst) 50 | if err != nil { 51 | t.Fatal(err) 52 | } 53 | def := hraft.DefaultConfig() 54 | if cfg.RaftConfig.LeaderLeaseTimeout != def.LeaderLeaseTimeout { 55 | t.Error("expected default leader lease") 56 | } 57 | } 58 | 59 | func TestToJSON(t *testing.T) { 60 | cfg := &Config{} 61 | cfg.LoadJSON(cfgJSON) 62 | newjson, err := cfg.ToJSON() 63 | if err != nil { 64 | t.Fatal(err) 65 | } 66 | cfg = &Config{} 67 | err = cfg.LoadJSON(newjson) 68 | if err != nil { 69 | t.Fatal(err) 70 | } 71 | } 72 | 73 | func TestDefault(t *testing.T) { 74 | cfg := &Config{} 75 | cfg.Default() 76 | if cfg.Validate() != nil { 77 | t.Fatal("error validating") 78 | } 79 | 80 | cfg.RaftConfig.HeartbeatTimeout = 0 81 | if cfg.Validate() == nil { 82 | t.Fatal("expected error validating") 83 | } 84 | 85 | cfg.Default() 86 | cfg.RaftConfig = nil 87 | if cfg.Validate() == nil { 88 | t.Fatal("expected error validating") 89 | } 90 | 91 | cfg.Default() 92 | cfg.CommitRetries = -1 93 | if cfg.Validate() == nil { 94 | t.Fatal("expected error validating") 95 | } 96 | 97 | cfg.Default() 98 | cfg.WaitForLeaderTimeout = 0 99 | if cfg.Validate() == nil { 100 | t.Fatal("expected error validating") 101 | } 102 | 103 | cfg.Default() 104 | cfg.BackupsRotate = 0 105 | 106 | if cfg.Validate() == nil { 107 | t.Fatal("expected error validating") 108 | } 109 | } 110 | 111 | func TestApplyEnvVars(t *testing.T) { 112 | os.Setenv("CLUSTER_RAFT_COMMITRETRIES", "300") 113 | cfg := &Config{} 114 | cfg.Default() 115 | cfg.ApplyEnvVars() 116 | 117 | if cfg.CommitRetries != 300 { 118 | t.Fatal("failed to override commit_retries with env var") 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /consensus/raft/data_helper.go: -------------------------------------------------------------------------------- 1 | package raft 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | ) 8 | 9 | // dataBackupHelper helps making and rotating backups from a folder. 10 | // it will name them .old.0, .old.1... and so on. 11 | // when a new backup is made, the old.0 is renamed to old.1 and so on. 12 | // when the "keep" number is reached, the oldest is always 13 | // discarded. 14 | type dataBackupHelper struct { 15 | baseDir string 16 | folderName string 17 | keep int 18 | } 19 | 20 | func newDataBackupHelper(dataFolder string, keep int) *dataBackupHelper { 21 | return &dataBackupHelper{ 22 | baseDir: filepath.Dir(dataFolder), 23 | folderName: filepath.Base(dataFolder), 24 | keep: keep, 25 | } 26 | } 27 | 28 | func (dbh *dataBackupHelper) makeName(i int) string { 29 | return filepath.Join(dbh.baseDir, fmt.Sprintf("%s.old.%d", dbh.folderName, i)) 30 | } 31 | 32 | func (dbh *dataBackupHelper) listBackups() []string { 33 | backups := []string{} 34 | for i := 0; i < dbh.keep; i++ { 35 | name := dbh.makeName(i) 36 | if _, err := os.Stat(name); os.IsNotExist(err) { 37 | return backups 38 | } 39 | backups = append(backups, name) 40 | } 41 | return backups 42 | } 43 | 44 | func (dbh *dataBackupHelper) makeBackup() error { 45 | folder := filepath.Join(dbh.baseDir, dbh.folderName) 46 | if _, err := os.Stat(folder); os.IsNotExist(err) { 47 | // nothing to backup 48 | logger.Debug("nothing to backup") 49 | return nil 50 | } 51 | 52 | // make sure config folder exists 53 | err := os.MkdirAll(dbh.baseDir, 0700) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | // list all backups in it 59 | backups := dbh.listBackups() 60 | // remove last / oldest. Ex. if max is five, remove name.old.4 61 | if len(backups) >= dbh.keep { 62 | os.RemoveAll(backups[len(backups)-1]) 63 | } else { // append new backup folder. Ex, if 2 exist: add name.old.2 64 | backups = append(backups, dbh.makeName(len(backups))) 65 | } 66 | 67 | // increase number for all backups folders. 68 | // If there are 3: 1->2, 0->1. 69 | // Note in all cases the last backup in the list does not exist 70 | // (either removed or not created, just added to this list) 71 | for i := len(backups) - 1; i > 0; i-- { 72 | err := os.Rename(backups[i-1], backups[i]) 73 | if err != nil { 74 | return err 75 | } 76 | } 77 | 78 | // save new as name.old.0 79 | return os.Rename(filepath.Join(dbh.baseDir, dbh.folderName), dbh.makeName(0)) 80 | } 81 | -------------------------------------------------------------------------------- /consensus/raft/data_helper_test.go: -------------------------------------------------------------------------------- 1 | package raft 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func TestDataBackupHelper(t *testing.T) { 10 | keep := 5 11 | 12 | cleanup := func() { 13 | os.RemoveAll("data_helper_testing") 14 | for i := 0; i < 2*keep; i++ { 15 | os.RemoveAll(fmt.Sprintf("data_helper_testing.old.%d", i)) 16 | } 17 | } 18 | cleanup() 19 | defer cleanup() 20 | 21 | os.MkdirAll("data_helper_testing", 0700) 22 | helper := newDataBackupHelper("data_helper_testing", keep) 23 | for i := 0; i < 2*keep; i++ { 24 | err := helper.makeBackup() 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | backups := helper.listBackups() 29 | if (i < keep && len(backups) != i+1) || 30 | (i >= keep && len(backups) != keep) { 31 | t.Fatal("incorrect number of backups saved") 32 | } 33 | os.MkdirAll("data_helper_testing", 0700) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /consensus/raft/log_op.go: -------------------------------------------------------------------------------- 1 | package raft 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "go.opencensus.io/tag" 8 | "go.opencensus.io/trace" 9 | 10 | "github.com/ipfs-cluster/ipfs-cluster/api" 11 | "github.com/ipfs-cluster/ipfs-cluster/state" 12 | 13 | consensus "github.com/libp2p/go-libp2p-consensus" 14 | ) 15 | 16 | // Type of consensus operation 17 | const ( 18 | LogOpPin = iota + 1 19 | LogOpUnpin 20 | ) 21 | 22 | // LogOpType expresses the type of a consensus Operation 23 | type LogOpType int 24 | 25 | // LogOp represents an operation for the OpLogConsensus system. 26 | // It implements the consensus.Op interface and it is used by the 27 | // Consensus component. 28 | type LogOp struct { 29 | SpanCtx trace.SpanContext `codec:"s,omitempty"` 30 | TagCtx []byte `codec:"t,omitempty"` 31 | Cid api.Pin `codec:"c,omitempty"` 32 | Type LogOpType `codec:"p,omitempty"` 33 | consensus *Consensus `codec:"-"` 34 | tracing bool `codec:"-"` 35 | } 36 | 37 | // ApplyTo applies the operation to the State 38 | func (op *LogOp) ApplyTo(cstate consensus.State) (consensus.State, error) { 39 | var err error 40 | ctx := context.Background() 41 | if op.tracing { 42 | tagmap, err := tag.Decode(op.TagCtx) 43 | if err != nil { 44 | logger.Error(err) 45 | } 46 | ctx = tag.NewContext(ctx, tagmap) 47 | var span *trace.Span 48 | ctx, span = trace.StartSpanWithRemoteParent(ctx, "consensus/raft/logop/ApplyTo", op.SpanCtx) 49 | defer span.End() 50 | } 51 | 52 | state, ok := cstate.(state.State) 53 | if !ok { 54 | // Should never be here 55 | panic("received unexpected state type") 56 | } 57 | 58 | pin := op.Cid 59 | 60 | switch op.Type { 61 | case LogOpPin: 62 | err = state.Add(ctx, pin) 63 | if err != nil { 64 | logger.Error(err) 65 | goto ROLLBACK 66 | } 67 | // Async, we let the PinTracker take care of any problems 68 | op.consensus.rpcClient.GoContext( 69 | ctx, 70 | "", 71 | "PinTracker", 72 | "Track", 73 | pin, 74 | &struct{}{}, 75 | nil, 76 | ) 77 | case LogOpUnpin: 78 | err = state.Rm(ctx, pin.Cid) 79 | if err != nil { 80 | logger.Error(err) 81 | goto ROLLBACK 82 | } 83 | // Async, we let the PinTracker take care of any problems 84 | op.consensus.rpcClient.GoContext( 85 | ctx, 86 | "", 87 | "PinTracker", 88 | "Untrack", 89 | pin, 90 | &struct{}{}, 91 | nil, 92 | ) 93 | default: 94 | logger.Error("unknown LogOp type. Ignoring") 95 | } 96 | return state, nil 97 | 98 | ROLLBACK: 99 | // We failed to apply the operation to the state 100 | // and therefore we need to request a rollback to the 101 | // cluster to the previous state. This operation can only be performed 102 | // by the cluster leader. 103 | logger.Error("Rollbacks are not implemented") 104 | return nil, errors.New("a rollback may be necessary. Reason: " + err.Error()) 105 | } 106 | -------------------------------------------------------------------------------- /consensus/raft/log_op_test.go: -------------------------------------------------------------------------------- 1 | package raft 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/ipfs-cluster/ipfs-cluster/api" 8 | "github.com/ipfs-cluster/ipfs-cluster/datastore/inmem" 9 | "github.com/ipfs-cluster/ipfs-cluster/state/dsstate" 10 | "github.com/ipfs-cluster/ipfs-cluster/test" 11 | ) 12 | 13 | func TestApplyToPin(t *testing.T) { 14 | ctx := context.Background() 15 | cc := testingConsensus(t, 1) 16 | op := &LogOp{ 17 | Cid: api.PinCid(test.Cid1), 18 | Type: LogOpPin, 19 | consensus: cc, 20 | } 21 | defer cleanRaft(1) 22 | defer cc.Shutdown(ctx) 23 | 24 | st, err := dsstate.New(ctx, inmem.New(), "", dsstate.DefaultHandle()) 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | op.ApplyTo(st) 29 | 30 | out := make(chan api.Pin, 100) 31 | err = st.List(ctx, out) 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | 36 | var pins []api.Pin 37 | for p := range out { 38 | pins = append(pins, p) 39 | } 40 | 41 | if len(pins) != 1 || !pins[0].Cid.Equals(test.Cid1) { 42 | t.Error("the state was not modified correctly") 43 | } 44 | } 45 | 46 | func TestApplyToUnpin(t *testing.T) { 47 | ctx := context.Background() 48 | cc := testingConsensus(t, 1) 49 | op := &LogOp{ 50 | Cid: api.PinCid(test.Cid1), 51 | Type: LogOpUnpin, 52 | consensus: cc, 53 | } 54 | defer cleanRaft(1) 55 | defer cc.Shutdown(ctx) 56 | 57 | st, err := dsstate.New(ctx, inmem.New(), "", dsstate.DefaultHandle()) 58 | if err != nil { 59 | t.Fatal(err) 60 | } 61 | st.Add(ctx, testPin(test.Cid1)) 62 | op.ApplyTo(st) 63 | 64 | out := make(chan api.Pin, 100) 65 | err = st.List(ctx, out) 66 | if err != nil { 67 | t.Fatal(err) 68 | } 69 | if len(out) != 0 { 70 | t.Error("the state was not modified correctly") 71 | } 72 | } 73 | 74 | func TestApplyToBadState(t *testing.T) { 75 | defer func() { 76 | if r := recover(); r == nil { 77 | t.Error("should have recovered an error") 78 | } 79 | }() 80 | 81 | op := &LogOp{ 82 | Cid: api.PinCid(test.Cid1), 83 | Type: LogOpUnpin, 84 | } 85 | 86 | var st interface{} 87 | op.ApplyTo(st) 88 | } 89 | -------------------------------------------------------------------------------- /datastore/badger/badger.go: -------------------------------------------------------------------------------- 1 | // Package badger provides a configurable BadgerDB go-datastore for use with 2 | // IPFS Cluster. 3 | package badger 4 | 5 | import ( 6 | "os" 7 | 8 | ds "github.com/ipfs/go-datastore" 9 | badgerds "github.com/ipfs/go-ds-badger" 10 | logging "github.com/ipfs/go-log/v2" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | var logger = logging.Logger("badger") 15 | 16 | // New returns a BadgerDB datastore configured with the given 17 | // configuration. 18 | func New(cfg *Config) (ds.Datastore, error) { 19 | folder := cfg.GetFolder() 20 | err := os.MkdirAll(folder, 0700) 21 | if err != nil { 22 | return nil, errors.Wrap(err, "creating badger folder") 23 | } 24 | opts := badgerds.Options{ 25 | GcDiscardRatio: cfg.GCDiscardRatio, 26 | GcInterval: cfg.GCInterval, 27 | GcSleep: cfg.GCSleep, 28 | Options: cfg.BadgerOptions, 29 | } 30 | return badgerds.NewDatastore(folder, &opts) 31 | } 32 | 33 | // Cleanup deletes the badger datastore. 34 | func Cleanup(cfg *Config) error { 35 | folder := cfg.GetFolder() 36 | if _, err := os.Stat(folder); os.IsNotExist(err) { 37 | return nil 38 | } 39 | return os.RemoveAll(cfg.GetFolder()) 40 | 41 | } 42 | -------------------------------------------------------------------------------- /datastore/badger/config_test.go: -------------------------------------------------------------------------------- 1 | package badger 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/dgraph-io/badger" 8 | "github.com/dgraph-io/badger/options" 9 | ) 10 | 11 | var cfgJSON = []byte(` 12 | { 13 | "folder": "test", 14 | "gc_discard_ratio": 0.1, 15 | "gc_sleep": "2m", 16 | "badger_options": { 17 | "max_levels": 4, 18 | "value_log_loading_mode": 0 19 | } 20 | } 21 | `) 22 | 23 | func TestLoadJSON(t *testing.T) { 24 | cfg := &Config{} 25 | err := cfg.LoadJSON(cfgJSON) 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | } 30 | 31 | func TestToJSON(t *testing.T) { 32 | cfg := &Config{} 33 | cfg.LoadJSON(cfgJSON) 34 | 35 | if cfg.GCDiscardRatio != 0.1 { 36 | t.Fatal("GCDiscardRatio should be 0.1") 37 | } 38 | 39 | if cfg.GCInterval != DefaultGCInterval { 40 | t.Fatal("GCInterval should default as it is unset") 41 | } 42 | 43 | if cfg.GCSleep != 2*time.Minute { 44 | t.Fatal("GCSleep should be 2m") 45 | } 46 | 47 | if cfg.BadgerOptions.ValueLogLoadingMode != options.FileIO { 48 | t.Fatalf("got: %d, want: %d", cfg.BadgerOptions.ValueLogLoadingMode, options.FileIO) 49 | } 50 | 51 | if cfg.BadgerOptions.ValueLogFileSize != badger.DefaultOptions("").ValueLogFileSize { 52 | t.Fatalf( 53 | "got: %d, want: %d", 54 | cfg.BadgerOptions.ValueLogFileSize, 55 | badger.DefaultOptions("").ValueLogFileSize, 56 | ) 57 | } 58 | 59 | if cfg.BadgerOptions.TableLoadingMode != badger.DefaultOptions("").TableLoadingMode { 60 | t.Fatalf("TableLoadingMode is not nil: got: %v, want: %v", cfg.BadgerOptions.TableLoadingMode, badger.DefaultOptions("").TableLoadingMode) 61 | } 62 | 63 | if cfg.BadgerOptions.MaxLevels != 4 { 64 | t.Fatalf("MaxLevels should be 4, got: %d", cfg.BadgerOptions.MaxLevels) 65 | } 66 | 67 | newjson, err := cfg.ToJSON() 68 | if err != nil { 69 | t.Fatal(err) 70 | } 71 | 72 | cfg = &Config{} 73 | err = cfg.LoadJSON(newjson) 74 | if err != nil { 75 | t.Fatal(err) 76 | } 77 | } 78 | 79 | func TestDefault(t *testing.T) { 80 | cfg := &Config{} 81 | cfg.Default() 82 | if cfg.Validate() != nil { 83 | t.Fatal("error validating") 84 | } 85 | 86 | cfg.GCDiscardRatio = 0 87 | if cfg.Validate() == nil { 88 | t.Fatal("expected error validating") 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /datastore/badger3/badger.go: -------------------------------------------------------------------------------- 1 | // Package badger3 provides a configurable BadgerDB v3 go-datastore for use with 2 | // IPFS Cluster. 3 | package badger3 4 | 5 | import ( 6 | "os" 7 | 8 | ds "github.com/ipfs/go-datastore" 9 | badgerds "github.com/ipfs/go-ds-badger3" 10 | logging "github.com/ipfs/go-log/v2" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | var logger = logging.Logger("badger3") 15 | 16 | // New returns a BadgerDB datastore configured with the given 17 | // configuration. 18 | func New(cfg *Config) (ds.Datastore, error) { 19 | folder := cfg.GetFolder() 20 | err := os.MkdirAll(folder, 0700) 21 | if err != nil { 22 | return nil, errors.Wrap(err, "creating badger folder") 23 | } 24 | opts := badgerds.Options{ 25 | GcDiscardRatio: cfg.GCDiscardRatio, 26 | GcInterval: cfg.GCInterval, 27 | GcSleep: cfg.GCSleep, 28 | Options: cfg.BadgerOptions, 29 | } 30 | return badgerds.NewDatastore(folder, &opts) 31 | } 32 | 33 | // Cleanup deletes the badger datastore. 34 | func Cleanup(cfg *Config) error { 35 | folder := cfg.GetFolder() 36 | if _, err := os.Stat(folder); os.IsNotExist(err) { 37 | return nil 38 | } 39 | return os.RemoveAll(cfg.GetFolder()) 40 | } 41 | -------------------------------------------------------------------------------- /datastore/badger3/config_test.go: -------------------------------------------------------------------------------- 1 | package badger3 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/dgraph-io/badger/v3" 8 | "github.com/dgraph-io/badger/v3/options" 9 | ) 10 | 11 | var cfgJSON = []byte(` 12 | { 13 | "folder": "test", 14 | "gc_discard_ratio": 0.1, 15 | "gc_sleep": "2m", 16 | "badger_options": { 17 | "max_levels": 4, 18 | "compression": 2 19 | } 20 | } 21 | `) 22 | 23 | func TestLoadJSON(t *testing.T) { 24 | cfg := &Config{} 25 | err := cfg.LoadJSON(cfgJSON) 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | } 30 | 31 | func TestToJSON(t *testing.T) { 32 | cfg := &Config{} 33 | cfg.LoadJSON(cfgJSON) 34 | 35 | if cfg.GCDiscardRatio != 0.1 { 36 | t.Fatal("GCDiscardRatio should be 0.1") 37 | } 38 | 39 | if cfg.GCInterval != DefaultGCInterval { 40 | t.Fatal("GCInterval should default as it is unset") 41 | } 42 | 43 | if cfg.GCSleep != 2*time.Minute { 44 | t.Fatal("GCSleep should be 2m") 45 | } 46 | 47 | if cfg.BadgerOptions.Compression != options.ZSTD { 48 | t.Fatalf("got: %d, want: %d", cfg.BadgerOptions.Compression, options.ZSTD) 49 | } 50 | 51 | if cfg.BadgerOptions.ValueLogFileSize != badger.DefaultOptions("").ValueLogFileSize { 52 | t.Fatalf( 53 | "got: %d, want: %d", 54 | cfg.BadgerOptions.ValueLogFileSize, 55 | badger.DefaultOptions("").ValueLogFileSize, 56 | ) 57 | } 58 | 59 | if cfg.BadgerOptions.ChecksumVerificationMode != badger.DefaultOptions("").ChecksumVerificationMode { 60 | t.Fatalf("ChecksumVerificationMode is not nil: got: %v, want: %v", cfg.BadgerOptions.ChecksumVerificationMode, badger.DefaultOptions("").ChecksumVerificationMode) 61 | } 62 | 63 | if cfg.BadgerOptions.MaxLevels != 4 { 64 | t.Fatalf("MaxLevels should be 4, got: %d", cfg.BadgerOptions.MaxLevels) 65 | } 66 | 67 | newjson, err := cfg.ToJSON() 68 | if err != nil { 69 | t.Fatal(err) 70 | } 71 | 72 | cfg = &Config{} 73 | err = cfg.LoadJSON(newjson) 74 | if err != nil { 75 | t.Fatal(err) 76 | } 77 | } 78 | 79 | func TestDefault(t *testing.T) { 80 | cfg := &Config{} 81 | cfg.Default() 82 | if cfg.Validate() != nil { 83 | t.Fatal("error validating") 84 | } 85 | 86 | cfg.GCDiscardRatio = 0 87 | if cfg.Validate() == nil { 88 | t.Fatal("expected error validating") 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /datastore/inmem/inmem.go: -------------------------------------------------------------------------------- 1 | // Package inmem provides a in-memory thread-safe datastore for use with 2 | // Cluster. 3 | package inmem 4 | 5 | import ( 6 | ds "github.com/ipfs/go-datastore" 7 | sync "github.com/ipfs/go-datastore/sync" 8 | ) 9 | 10 | // New returns a new thread-safe in-memory go-datastore. 11 | func New() ds.Datastore { 12 | mapDs := ds.NewMapDatastore() 13 | return sync.MutexWrap(mapDs) 14 | } 15 | -------------------------------------------------------------------------------- /datastore/leveldb/config_test.go: -------------------------------------------------------------------------------- 1 | package leveldb 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | var cfgJSON = []byte(` 8 | { 9 | "folder": "test", 10 | "leveldb_options": { 11 | "no_sync": true, 12 | "compaction_total_size_multiplier": 1.5 13 | } 14 | } 15 | `) 16 | 17 | func TestLoadJSON(t *testing.T) { 18 | cfg := &Config{} 19 | err := cfg.LoadJSON(cfgJSON) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | } 24 | 25 | func TestToJSON(t *testing.T) { 26 | cfg := &Config{} 27 | cfg.LoadJSON(cfgJSON) 28 | 29 | if !cfg.LevelDBOptions.NoSync { 30 | t.Fatalf("NoSync should be true") 31 | } 32 | 33 | if cfg.LevelDBOptions.CompactionTotalSizeMultiplier != 1.5 { 34 | t.Fatal("TotalSizeMultiplier should be 1.5") 35 | } 36 | 37 | newjson, err := cfg.ToJSON() 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | 42 | cfg = &Config{} 43 | err = cfg.LoadJSON(newjson) 44 | if err != nil { 45 | t.Fatal(err) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /datastore/leveldb/leveldb.go: -------------------------------------------------------------------------------- 1 | // Package leveldb provides a configurable LevelDB go-datastore for use with 2 | // IPFS Cluster. 3 | package leveldb 4 | 5 | import ( 6 | "os" 7 | 8 | ds "github.com/ipfs/go-datastore" 9 | leveldbds "github.com/ipfs/go-ds-leveldb" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | // New returns a LevelDB datastore configured with the given 14 | // configuration. 15 | func New(cfg *Config) (ds.Datastore, error) { 16 | folder := cfg.GetFolder() 17 | err := os.MkdirAll(folder, 0700) 18 | if err != nil { 19 | return nil, errors.Wrap(err, "creating leveldb folder") 20 | } 21 | return leveldbds.NewDatastore(folder, (*leveldbds.Options)(&cfg.LevelDBOptions)) 22 | } 23 | 24 | // Cleanup deletes the leveldb datastore. 25 | func Cleanup(cfg *Config) error { 26 | folder := cfg.GetFolder() 27 | if _, err := os.Stat(folder); os.IsNotExist(err) { 28 | return nil 29 | } 30 | return os.RemoveAll(cfg.GetFolder()) 31 | 32 | } 33 | -------------------------------------------------------------------------------- /datastore/pebble/config_test.go: -------------------------------------------------------------------------------- 1 | package pebble 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | var cfgJSON = []byte(` 8 | { 9 | "folder": "test", 10 | "pebble_options": { 11 | "bytes_per_sync": 524288, 12 | "disable_wal": true, 13 | "flush_delay_delete_range": 0, 14 | "flush_delay_range_key": 0, 15 | "flush_split_bytes": 4194304, 16 | "format_major_version": 16, 17 | "l0_compaction_file_threshold": 500, 18 | "l0_compaction_threshold": 2, 19 | "l0_stop_writes_threshold": 12, 20 | "l_base_max_bytes": 67108864, 21 | "levels": [ 22 | { 23 | "block_restart_interval": 16, 24 | "block_size": 4096, 25 | "block_size_threshold": 90, 26 | "Compression": 2, 27 | "filter_type": 0, 28 | "index_block_size": 8000, 29 | "target_file_size": 2097152 30 | } 31 | ], 32 | "max_open_files": 1000, 33 | "mem_table_size": 4194304, 34 | "mem_table_stop_writes_threshold": 2, 35 | "read_only": false, 36 | "wal_bytes_per_sync": 0, 37 | "max_concurrent_compactions": 2 38 | } 39 | } 40 | `) 41 | 42 | func TestLoadJSON(t *testing.T) { 43 | cfg := &Config{} 44 | err := cfg.LoadJSON(cfgJSON) 45 | if err != nil { 46 | t.Fatal(err) 47 | } 48 | } 49 | 50 | func TestToJSON(t *testing.T) { 51 | cfg := &Config{} 52 | cfg.LoadJSON(cfgJSON) 53 | 54 | if cfg.PebbleOptions.L0CompactionThreshold != 2 { 55 | t.Fatalf("got: %d, want: %d", cfg.PebbleOptions.L0CompactionThreshold, 2) 56 | } 57 | 58 | if !cfg.PebbleOptions.DisableWAL { 59 | t.Fatal("Disable WAL should be true") 60 | } 61 | 62 | if cfg.PebbleOptions.MaxConcurrentCompactions() != 2 { 63 | t.Fatalf("Wrong max concuncurrent compactions value, got: %d, want: %d", cfg.PebbleOptions.MaxConcurrentCompactions(), 2) 64 | } 65 | 66 | newjson, err := cfg.ToJSON() 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | 71 | cfg = &Config{} 72 | err = cfg.LoadJSON(newjson) 73 | if err != nil { 74 | t.Fatal(err) 75 | } 76 | } 77 | 78 | func TestDefault(t *testing.T) { 79 | cfg := &Config{} 80 | cfg.Default() 81 | if cfg.Validate() != nil { 82 | t.Fatal("error validating") 83 | } 84 | 85 | cfg.PebbleOptions.MemTableStopWritesThreshold = 0 86 | if cfg.Validate() == nil { 87 | t.Fatal("expected error validating") 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /datastore/pebble/pebble.go: -------------------------------------------------------------------------------- 1 | // Package pebble provides a configurable Pebble database backend for use with 2 | // IPFS Cluster. 3 | package pebble 4 | 5 | import ( 6 | "context" 7 | "os" 8 | "time" 9 | 10 | "github.com/cockroachdb/pebble" 11 | ds "github.com/ipfs/go-datastore" 12 | pebbleds "github.com/ipfs/go-ds-pebble" 13 | logging "github.com/ipfs/go-log/v2" 14 | "github.com/pkg/errors" 15 | ) 16 | 17 | var logger = logging.Logger("pebble") 18 | 19 | // New returns a Pebble datastore configured with the given 20 | // configuration. 21 | func New(cfg *Config) (ds.Datastore, error) { 22 | folder := cfg.GetFolder() 23 | err := os.MkdirAll(folder, 0700) 24 | if err != nil { 25 | return nil, errors.Wrap(err, "creating pebble folder") 26 | } 27 | 28 | // Deal with Pebble updates... user should try to be up to date with 29 | // latest Pebble table formats. 30 | fmv := cfg.PebbleOptions.FormatMajorVersion 31 | newest := pebble.FormatNewest 32 | if fmv < newest { 33 | logger.Warnf(`Pebble's format_major_version is set to %d, but newest version is %d. 34 | 35 | It is recommended to increase format_major_version and restart. If an error 36 | occurrs when increasing the number several versions at once, it may help to 37 | increase them one by one, restarting the daemon every time. 38 | `, fmv, newest) 39 | } 40 | 41 | db, err := pebbleds.NewDatastore(folder, pebbleds.WithPebbleOpts(&cfg.PebbleOptions)) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | // Calling regularly DB's DiskUsage is a way to printout debug 47 | // database statistics. 48 | go func() { 49 | ctx := context.Background() 50 | db.DiskUsage(ctx) 51 | ticker := time.NewTicker(time.Minute) 52 | defer ticker.Stop() 53 | for { 54 | <-ticker.C 55 | db.DiskUsage(ctx) 56 | } 57 | }() 58 | return db, nil 59 | } 60 | 61 | // Cleanup deletes the pebble datastore. 62 | func Cleanup(cfg *Config) error { 63 | folder := cfg.GetFolder() 64 | if _, err := os.Stat(folder); os.IsNotExist(err) { 65 | return nil 66 | } 67 | return os.RemoveAll(cfg.GetFolder()) 68 | } 69 | -------------------------------------------------------------------------------- /docker/cluster-restart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Restart the cluster process 4 | sleep 4 5 | while true; do 6 | export CLUSTER_SECRET="" 7 | pgrep ipfs-cluster-service || echo "CLUSTER RESTARTING"; ipfs-cluster-service daemon --debug & 8 | sleep 10 9 | done 10 | -------------------------------------------------------------------------------- /docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | user=ipfs 5 | 6 | if [ -n "$DOCKER_DEBUG" ]; then 7 | set -x 8 | fi 9 | 10 | if [ `id -u` -eq 0 ]; then 11 | echo "Changing user to $user" 12 | # ensure directories are writable 13 | su-exec "$user" test -w "${IPFS_CLUSTER_PATH}" || chown -R -- "$user" "${IPFS_CLUSTER_PATH}" 14 | exec su-exec "$user" "$0" $@ 15 | fi 16 | 17 | # Only ipfs user can get here 18 | ipfs-cluster-service --version 19 | 20 | if [ -e "${IPFS_CLUSTER_PATH}/service.json" ]; then 21 | echo "Found IPFS cluster configuration at ${IPFS_CLUSTER_PATH}" 22 | else 23 | echo "This container only runs ipfs-cluster-service. ipfs needs to be run separately!" 24 | echo "Initializing default configuration..." 25 | ipfs-cluster-service init --consensus "${IPFS_CLUSTER_CONSENSUS}" 26 | fi 27 | 28 | exec ipfs-cluster-service $@ 29 | -------------------------------------------------------------------------------- /docker/get-docker-tags.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # get-docker-tags.sh produces Docker tags for the current build 4 | # 5 | # Usage: 6 | # ./get-docker-tags.sh [git tag name] 7 | # 8 | # Example: 9 | # 10 | # # get tag for the main branch 11 | # ./get-docker-tags.sh $(date -u +%F) testingsha main 12 | # 13 | # # get tag for a release tag 14 | # ./get-docker-tags.sh $(date -u +%F) testingsha release v0.5.0 15 | # 16 | # # Serving suggestion in CI 17 | # ./get-docker-tags.sh $(date -u +%F) "$CI_SHA1" "$CI_BRANCH" "$CI_TAG" 18 | # 19 | set -euo pipefail 20 | 21 | if [[ $# -lt 1 ]] ; then 22 | echo 'At least 1 arg required.' 23 | echo 'Usage:' 24 | echo './get-docker-tags.sh [git commit sha1] [git branch name] [git tag name]' 25 | exit 1 26 | fi 27 | 28 | BUILD_NUM=$1 29 | GIT_SHA1=${2:-$(git rev-parse HEAD)} 30 | GIT_SHA1_SHORT=$(echo "$GIT_SHA1" | cut -c 1-7) 31 | GIT_BRANCH=${3:-$(git symbolic-ref -q --short HEAD || echo "unknown")} 32 | GIT_TAG=${4:-$(git describe --tags --exact-match 2> /dev/null || echo "")} 33 | 34 | IMAGE_NAME=${IMAGE_NAME:-ipfs/ipfs-cluster} 35 | 36 | echoImageName () { 37 | local IMAGE_TAG=$1 38 | echo "$IMAGE_NAME:$IMAGE_TAG" 39 | } 40 | 41 | if [[ $GIT_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+-rc ]]; then 42 | echoImageName "$GIT_TAG" 43 | 44 | elif [[ $GIT_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then 45 | echoImageName "$GIT_TAG" 46 | echoImageName "stable" 47 | echoImageName "latest" # assume we are not going to be tagging old versions 48 | 49 | elif [ "$GIT_BRANCH" = "master" ]; then 50 | echoImageName "master-${BUILD_NUM}-${GIT_SHA1_SHORT}" 51 | echoImageName "master-latest" 52 | 53 | else 54 | echo "Nothing to do. No docker tag defined for branch: $GIT_BRANCH, tag: $GIT_TAG" 55 | 56 | fi 57 | -------------------------------------------------------------------------------- /docker/random-killer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # $1=sleep time, $2=first command, $3=second command 4 | # Loop forever 5 | KILLED=1 6 | sleep 2 7 | while true; do 8 | if [ "$(($RANDOM % 10))" -gt "4" ]; then 9 | # Take down cluster 10 | if [ "$KILLED" -eq "1" ]; then 11 | KILLED=0 12 | killall ipfs-cluster-service 13 | else # Bring up cluster 14 | KILLED=1 15 | ipfs-cluster-service & 16 | fi 17 | fi 18 | sleep 2.5 19 | done 20 | -------------------------------------------------------------------------------- /docker/random-stopper.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | # $1=sleep time, $2=first command, $3=second command 4 | # Loop forever 5 | STOPPED=1 6 | sleep 2 7 | while true; do 8 | if [ "$(($RANDOM % 10))" -gt "4" ]; then 9 | # Take down cluster 10 | if [ "$STOPPED" -eq "1" ]; then 11 | STOPPED=0 12 | killall -STOP ipfs-cluster-service 13 | else # Take down cluster 14 | STOPPED=1 15 | killall -CONT ipfs-cluster-service 16 | fi 17 | fi 18 | sleep 1 19 | done 20 | -------------------------------------------------------------------------------- /docker/start-daemons.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | if [ -n "$DOCKER_DEBUG" ]; then 5 | set -x 6 | fi 7 | user=ipfs 8 | 9 | if [ `id -u` -eq 0 ]; then 10 | echo "Changing user to $user" 11 | # ensure directories are writable 12 | su-exec "$user" test -w "${IPFS_PATH}" || chown -R -- "$user" "${IPFS_PATH}" 13 | su-exec "$user" test -w "${IPFS_CLUSTER_PATH}" || chown -R -- "$user" "${IPFS_CLUSTER_PATH}" 14 | exec su-exec "$user" "$0" $@ 15 | fi 16 | 17 | # Second invocation with regular user 18 | ipfs version 19 | 20 | if [ -e "${IPFS_PATH}/config" ]; then 21 | echo "Found IPFS fs-repo at ${IPFS_PATH}" 22 | else 23 | ipfs init 24 | ipfs config Addresses.API /ip4/0.0.0.0/tcp/5001 25 | ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/8080 26 | fi 27 | 28 | ipfs daemon --migrate=true & 29 | sleep 3 30 | 31 | ipfs-cluster-service --version 32 | 33 | if [ -e "${IPFS_CLUSTER_PATH}/service.json" ]; then 34 | echo "Found IPFS cluster configuration at ${IPFS_CLUSTER_PATH}" 35 | else 36 | ipfs-cluster-service init --consensus "${IPFS_CLUSTER_CONSENSUS}" 37 | fi 38 | 39 | exec ipfs-cluster-service $@ 40 | -------------------------------------------------------------------------------- /docker/test-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | if [ -n "$DOCKER_DEBUG" ]; then 5 | set -x 6 | fi 7 | user=ipfs 8 | 9 | if [ `id -u` -eq 0 ]; then 10 | echo "Changing user to $user" 11 | # ensure directories are writable 12 | su-exec "$user" test -w "${IPFS_PATH}" || chown -R -- "$user" "${IPFS_PATH}" 13 | su-exec "$user" test -w "${IPFS_CLUSTER_PATH}" || chown -R -- "$user" "${IPFS_CLUSTER_PATH}" 14 | exec su-exec "$user" "$0" $@ 15 | fi 16 | 17 | ipfs version 18 | 19 | if [ -e "${IPFS_PATH}/config" ]; then 20 | echo "Found IPFS fs-repo at ${IPFS_PATH}" 21 | else 22 | ipfs init 23 | ipfs config Addresses.API /ip4/0.0.0.0/tcp/5001 24 | ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/8080 25 | fi 26 | 27 | ipfs daemon --migrate=true & 28 | sleep 3 29 | 30 | ipfs-cluster-service --version 31 | 32 | if [ -e "$IPFS_CLUSTER_PATH/service.json" ]; then 33 | echo "Found IPFS cluster configuration at $IPFS_CLUSTER_PATH" 34 | else 35 | export CLUSTER_SECRET="" 36 | ipfs-cluster-service init --consensus "${IPFS_CLUSTER_CONSENSUS}" 37 | fi 38 | 39 | ipfs-cluster-service --debug $@ & 40 | # Testing scripts that spawn background processes are spawned and stopped here 41 | /usr/local/bin/random-stopper.sh & 42 | kill -STOP $! 43 | echo $! > /data/ipfs-cluster/random-stopper-pid 44 | /usr/local/bin/random-killer.sh & 45 | kill -STOP $! 46 | echo $! > /data/ipfs-cluster/random-killer-pid 47 | /usr/local/bin/cluster-restart.sh & 48 | kill -STOP $! 49 | echo $! > /data/ipfs-cluster/cluster-restart-pid 50 | 51 | echo "Daemons launched" 52 | exec tail -f /dev/null 53 | -------------------------------------------------------------------------------- /docker/wait-killer-stopper.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | ## Wait for cluster service process to exist and the stop the 4 | ## killer 5 | DONE=1 6 | while [ $DONE = 1 ]; do 7 | sleep 0.1 8 | if [ $(pgrep -f ipfs-cluster-service) ]; then 9 | kill -STOP $(cat /data/ipfs-cluster/random-killer-pid) 10 | DONE=0 11 | fi 12 | done 13 | -------------------------------------------------------------------------------- /informer/disk/config.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "time" 7 | 8 | "github.com/ipfs-cluster/ipfs-cluster/config" 9 | "github.com/kelseyhightower/envconfig" 10 | ) 11 | 12 | const configKey = "disk" 13 | const envConfigKey = "cluster_disk" 14 | 15 | // Default values for disk Config 16 | const ( 17 | DefaultMetricTTL = 30 * time.Second 18 | DefaultMetricType = MetricFreeSpace 19 | ) 20 | 21 | // Config is used to initialize an Informer and customize 22 | // the type and parameters of the metric it produces. 23 | type Config struct { 24 | config.Saver 25 | 26 | MetricTTL time.Duration 27 | MetricType MetricType 28 | } 29 | 30 | type jsonConfig struct { 31 | MetricTTL string `json:"metric_ttl"` 32 | MetricType string `json:"metric_type"` 33 | } 34 | 35 | // ConfigKey returns a human-friendly identifier for this type of Metric. 36 | func (cfg *Config) ConfigKey() string { 37 | return configKey 38 | } 39 | 40 | // Default initializes this Config with sensible values. 41 | func (cfg *Config) Default() error { 42 | cfg.MetricTTL = DefaultMetricTTL 43 | cfg.MetricType = DefaultMetricType 44 | return nil 45 | } 46 | 47 | // ApplyEnvVars fills in any Config fields found 48 | // as environment variables. 49 | func (cfg *Config) ApplyEnvVars() error { 50 | jcfg := cfg.toJSONConfig() 51 | 52 | err := envconfig.Process(envConfigKey, jcfg) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | return cfg.applyJSONConfig(jcfg) 58 | } 59 | 60 | // Validate checks that the fields of this Config have working values, 61 | // at least in appearance. 62 | func (cfg *Config) Validate() error { 63 | if cfg.MetricTTL <= 0 { 64 | return errors.New("disk.metric_ttl is invalid") 65 | } 66 | 67 | if cfg.MetricType.String() == "" { 68 | return errors.New("disk.metric_type is invalid") 69 | } 70 | return nil 71 | } 72 | 73 | // LoadJSON reads the fields of this Config from a JSON byteslice as 74 | // generated by ToJSON. 75 | func (cfg *Config) LoadJSON(raw []byte) error { 76 | jcfg := &jsonConfig{} 77 | err := json.Unmarshal(raw, jcfg) 78 | if err != nil { 79 | logger.Error("Error unmarshaling disk informer config") 80 | return err 81 | } 82 | 83 | cfg.Default() 84 | 85 | return cfg.applyJSONConfig(jcfg) 86 | } 87 | 88 | func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error { 89 | t, _ := time.ParseDuration(jcfg.MetricTTL) 90 | cfg.MetricTTL = t 91 | 92 | switch jcfg.MetricType { 93 | case "reposize": 94 | cfg.MetricType = MetricRepoSize 95 | case "freespace": 96 | cfg.MetricType = MetricFreeSpace 97 | default: 98 | return errors.New("disk.metric_type is invalid") 99 | } 100 | 101 | return cfg.Validate() 102 | } 103 | 104 | // ToJSON generates a JSON-formatted human-friendly representation of this 105 | // Config. 106 | func (cfg *Config) ToJSON() (raw []byte, err error) { 107 | jcfg := cfg.toJSONConfig() 108 | 109 | raw, err = config.DefaultJSONMarshal(jcfg) 110 | return 111 | } 112 | 113 | func (cfg *Config) toJSONConfig() *jsonConfig { 114 | return &jsonConfig{ 115 | MetricTTL: cfg.MetricTTL.String(), 116 | MetricType: cfg.MetricType.String(), 117 | } 118 | } 119 | 120 | // ToDisplayJSON returns JSON config as a string. 121 | func (cfg *Config) ToDisplayJSON() ([]byte, error) { 122 | return config.DisplayJSON(cfg.toJSONConfig()) 123 | } 124 | -------------------------------------------------------------------------------- /informer/disk/config_test.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | var cfgJSON = []byte(` 11 | { 12 | "metric_ttl": "1s", 13 | "metric_type": "freespace" 14 | } 15 | `) 16 | 17 | func TestLoadJSON(t *testing.T) { 18 | cfg := &Config{} 19 | err := cfg.LoadJSON(cfgJSON) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | 24 | j := &jsonConfig{} 25 | 26 | json.Unmarshal(cfgJSON, j) 27 | j.MetricTTL = "-10" 28 | tst, _ := json.Marshal(j) 29 | err = cfg.LoadJSON(tst) 30 | if err == nil { 31 | t.Error("expected error decoding metric_ttl") 32 | } 33 | 34 | j = &jsonConfig{} 35 | json.Unmarshal(cfgJSON, j) 36 | j.MetricType = "abc" 37 | tst, _ = json.Marshal(j) 38 | err = cfg.LoadJSON(tst) 39 | if err == nil { 40 | t.Error("expected error decoding check_interval") 41 | } 42 | 43 | j = &jsonConfig{} 44 | json.Unmarshal(cfgJSON, j) 45 | j.MetricType = "reposize" 46 | tst, _ = json.Marshal(j) 47 | err = cfg.LoadJSON(tst) 48 | if err != nil { 49 | t.Error("reposize should be a valid type") 50 | } 51 | 52 | } 53 | 54 | func TestToJSON(t *testing.T) { 55 | cfg := &Config{} 56 | cfg.LoadJSON(cfgJSON) 57 | newjson, err := cfg.ToJSON() 58 | if err != nil { 59 | t.Fatal(err) 60 | } 61 | cfg = &Config{} 62 | err = cfg.LoadJSON(newjson) 63 | if err != nil { 64 | t.Fatal(err) 65 | } 66 | } 67 | 68 | func TestDefault(t *testing.T) { 69 | cfg := &Config{} 70 | cfg.Default() 71 | if cfg.Validate() != nil { 72 | t.Fatal("error validating") 73 | } 74 | 75 | cfg.MetricTTL = 0 76 | if cfg.Validate() == nil { 77 | t.Fatal("expected error validating") 78 | } 79 | 80 | cfg.Default() 81 | cfg.MetricType = MetricRepoSize 82 | if cfg.Validate() != nil { 83 | t.Fatal("MetricRepoSize is a valid type") 84 | } 85 | } 86 | 87 | func TestApplyEnvVars(t *testing.T) { 88 | os.Setenv("CLUSTER_DISK_METRICTTL", "22s") 89 | cfg := &Config{} 90 | cfg.ApplyEnvVars() 91 | 92 | if cfg.MetricTTL != 22*time.Second { 93 | t.Fatal("failed to override metric_ttl with env var") 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /informer/disk/disk_test.go: -------------------------------------------------------------------------------- 1 | package disk 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | 8 | "github.com/ipfs-cluster/ipfs-cluster/api" 9 | "github.com/ipfs-cluster/ipfs-cluster/test" 10 | 11 | rpc "github.com/libp2p/go-libp2p-gorpc" 12 | ) 13 | 14 | type badRPCService struct { 15 | } 16 | 17 | func badRPCClient(t *testing.T) *rpc.Client { 18 | s := rpc.NewServer(nil, "mock") 19 | c := rpc.NewClientWithServer(nil, "mock", s) 20 | err := s.RegisterName("IPFSConnector", &badRPCService{}) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | return c 25 | } 26 | 27 | func (mock *badRPCService) RepoStat(ctx context.Context, in struct{}, out *api.IPFSRepoStat) error { 28 | return errors.New("fake error") 29 | } 30 | 31 | // Returns the first metric 32 | func getMetrics(t *testing.T, inf *Informer) api.Metric { 33 | t.Helper() 34 | metrics := inf.GetMetrics(context.Background()) 35 | if len(metrics) != 1 { 36 | t.Fatal("expected 1 metric") 37 | } 38 | return metrics[0] 39 | } 40 | 41 | func Test(t *testing.T) { 42 | ctx := context.Background() 43 | cfg := &Config{} 44 | cfg.Default() 45 | inf, err := NewInformer(cfg) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | defer inf.Shutdown(ctx) 50 | m := getMetrics(t, inf) 51 | if m.Valid { 52 | t.Error("metric should be invalid") 53 | } 54 | inf.SetClient(test.NewMockRPCClient(t)) 55 | m = getMetrics(t, inf) 56 | if !m.Valid { 57 | t.Error("metric should be valid") 58 | } 59 | } 60 | 61 | func TestFreeSpace(t *testing.T) { 62 | ctx := context.Background() 63 | cfg := &Config{} 64 | cfg.Default() 65 | cfg.MetricType = MetricFreeSpace 66 | 67 | inf, err := NewInformer(cfg) 68 | if err != nil { 69 | t.Fatal(err) 70 | } 71 | defer inf.Shutdown(ctx) 72 | m := getMetrics(t, inf) 73 | if m.Valid { 74 | t.Error("metric should be invalid") 75 | } 76 | inf.SetClient(test.NewMockRPCClient(t)) 77 | m = getMetrics(t, inf) 78 | if !m.Valid { 79 | t.Error("metric should be valid") 80 | } 81 | // The mock client reports 100KB and 2 pins of 1 KB 82 | if m.Value != "98000" { 83 | t.Error("bad metric value") 84 | } 85 | } 86 | 87 | func TestRepoSize(t *testing.T) { 88 | ctx := context.Background() 89 | cfg := &Config{} 90 | cfg.Default() 91 | cfg.MetricType = MetricRepoSize 92 | 93 | inf, err := NewInformer(cfg) 94 | if err != nil { 95 | t.Fatal(err) 96 | } 97 | defer inf.Shutdown(ctx) 98 | m := getMetrics(t, inf) 99 | if m.Valid { 100 | t.Error("metric should be invalid") 101 | } 102 | inf.SetClient(test.NewMockRPCClient(t)) 103 | m = getMetrics(t, inf) 104 | if !m.Valid { 105 | t.Error("metric should be valid") 106 | } 107 | // The mock client reports 100KB and 2 pins of 1 KB 108 | if m.Value != "2000" { 109 | t.Error("bad metric value") 110 | } 111 | } 112 | 113 | func TestWithErrors(t *testing.T) { 114 | ctx := context.Background() 115 | cfg := &Config{} 116 | cfg.Default() 117 | inf, err := NewInformer(cfg) 118 | if err != nil { 119 | t.Fatal(err) 120 | } 121 | defer inf.Shutdown(ctx) 122 | inf.SetClient(badRPCClient(t)) 123 | m := getMetrics(t, inf) 124 | if m.Valid { 125 | t.Errorf("metric should be invalid") 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /informer/numpin/config.go: -------------------------------------------------------------------------------- 1 | package numpin 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "time" 7 | 8 | "github.com/ipfs-cluster/ipfs-cluster/config" 9 | "github.com/kelseyhightower/envconfig" 10 | ) 11 | 12 | const configKey = "numpin" 13 | const envConfigKey = "cluster_numpin" 14 | 15 | // These are the default values for a Config. 16 | const ( 17 | DefaultMetricTTL = 10 * time.Second 18 | ) 19 | 20 | // Config allows to initialize an Informer. 21 | type Config struct { 22 | config.Saver 23 | 24 | MetricTTL time.Duration 25 | } 26 | 27 | type jsonConfig struct { 28 | MetricTTL string `json:"metric_ttl"` 29 | } 30 | 31 | // ConfigKey returns a human-friendly identifier for this 32 | // Config's type. 33 | func (cfg *Config) ConfigKey() string { 34 | return configKey 35 | } 36 | 37 | // Default initializes this Config with sensible values. 38 | func (cfg *Config) Default() error { 39 | cfg.MetricTTL = DefaultMetricTTL 40 | return nil 41 | } 42 | 43 | // ApplyEnvVars fills in any Config fields found 44 | // as environment variables. 45 | func (cfg *Config) ApplyEnvVars() error { 46 | jcfg := cfg.toJSONConfig() 47 | 48 | err := envconfig.Process(envConfigKey, jcfg) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | return cfg.applyJSONConfig(jcfg) 54 | } 55 | 56 | // Validate checks that the fields of this configuration have 57 | // sensible values. 58 | func (cfg *Config) Validate() error { 59 | if cfg.MetricTTL <= 0 { 60 | return errors.New("disk.metric_ttl is invalid") 61 | } 62 | 63 | return nil 64 | } 65 | 66 | // LoadJSON parses a raw JSON byte-slice as generated by ToJSON(). 67 | func (cfg *Config) LoadJSON(raw []byte) error { 68 | jcfg := &jsonConfig{} 69 | err := json.Unmarshal(raw, jcfg) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | cfg.Default() 75 | 76 | return cfg.applyJSONConfig(jcfg) 77 | } 78 | 79 | func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error { 80 | t, _ := time.ParseDuration(jcfg.MetricTTL) 81 | cfg.MetricTTL = t 82 | 83 | return cfg.Validate() 84 | } 85 | 86 | // ToJSON generates a human-friendly JSON representation of this Config. 87 | func (cfg *Config) ToJSON() ([]byte, error) { 88 | jcfg := cfg.toJSONConfig() 89 | 90 | return config.DefaultJSONMarshal(jcfg) 91 | } 92 | 93 | func (cfg *Config) toJSONConfig() *jsonConfig { 94 | return &jsonConfig{ 95 | MetricTTL: cfg.MetricTTL.String(), 96 | } 97 | } 98 | 99 | // ToDisplayJSON returns JSON config as a string. 100 | func (cfg *Config) ToDisplayJSON() ([]byte, error) { 101 | return config.DisplayJSON(cfg.toJSONConfig()) 102 | } 103 | -------------------------------------------------------------------------------- /informer/numpin/config_test.go: -------------------------------------------------------------------------------- 1 | package numpin 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | var cfgJSON = []byte(` 11 | { 12 | "metric_ttl": "1s" 13 | } 14 | `) 15 | 16 | func TestLoadJSON(t *testing.T) { 17 | cfg := &Config{} 18 | err := cfg.LoadJSON(cfgJSON) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | j := &jsonConfig{} 24 | 25 | json.Unmarshal(cfgJSON, j) 26 | j.MetricTTL = "-10" 27 | tst, _ := json.Marshal(j) 28 | err = cfg.LoadJSON(tst) 29 | if err == nil { 30 | t.Error("expected error decoding metric_ttl") 31 | } 32 | } 33 | 34 | func TestToJSON(t *testing.T) { 35 | cfg := &Config{} 36 | cfg.LoadJSON(cfgJSON) 37 | newjson, err := cfg.ToJSON() 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | cfg = &Config{} 42 | err = cfg.LoadJSON(newjson) 43 | if err != nil { 44 | t.Fatal(err) 45 | } 46 | } 47 | 48 | func TestDefault(t *testing.T) { 49 | cfg := &Config{} 50 | cfg.Default() 51 | if cfg.Validate() != nil { 52 | t.Fatal("error validating") 53 | } 54 | 55 | cfg.MetricTTL = 0 56 | if cfg.Validate() == nil { 57 | t.Fatal("expected error validating") 58 | } 59 | 60 | } 61 | 62 | func TestApplyEnvVars(t *testing.T) { 63 | os.Setenv("CLUSTER_NUMPIN_METRICTTL", "22s") 64 | cfg := &Config{} 65 | cfg.ApplyEnvVars() 66 | 67 | if cfg.MetricTTL != 22*time.Second { 68 | t.Fatal("failed to override metric_ttl with env var") 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /informer/numpin/numpin.go: -------------------------------------------------------------------------------- 1 | // Package numpin implements an ipfs-cluster informer which determines how many 2 | // items this peer is pinning and returns it as api.Metric 3 | package numpin 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | "sync" 9 | 10 | "github.com/ipfs-cluster/ipfs-cluster/api" 11 | 12 | rpc "github.com/libp2p/go-libp2p-gorpc" 13 | 14 | "go.opencensus.io/trace" 15 | ) 16 | 17 | // MetricName specifies the name of our metric 18 | var MetricName = "numpin" 19 | 20 | // Informer is a simple object to implement the ipfscluster.Informer 21 | // and Component interfaces 22 | type Informer struct { 23 | config *Config 24 | 25 | mu sync.Mutex 26 | rpcClient *rpc.Client 27 | } 28 | 29 | // NewInformer returns an initialized Informer. 30 | func NewInformer(cfg *Config) (*Informer, error) { 31 | err := cfg.Validate() 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | return &Informer{ 37 | config: cfg, 38 | }, nil 39 | } 40 | 41 | // SetClient provides us with an rpc.Client which allows 42 | // contacting other components in the cluster. 43 | func (npi *Informer) SetClient(c *rpc.Client) { 44 | npi.mu.Lock() 45 | npi.rpcClient = c 46 | npi.mu.Unlock() 47 | } 48 | 49 | // Shutdown is called on cluster shutdown. We just invalidate 50 | // any metrics from this point. 51 | func (npi *Informer) Shutdown(ctx context.Context) error { 52 | _, span := trace.StartSpan(ctx, "informer/numpin/Shutdown") 53 | defer span.End() 54 | 55 | npi.mu.Lock() 56 | npi.rpcClient = nil 57 | npi.mu.Unlock() 58 | return nil 59 | } 60 | 61 | // Name returns the name of this informer 62 | func (npi *Informer) Name() string { 63 | return MetricName 64 | } 65 | 66 | // GetMetrics contacts the IPFSConnector component and requests the `pin ls` 67 | // command. We return the number of pins in IPFS. It must always return at 68 | // least one metric. 69 | func (npi *Informer) GetMetrics(ctx context.Context) []api.Metric { 70 | ctx, span := trace.StartSpan(ctx, "informer/numpin/GetMetric") 71 | defer span.End() 72 | 73 | npi.mu.Lock() 74 | rpcClient := npi.rpcClient 75 | npi.mu.Unlock() 76 | 77 | if rpcClient == nil { 78 | return []api.Metric{ 79 | { 80 | Valid: false, 81 | }, 82 | } 83 | } 84 | 85 | // make use of the RPC API to obtain information 86 | // about the number of pins in IPFS. See RPCAPI docs. 87 | in := make(chan []string, 1) 88 | in <- []string{"recursive", "direct"} 89 | close(in) 90 | out := make(chan api.IPFSPinInfo, 1024) 91 | 92 | errCh := make(chan error, 1) 93 | go func() { 94 | defer close(errCh) 95 | err := rpcClient.Stream( 96 | ctx, 97 | "", // Local call 98 | "IPFSConnector", // Service name 99 | "PinLs", // Method name 100 | in, 101 | out, 102 | ) 103 | errCh <- err 104 | }() 105 | 106 | n := 0 107 | for range out { 108 | n++ 109 | } 110 | 111 | err := <-errCh 112 | 113 | valid := err == nil 114 | 115 | m := api.Metric{ 116 | Name: MetricName, 117 | Value: fmt.Sprintf("%d", n), 118 | Valid: valid, 119 | Partitionable: false, 120 | } 121 | 122 | m.SetTTL(npi.config.MetricTTL) 123 | return []api.Metric{m} 124 | } 125 | -------------------------------------------------------------------------------- /informer/numpin/numpin_test.go: -------------------------------------------------------------------------------- 1 | package numpin 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/ipfs-cluster/ipfs-cluster/api" 8 | "github.com/ipfs-cluster/ipfs-cluster/test" 9 | 10 | rpc "github.com/libp2p/go-libp2p-gorpc" 11 | ) 12 | 13 | type mockService struct{} 14 | 15 | func mockRPCClient(t *testing.T) *rpc.Client { 16 | s := rpc.NewServer(nil, "mock") 17 | c := rpc.NewClientWithServer(nil, "mock", s) 18 | err := s.RegisterName("IPFSConnector", &mockService{}) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | return c 23 | } 24 | 25 | func (mock *mockService) PinLs(ctx context.Context, in <-chan []string, out chan<- api.IPFSPinInfo) error { 26 | out <- api.IPFSPinInfo{Cid: api.Cid(test.Cid1), Type: api.IPFSPinStatusRecursive} 27 | out <- api.IPFSPinInfo{Cid: api.Cid(test.Cid2), Type: api.IPFSPinStatusRecursive} 28 | close(out) 29 | return nil 30 | } 31 | 32 | func Test(t *testing.T) { 33 | ctx := context.Background() 34 | cfg := &Config{} 35 | cfg.Default() 36 | inf, err := NewInformer(cfg) 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | metrics := inf.GetMetrics(ctx) 41 | if len(metrics) != 1 { 42 | t.Fatal("expected 1 metric") 43 | } 44 | m := metrics[0] 45 | 46 | if m.Valid { 47 | t.Error("metric should be invalid") 48 | } 49 | inf.SetClient(mockRPCClient(t)) 50 | 51 | metrics = inf.GetMetrics(ctx) 52 | if len(metrics) != 1 { 53 | t.Fatal("expected 1 metric") 54 | } 55 | m = metrics[0] 56 | if !m.Valid { 57 | t.Error("metric should be valid") 58 | } 59 | if m.Value != "2" { 60 | t.Error("bad metric value") 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /informer/pinqueue/config.go: -------------------------------------------------------------------------------- 1 | package pinqueue 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "time" 7 | 8 | "github.com/ipfs-cluster/ipfs-cluster/config" 9 | "github.com/kelseyhightower/envconfig" 10 | ) 11 | 12 | const configKey = "pinqueue" 13 | const envConfigKey = "cluster_pinqueue" 14 | 15 | // These are the default values for a Config. 16 | const ( 17 | DefaultMetricTTL = 30 * time.Second 18 | DefaultWeightBucketSize = 100000 // 100k pins 19 | ) 20 | 21 | // Config allows to initialize an Informer. 22 | type Config struct { 23 | config.Saver 24 | 25 | MetricTTL time.Duration 26 | WeightBucketSize int 27 | } 28 | 29 | type jsonConfig struct { 30 | MetricTTL string `json:"metric_ttl"` 31 | WeightBucketSize int `json:"weight_bucket_size"` 32 | } 33 | 34 | // ConfigKey returns a human-friendly identifier for this 35 | // Config's type. 36 | func (cfg *Config) ConfigKey() string { 37 | return configKey 38 | } 39 | 40 | // Default initializes this Config with sensible values. 41 | func (cfg *Config) Default() error { 42 | cfg.MetricTTL = DefaultMetricTTL 43 | cfg.WeightBucketSize = DefaultWeightBucketSize 44 | return nil 45 | } 46 | 47 | // ApplyEnvVars fills in any Config fields found 48 | // as environment variables. 49 | func (cfg *Config) ApplyEnvVars() error { 50 | jcfg := cfg.toJSONConfig() 51 | 52 | err := envconfig.Process(envConfigKey, jcfg) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | return cfg.applyJSONConfig(jcfg) 58 | } 59 | 60 | // Validate checks that the fields of this configuration have 61 | // sensible values. 62 | func (cfg *Config) Validate() error { 63 | if cfg.MetricTTL <= 0 { 64 | return errors.New("pinqueue.metric_ttl is invalid") 65 | } 66 | if cfg.WeightBucketSize < 0 { 67 | return errors.New("pinqueue.WeightBucketSize is invalid") 68 | } 69 | 70 | return nil 71 | } 72 | 73 | // LoadJSON parses a raw JSON byte-slice as generated by ToJSON(). 74 | func (cfg *Config) LoadJSON(raw []byte) error { 75 | jcfg := &jsonConfig{} 76 | err := json.Unmarshal(raw, jcfg) 77 | if err != nil { 78 | return err 79 | } 80 | 81 | cfg.Default() 82 | 83 | return cfg.applyJSONConfig(jcfg) 84 | } 85 | 86 | func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error { 87 | t, _ := time.ParseDuration(jcfg.MetricTTL) 88 | cfg.MetricTTL = t 89 | cfg.WeightBucketSize = jcfg.WeightBucketSize 90 | 91 | return cfg.Validate() 92 | } 93 | 94 | // ToJSON generates a human-friendly JSON representation of this Config. 95 | func (cfg *Config) ToJSON() ([]byte, error) { 96 | jcfg := cfg.toJSONConfig() 97 | 98 | return config.DefaultJSONMarshal(jcfg) 99 | } 100 | 101 | func (cfg *Config) toJSONConfig() *jsonConfig { 102 | return &jsonConfig{ 103 | MetricTTL: cfg.MetricTTL.String(), 104 | WeightBucketSize: cfg.WeightBucketSize, 105 | } 106 | } 107 | 108 | // ToDisplayJSON returns JSON config as a string. 109 | func (cfg *Config) ToDisplayJSON() ([]byte, error) { 110 | return config.DisplayJSON(cfg.toJSONConfig()) 111 | } 112 | -------------------------------------------------------------------------------- /informer/pinqueue/config_test.go: -------------------------------------------------------------------------------- 1 | package pinqueue 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | var cfgJSON = []byte(` 11 | { 12 | "metric_ttl": "1s" 13 | } 14 | `) 15 | 16 | func TestLoadJSON(t *testing.T) { 17 | cfg := &Config{} 18 | err := cfg.LoadJSON(cfgJSON) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | j := &jsonConfig{} 24 | 25 | json.Unmarshal(cfgJSON, j) 26 | j.MetricTTL = "-10" 27 | tst, _ := json.Marshal(j) 28 | err = cfg.LoadJSON(tst) 29 | if err == nil { 30 | t.Error("expected error decoding metric_ttl") 31 | } 32 | } 33 | 34 | func TestToJSON(t *testing.T) { 35 | cfg := &Config{} 36 | cfg.LoadJSON(cfgJSON) 37 | newjson, err := cfg.ToJSON() 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | cfg = &Config{} 42 | err = cfg.LoadJSON(newjson) 43 | if err != nil { 44 | t.Fatal(err) 45 | } 46 | } 47 | 48 | func TestDefault(t *testing.T) { 49 | cfg := &Config{} 50 | cfg.Default() 51 | if cfg.Validate() != nil { 52 | t.Fatal("error validating") 53 | } 54 | 55 | cfg.MetricTTL = 0 56 | if cfg.Validate() == nil { 57 | t.Fatal("expected error validating") 58 | } 59 | 60 | cfg.Default() 61 | cfg.WeightBucketSize = -2 62 | if cfg.Validate() == nil { 63 | t.Fatal("expected error validating") 64 | } 65 | 66 | } 67 | 68 | func TestApplyEnvVars(t *testing.T) { 69 | os.Setenv("CLUSTER_PINQUEUE_METRICTTL", "22s") 70 | cfg := &Config{} 71 | cfg.ApplyEnvVars() 72 | 73 | if cfg.MetricTTL != 22*time.Second { 74 | t.Fatal("failed to override metric_ttl with env var") 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /informer/pinqueue/pinqueue.go: -------------------------------------------------------------------------------- 1 | // Package pinqueue implements an ipfs-cluster informer which issues the 2 | // current size of the pinning queue. 3 | package pinqueue 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | "sync" 9 | 10 | "github.com/ipfs-cluster/ipfs-cluster/api" 11 | 12 | rpc "github.com/libp2p/go-libp2p-gorpc" 13 | 14 | "go.opencensus.io/trace" 15 | ) 16 | 17 | // MetricName specifies the name of our metric 18 | var MetricName = "pinqueue" 19 | 20 | // Informer is a simple object to implement the ipfscluster.Informer 21 | // and Component interfaces 22 | type Informer struct { 23 | config *Config 24 | 25 | mu sync.Mutex 26 | rpcClient *rpc.Client 27 | } 28 | 29 | // New returns an initialized Informer. 30 | func New(cfg *Config) (*Informer, error) { 31 | err := cfg.Validate() 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | return &Informer{ 37 | config: cfg, 38 | }, nil 39 | } 40 | 41 | // SetClient provides us with an rpc.Client which allows 42 | // contacting other components in the cluster. 43 | func (inf *Informer) SetClient(c *rpc.Client) { 44 | inf.mu.Lock() 45 | inf.rpcClient = c 46 | inf.mu.Unlock() 47 | } 48 | 49 | // Shutdown is called on cluster shutdown. We just invalidate 50 | // any metrics from this point. 51 | func (inf *Informer) Shutdown(ctx context.Context) error { 52 | _, span := trace.StartSpan(ctx, "informer/numpin/Shutdown") 53 | defer span.End() 54 | 55 | inf.mu.Lock() 56 | inf.rpcClient = nil 57 | inf.mu.Unlock() 58 | return nil 59 | } 60 | 61 | // Name returns the name of this informer 62 | func (inf *Informer) Name() string { 63 | return MetricName 64 | } 65 | 66 | // GetMetrics contacts the Pintracker component and requests the number of 67 | // queued items for pinning. 68 | func (inf *Informer) GetMetrics(ctx context.Context) []api.Metric { 69 | ctx, span := trace.StartSpan(ctx, "informer/pinqueue/GetMetric") 70 | defer span.End() 71 | 72 | inf.mu.Lock() 73 | rpcClient := inf.rpcClient 74 | inf.mu.Unlock() 75 | 76 | if rpcClient == nil { 77 | return []api.Metric{ 78 | { 79 | Valid: false, 80 | }, 81 | } 82 | } 83 | 84 | var queued int64 85 | 86 | err := rpcClient.CallContext( 87 | ctx, 88 | "", 89 | "PinTracker", 90 | "PinQueueSize", 91 | struct{}{}, 92 | &queued, 93 | ) 94 | valid := err == nil 95 | weight := -queued // smaller pin queues have more priority 96 | if div := inf.config.WeightBucketSize; div > 0 { 97 | weight = weight / int64(div) 98 | } 99 | 100 | m := api.Metric{ 101 | Name: MetricName, 102 | Value: fmt.Sprintf("%d", queued), 103 | Valid: valid, 104 | Partitionable: false, 105 | Weight: weight, 106 | } 107 | 108 | m.SetTTL(inf.config.MetricTTL) 109 | return []api.Metric{m} 110 | } 111 | -------------------------------------------------------------------------------- /informer/pinqueue/pinqueue_test.go: -------------------------------------------------------------------------------- 1 | package pinqueue 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | rpc "github.com/libp2p/go-libp2p-gorpc" 8 | ) 9 | 10 | type mockService struct{} 11 | 12 | func (mock *mockService) PinQueueSize(ctx context.Context, in struct{}, out *int64) error { 13 | *out = 42 14 | return nil 15 | } 16 | 17 | func mockRPCClient(t *testing.T) *rpc.Client { 18 | s := rpc.NewServer(nil, "mock") 19 | c := rpc.NewClientWithServer(nil, "mock", s) 20 | err := s.RegisterName("PinTracker", &mockService{}) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | return c 25 | } 26 | 27 | func Test(t *testing.T) { 28 | ctx := context.Background() 29 | cfg := &Config{} 30 | cfg.Default() 31 | cfg.WeightBucketSize = 0 32 | inf, err := New(cfg) 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | metrics := inf.GetMetrics(ctx) 37 | if len(metrics) != 1 { 38 | t.Fatal("expected 1 metric") 39 | } 40 | m := metrics[0] 41 | 42 | if m.Valid { 43 | t.Error("metric should be invalid") 44 | } 45 | inf.SetClient(mockRPCClient(t)) 46 | 47 | metrics = inf.GetMetrics(ctx) 48 | if len(metrics) != 1 { 49 | t.Fatal("expected 1 metric") 50 | } 51 | m = metrics[0] 52 | if !m.Valid { 53 | t.Error("metric should be valid") 54 | } 55 | if m.Value != "42" { 56 | t.Error("bad metric value", m.Value) 57 | } 58 | if m.Partitionable { 59 | t.Error("should not be a partitionable metric") 60 | } 61 | if m.Weight != -42 { 62 | t.Error("weight should be -42") 63 | } 64 | 65 | cfg.WeightBucketSize = 5 66 | inf, err = New(cfg) 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | inf.SetClient(mockRPCClient(t)) 71 | metrics = inf.GetMetrics(ctx) 72 | if len(metrics) != 1 { 73 | t.Fatal("expected 1 metric") 74 | } 75 | m = metrics[0] 76 | if m.Weight != -8 { 77 | t.Error("weight should be -8, not", m.Weight) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /informer/tags/config.go: -------------------------------------------------------------------------------- 1 | package tags 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "time" 7 | 8 | "github.com/ipfs-cluster/ipfs-cluster/config" 9 | "github.com/kelseyhightower/envconfig" 10 | ) 11 | 12 | const configKey = "tags" 13 | const envConfigKey = "cluster_tags" 14 | 15 | // Default values for tags Config 16 | const ( 17 | DefaultMetricTTL = 30 * time.Second 18 | ) 19 | 20 | // Default values for tags config 21 | var ( 22 | DefaultTags = map[string]string{ 23 | "group": "default", 24 | } 25 | ) 26 | 27 | // Config is used to initialize an Informer and customize 28 | // the type and parameters of the metric it produces. 29 | type Config struct { 30 | config.Saver 31 | 32 | MetricTTL time.Duration 33 | Tags map[string]string 34 | } 35 | 36 | type jsonConfig struct { 37 | MetricTTL string `json:"metric_ttl"` 38 | Tags map[string]string `json:"tags"` 39 | } 40 | 41 | // ConfigKey returns a human-friendly identifier for this type of Metric. 42 | func (cfg *Config) ConfigKey() string { 43 | return configKey 44 | } 45 | 46 | // Default initializes this Config with sensible values. 47 | func (cfg *Config) Default() error { 48 | cfg.MetricTTL = DefaultMetricTTL 49 | cfg.Tags = DefaultTags 50 | return nil 51 | } 52 | 53 | // ApplyEnvVars fills in any Config fields found 54 | // as environment variables. 55 | func (cfg *Config) ApplyEnvVars() error { 56 | jcfg := cfg.toJSONConfig() 57 | 58 | err := envconfig.Process(envConfigKey, jcfg) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | return cfg.applyJSONConfig(jcfg) 64 | } 65 | 66 | // Validate checks that the fields of this Config have working values, 67 | // at least in appearance. 68 | func (cfg *Config) Validate() error { 69 | if cfg.MetricTTL <= 0 { 70 | return errors.New("tags.metric_ttl is invalid") 71 | } 72 | 73 | return nil 74 | } 75 | 76 | // LoadJSON reads the fields of this Config from a JSON byteslice as 77 | // generated by ToJSON. 78 | func (cfg *Config) LoadJSON(raw []byte) error { 79 | jcfg := &jsonConfig{} 80 | err := json.Unmarshal(raw, jcfg) 81 | if err != nil { 82 | logger.Error("Error unmarshaling disk informer config") 83 | return err 84 | } 85 | 86 | cfg.Default() 87 | 88 | return cfg.applyJSONConfig(jcfg) 89 | } 90 | 91 | func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error { 92 | err := config.ParseDurations( 93 | cfg.ConfigKey(), 94 | &config.DurationOpt{Duration: jcfg.MetricTTL, Dst: &cfg.MetricTTL, Name: "metric_ttl"}, 95 | ) 96 | if err != nil { 97 | return err 98 | } 99 | 100 | cfg.Tags = jcfg.Tags 101 | 102 | return cfg.Validate() 103 | } 104 | 105 | // ToJSON generates a JSON-formatted human-friendly representation of this 106 | // Config. 107 | func (cfg *Config) ToJSON() (raw []byte, err error) { 108 | jcfg := cfg.toJSONConfig() 109 | 110 | raw, err = config.DefaultJSONMarshal(jcfg) 111 | return 112 | } 113 | 114 | func (cfg *Config) toJSONConfig() *jsonConfig { 115 | return &jsonConfig{ 116 | MetricTTL: cfg.MetricTTL.String(), 117 | Tags: cfg.Tags, 118 | } 119 | } 120 | 121 | // ToDisplayJSON returns JSON config as a string. 122 | func (cfg *Config) ToDisplayJSON() ([]byte, error) { 123 | return config.DisplayJSON(cfg.toJSONConfig()) 124 | } 125 | -------------------------------------------------------------------------------- /informer/tags/config_test.go: -------------------------------------------------------------------------------- 1 | package tags 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | var cfgJSON = []byte(` 11 | { 12 | "metric_ttl": "1s", 13 | "tags": { "a": "b" } 14 | } 15 | `) 16 | 17 | func TestLoadJSON(t *testing.T) { 18 | cfg := &Config{} 19 | err := cfg.LoadJSON(cfgJSON) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | 24 | if cfg.Tags["a"] != "b" { 25 | t.Fatal("tags not parsed") 26 | } 27 | 28 | j := &jsonConfig{} 29 | json.Unmarshal(cfgJSON, j) 30 | j.MetricTTL = "-10" 31 | tst, _ := json.Marshal(j) 32 | err = cfg.LoadJSON(tst) 33 | if err == nil { 34 | t.Error("expected error decoding metric_ttl") 35 | } 36 | } 37 | 38 | func TestToJSON(t *testing.T) { 39 | cfg := &Config{} 40 | cfg.LoadJSON(cfgJSON) 41 | newjson, err := cfg.ToJSON() 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | cfg = &Config{} 46 | err = cfg.LoadJSON(newjson) 47 | if err != nil { 48 | t.Fatal(err) 49 | } 50 | } 51 | 52 | func TestDefault(t *testing.T) { 53 | cfg := &Config{} 54 | cfg.Default() 55 | if cfg.Validate() != nil { 56 | t.Fatal("error validating") 57 | } 58 | 59 | cfg.MetricTTL = 0 60 | if cfg.Validate() == nil { 61 | t.Fatal("expected error validating") 62 | } 63 | 64 | cfg.Default() 65 | if cfg.Tags["group"] != "default" { 66 | t.Fatal("Tags default not set") 67 | } 68 | } 69 | 70 | func TestApplyEnvVars(t *testing.T) { 71 | os.Setenv("CLUSTER_TAGS_METRICTTL", "22s") 72 | cfg := &Config{} 73 | cfg.ApplyEnvVars() 74 | 75 | if cfg.MetricTTL != 22*time.Second { 76 | t.Fatal("failed to override metric_ttl with env var") 77 | } 78 | 79 | os.Setenv("CLUSTER_TAGS_TAGS", "z:q,y:w") 80 | cfg = &Config{} 81 | cfg.ApplyEnvVars() 82 | 83 | if cfg.Tags["z"] != "q" || cfg.Tags["y"] != "w" { 84 | t.Fatal("could not override tags with env vars") 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /informer/tags/tags.go: -------------------------------------------------------------------------------- 1 | // Package tags implements an ipfs-cluster informer publishes user-defined 2 | // tags as metrics. 3 | package tags 4 | 5 | import ( 6 | "context" 7 | "sync" 8 | 9 | "github.com/ipfs-cluster/ipfs-cluster/api" 10 | 11 | logging "github.com/ipfs/go-log/v2" 12 | rpc "github.com/libp2p/go-libp2p-gorpc" 13 | ) 14 | 15 | var logger = logging.Logger("tags") 16 | 17 | // MetricName specifies the name of our metric 18 | var MetricName = "tags" 19 | 20 | // Informer is a simple object to implement the ipfscluster.Informer 21 | // and Component interfaces. 22 | type Informer struct { 23 | config *Config // set when created, readonly 24 | 25 | mu sync.Mutex // guards access to following fields 26 | rpcClient *rpc.Client 27 | } 28 | 29 | // New returns an initialized informer using the given InformerConfig. 30 | func New(cfg *Config) (*Informer, error) { 31 | err := cfg.Validate() 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | return &Informer{ 37 | config: cfg, 38 | }, nil 39 | } 40 | 41 | // Name returns the name of this informer. Note the informer issues metrics 42 | // with custom names. 43 | func (tags *Informer) Name() string { 44 | return MetricName 45 | } 46 | 47 | // SetClient provides us with an rpc.Client which allows 48 | // contacting other components in the cluster. 49 | func (tags *Informer) SetClient(c *rpc.Client) { 50 | tags.mu.Lock() 51 | defer tags.mu.Unlock() 52 | tags.rpcClient = c 53 | } 54 | 55 | // Shutdown is called on cluster shutdown. We just invalidate 56 | // any metrics from this point. 57 | func (tags *Informer) Shutdown(ctx context.Context) error { 58 | tags.mu.Lock() 59 | defer tags.mu.Unlock() 60 | 61 | tags.rpcClient = nil 62 | return nil 63 | } 64 | 65 | // GetMetrics returns one metric for each tag defined in the configuration. 66 | // The metric name is set as "tags:". When no tags are defined, 67 | // a single invalid metric is returned. 68 | func (tags *Informer) GetMetrics(ctx context.Context) []api.Metric { 69 | // Note we could potentially extend the tag:value syntax to include manual weights 70 | // ie: { "region": "us:100", ... } 71 | // This would potentially allow to always give priority to peers of a certain group 72 | 73 | if len(tags.config.Tags) == 0 { 74 | logger.Debug("no tags defined in tags informer") 75 | m := api.Metric{ 76 | Name: "tag:none", 77 | Value: "", 78 | Valid: false, 79 | Partitionable: true, 80 | } 81 | m.SetTTL(tags.config.MetricTTL) 82 | return []api.Metric{m} 83 | } 84 | 85 | metrics := make([]api.Metric, 0, len(tags.config.Tags)) 86 | for n, v := range tags.config.Tags { 87 | m := api.Metric{ 88 | Name: "tag:" + n, 89 | Value: v, 90 | Valid: true, 91 | Partitionable: true, 92 | } 93 | m.SetTTL(tags.config.MetricTTL) 94 | metrics = append(metrics, m) 95 | } 96 | 97 | return metrics 98 | } 99 | -------------------------------------------------------------------------------- /informer/tags/tags_test.go: -------------------------------------------------------------------------------- 1 | package tags 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | func Test(t *testing.T) { 9 | ctx := context.Background() 10 | cfg := &Config{} 11 | cfg.Default() 12 | inf, err := New(cfg) 13 | if err != nil { 14 | t.Fatal(err) 15 | } 16 | defer inf.Shutdown(ctx) 17 | m := inf.GetMetrics(ctx) 18 | if len(m) != 1 || !m[0].Valid { 19 | t.Error("metric should be valid") 20 | } 21 | 22 | inf.config.Tags["x"] = "y" 23 | m = inf.GetMetrics(ctx) 24 | if len(m) != 2 { 25 | t.Error("there should be 2 metrics") 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /internal/fd/fd.go: -------------------------------------------------------------------------------- 1 | // Package fd is used to obtain the available number of file descriptors. 2 | package fd 3 | -------------------------------------------------------------------------------- /internal/fd/sys_not_unix.go: -------------------------------------------------------------------------------- 1 | //go:build !(aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris) 2 | 3 | package fd 4 | 5 | import "math" 6 | 7 | // GetNumFDs returns the File Descriptors for non unix systems as MaxInt. 8 | func GetNumFDs() uint64 { 9 | return uint64(math.MaxUint64) 10 | } 11 | -------------------------------------------------------------------------------- /internal/fd/sys_unix.go: -------------------------------------------------------------------------------- 1 | //go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris 2 | 3 | package fd 4 | 5 | import ( 6 | "golang.org/x/sys/unix" 7 | ) 8 | 9 | // GetNumFDs returns the File Descriptors limit. 10 | func GetNumFDs() uint64 { 11 | var l unix.Rlimit 12 | if err := unix.Getrlimit(unix.RLIMIT_NOFILE, &l); err != nil { 13 | return 0 14 | } 15 | return uint64(l.Cur) 16 | } 17 | -------------------------------------------------------------------------------- /ipfs-cluster.fundring: -------------------------------------------------------------------------------- 1 | 0x03eff7d844423abee393015d14647744d51f59124a48a4ff68123e36a343a2255649d7fee552b3e35db8d9ac7b512dd168f5451bbf97e86e0a5f049c635a94261c -------------------------------------------------------------------------------- /ipfsconn/ipfshttp/config_test.go: -------------------------------------------------------------------------------- 1 | package ipfshttp 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | var cfgJSON = []byte(` 11 | { 12 | "node_multiaddress": "/ip4/127.0.0.1/tcp/5001", 13 | "connect_swarms_delay": "7s", 14 | "ipfs_request_timeout": "5m0s", 15 | "pin_timeout": "2m", 16 | "unpin_timeout": "3h", 17 | "repogc_timeout": "24h", 18 | "informer_trigger_interval": 10 19 | } 20 | `) 21 | 22 | func TestLoadJSON(t *testing.T) { 23 | cfg := &Config{} 24 | err := cfg.LoadJSON(cfgJSON) 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | 29 | j := &jsonConfig{} 30 | json.Unmarshal(cfgJSON, j) 31 | 32 | if cfg.InformerTriggerInterval != 10 { 33 | t.Error("missing value") 34 | } 35 | 36 | j.NodeMultiaddress = "abc" 37 | tst, _ := json.Marshal(j) 38 | err = cfg.LoadJSON(tst) 39 | if err == nil { 40 | t.Error("expected error in node_multiaddress") 41 | } 42 | } 43 | 44 | func TestToJSON(t *testing.T) { 45 | cfg := &Config{} 46 | cfg.LoadJSON(cfgJSON) 47 | newjson, err := cfg.ToJSON() 48 | if err != nil { 49 | t.Fatal(err) 50 | } 51 | cfg = &Config{} 52 | err = cfg.LoadJSON(newjson) 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | } 57 | 58 | func TestDefault(t *testing.T) { 59 | cfg := &Config{} 60 | cfg.Default() 61 | if cfg.Validate() != nil { 62 | t.Fatal("error validating") 63 | } 64 | 65 | cfg.NodeAddr = nil 66 | if cfg.Validate() == nil { 67 | t.Fatal("expected error validating") 68 | } 69 | } 70 | 71 | func TestApplyEnvVar(t *testing.T) { 72 | os.Setenv("CLUSTER_IPFSHTTP_PINTIMEOUT", "22m") 73 | cfg := &Config{} 74 | cfg.Default() 75 | cfg.ApplyEnvVars() 76 | 77 | if cfg.PinTimeout != 22*time.Minute { 78 | t.Fatal("failed to override pin_timeout with env var") 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /logging.go: -------------------------------------------------------------------------------- 1 | package ipfscluster 2 | 3 | import ( 4 | logging "github.com/ipfs/go-log/v2" 5 | ) 6 | 7 | var logger = logging.Logger("cluster") 8 | 9 | // LoggingFacilities provides a list of logging identifiers 10 | // used by cluster and their default logging level. 11 | var LoggingFacilities = map[string]string{ 12 | "cluster": "INFO", 13 | "restapi": "INFO", 14 | "restapilog": "INFO", 15 | "pinsvcapi": "INFO", 16 | "pinsvcapilog": "INFO", 17 | "ipfsproxy": "INFO", 18 | "ipfsproxylog": "INFO", 19 | "ipfshttp": "INFO", 20 | "monitor": "INFO", 21 | "dsstate": "INFO", 22 | "raft": "INFO", 23 | "crdt": "INFO", 24 | "pintracker": "INFO", 25 | "diskinfo": "INFO", 26 | "tags": "INFO", 27 | "apitypes": "INFO", 28 | "config": "INFO", 29 | "shardingdags": "INFO", 30 | "singledags": "INFO", 31 | "adder": "INFO", 32 | "optracker": "INFO", 33 | "pstoremgr": "INFO", 34 | "allocator": "INFO", 35 | } 36 | 37 | // LoggingFacilitiesExtra provides logging identifiers 38 | // used in ipfs-cluster dependencies, which may be useful 39 | // to display. Along with their default value. 40 | var LoggingFacilitiesExtra = map[string]string{ 41 | "p2p-gorpc": "ERROR", 42 | "swarm2": "ERROR", 43 | "libp2p-raft": "FATAL", 44 | "raftlib": "ERROR", 45 | "badger": "INFO", 46 | "badger3": "INFO", 47 | "pebble": "WARN", // pebble logs with INFO and FATAL only 48 | } 49 | 50 | // SetFacilityLogLevel sets the log level for a given module 51 | func SetFacilityLogLevel(f, l string) { 52 | /* 53 | case "debug", "DEBUG": 54 | *l = DebugLevel 55 | case "info", "INFO", "": // make the zero value useful 56 | *l = InfoLevel 57 | case "warn", "WARN": 58 | *l = WarnLevel 59 | case "error", "ERROR": 60 | *l = ErrorLevel 61 | case "dpanic", "DPANIC": 62 | *l = DPanicLevel 63 | case "panic", "PANIC": 64 | *l = PanicLevel 65 | case "fatal", "FATAL": 66 | *l = FatalLevel 67 | */ 68 | logging.SetLogLevel(f, l) 69 | } 70 | -------------------------------------------------------------------------------- /monitor/metrics/store_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/ipfs-cluster/ipfs-cluster/api" 8 | "github.com/ipfs-cluster/ipfs-cluster/test" 9 | ) 10 | 11 | func TestStoreLatest(t *testing.T) { 12 | store := NewStore() 13 | 14 | metr := api.Metric{ 15 | Name: "test", 16 | Peer: test.PeerID1, 17 | Value: "1", 18 | Valid: true, 19 | } 20 | metr.SetTTL(200 * time.Millisecond) 21 | store.Add(metr) 22 | 23 | latest := store.LatestValid("test") 24 | if len(latest) != 1 { 25 | t.Error("expected 1 metric") 26 | } 27 | 28 | time.Sleep(220 * time.Millisecond) 29 | 30 | latest = store.LatestValid("test") 31 | if len(latest) != 0 { 32 | t.Error("expected no metrics") 33 | } 34 | } 35 | 36 | func TestRemovePeer(t *testing.T) { 37 | store := NewStore() 38 | 39 | metr := api.Metric{ 40 | Name: "test", 41 | Peer: test.PeerID1, 42 | Value: "1", 43 | Valid: true, 44 | } 45 | metr.SetTTL(200 * time.Millisecond) 46 | store.Add(metr) 47 | 48 | if pmtrs := store.PeerMetrics(test.PeerID1); len(pmtrs) <= 0 { 49 | t.Errorf("there should be one peer metric; got: %v", pmtrs) 50 | } 51 | store.RemovePeer(test.PeerID1) 52 | if pmtrs := store.PeerMetrics(test.PeerID1); len(pmtrs) > 0 { 53 | t.Errorf("there should be no peer metrics; got: %v", pmtrs) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /monitor/metrics/util.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "github.com/ipfs-cluster/ipfs-cluster/api" 5 | 6 | peer "github.com/libp2p/go-libp2p/core/peer" 7 | ) 8 | 9 | // PeersetFilter removes all metrics not belonging to the given 10 | // peerset 11 | func PeersetFilter(metrics []api.Metric, peerset []peer.ID) []api.Metric { 12 | peerMap := make(map[peer.ID]struct{}) 13 | for _, pid := range peerset { 14 | peerMap[pid] = struct{}{} 15 | } 16 | 17 | filtered := make([]api.Metric, 0, len(metrics)) 18 | 19 | for _, metric := range metrics { 20 | _, ok := peerMap[metric.Peer] 21 | if !ok { 22 | continue 23 | } 24 | filtered = append(filtered, metric) 25 | } 26 | 27 | return filtered 28 | } 29 | -------------------------------------------------------------------------------- /monitor/metrics/window.go: -------------------------------------------------------------------------------- 1 | // Package metrics provides common functionality for working with metrics, 2 | // particularly useful for monitoring components. It includes types to store, 3 | // check and filter metrics. 4 | package metrics 5 | 6 | import ( 7 | "container/ring" 8 | "errors" 9 | "sync" 10 | "time" 11 | 12 | "github.com/ipfs-cluster/ipfs-cluster/api" 13 | ) 14 | 15 | // DefaultWindowCap sets the amount of metrics to store per peer. 16 | var DefaultWindowCap = 25 17 | 18 | // ErrNoMetrics is returned when there are no metrics in a Window. 19 | var ErrNoMetrics = errors.New("no metrics have been added to this window") 20 | 21 | // Window implements a circular queue to store metrics. 22 | type Window struct { 23 | wMu sync.RWMutex 24 | window *ring.Ring 25 | } 26 | 27 | // NewWindow creates an instance with the given 28 | // window capacity. 29 | func NewWindow(windowCap int) *Window { 30 | if windowCap <= 0 { 31 | panic("invalid windowCap") 32 | } 33 | 34 | w := ring.New(windowCap) 35 | return &Window{ 36 | window: w, 37 | } 38 | } 39 | 40 | // Add adds a new metric to the window. If the window capacity 41 | // has been reached, the oldest metric (by the time it was added), 42 | // will be discarded. Add leaves the cursor on the next spot, 43 | // which is either empty or the oldest record. 44 | func (mw *Window) Add(m api.Metric) { 45 | m.ReceivedAt = time.Now().UnixNano() 46 | 47 | mw.wMu.Lock() 48 | mw.window.Value = m 49 | mw.window = mw.window.Next() 50 | mw.wMu.Unlock() 51 | } 52 | 53 | // Latest returns the last metric added. It returns an error 54 | // if no metrics were added. 55 | func (mw *Window) Latest() (api.Metric, error) { 56 | var last api.Metric 57 | var ok bool 58 | 59 | mw.wMu.RLock() 60 | // This just returns the previous ring and 61 | // doesn't set the window "cursor" to the previous 62 | // ring. Therefore this is just a read operation 63 | // as well. 64 | prevRing := mw.window.Prev() 65 | mw.wMu.RUnlock() 66 | 67 | last, ok = prevRing.Value.(api.Metric) 68 | 69 | if !ok || !last.Defined() { 70 | return last, ErrNoMetrics 71 | } 72 | 73 | return last, nil 74 | } 75 | 76 | // All returns all the metrics in the window, in the inverse order 77 | // they were Added. That is, result[0] will be the last added 78 | // metric. 79 | func (mw *Window) All() []api.Metric { 80 | values := make([]api.Metric, 0, mw.window.Len()) 81 | 82 | mw.wMu.RLock() 83 | mw.window.Do(func(v interface{}) { 84 | i, ok := v.(api.Metric) 85 | if ok { 86 | // append younger values to older value 87 | values = append([]api.Metric{i}, values...) 88 | } 89 | }) 90 | mw.wMu.RUnlock() 91 | 92 | return values 93 | } 94 | -------------------------------------------------------------------------------- /monitor/pubsubmon/config.go: -------------------------------------------------------------------------------- 1 | package pubsubmon 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "time" 7 | 8 | "github.com/ipfs-cluster/ipfs-cluster/config" 9 | "github.com/kelseyhightower/envconfig" 10 | ) 11 | 12 | const configKey = "pubsubmon" 13 | const envConfigKey = "cluster_pubsubmon" 14 | 15 | // Default values for this Config. 16 | const ( 17 | DefaultCheckInterval = 15 * time.Second 18 | ) 19 | 20 | // Config allows to initialize a Monitor and customize some parameters. 21 | type Config struct { 22 | config.Saver 23 | 24 | CheckInterval time.Duration 25 | } 26 | 27 | type jsonConfig struct { 28 | CheckInterval string `json:"check_interval"` 29 | } 30 | 31 | // ConfigKey provides a human-friendly identifier for this type of Config. 32 | func (cfg *Config) ConfigKey() string { 33 | return configKey 34 | } 35 | 36 | // Default sets the fields of this Config to sensible values. 37 | func (cfg *Config) Default() error { 38 | cfg.CheckInterval = DefaultCheckInterval 39 | return nil 40 | } 41 | 42 | // ApplyEnvVars fills in any Config fields found 43 | // as environment variables. 44 | func (cfg *Config) ApplyEnvVars() error { 45 | jcfg := cfg.toJSONConfig() 46 | 47 | err := envconfig.Process(envConfigKey, jcfg) 48 | if err != nil { 49 | return err 50 | } 51 | 52 | return cfg.applyJSONConfig(jcfg) 53 | } 54 | 55 | // Validate checks that the fields of this Config have working values, 56 | // at least in appearance. 57 | func (cfg *Config) Validate() error { 58 | if cfg.CheckInterval <= 0 { 59 | return errors.New("pubsubmon.check_interval too low") 60 | } 61 | 62 | return nil 63 | } 64 | 65 | // LoadJSON sets the fields of this Config to the values defined by the JSON 66 | // representation of it, as generated by ToJSON. 67 | func (cfg *Config) LoadJSON(raw []byte) error { 68 | jcfg := &jsonConfig{} 69 | err := json.Unmarshal(raw, jcfg) 70 | if err != nil { 71 | logger.Error("Error unmarshaling pubsubmon monitor config") 72 | return err 73 | } 74 | 75 | cfg.Default() 76 | 77 | return cfg.applyJSONConfig(jcfg) 78 | } 79 | 80 | func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error { 81 | interval, _ := time.ParseDuration(jcfg.CheckInterval) 82 | cfg.CheckInterval = interval 83 | 84 | return cfg.Validate() 85 | } 86 | 87 | // ToJSON generates a human-friendly JSON representation of this Config. 88 | func (cfg *Config) ToJSON() ([]byte, error) { 89 | jcfg := cfg.toJSONConfig() 90 | 91 | return json.MarshalIndent(jcfg, "", " ") 92 | } 93 | 94 | func (cfg *Config) toJSONConfig() *jsonConfig { 95 | return &jsonConfig{ 96 | CheckInterval: cfg.CheckInterval.String(), 97 | } 98 | } 99 | 100 | // ToDisplayJSON returns JSON config as a string. 101 | func (cfg *Config) ToDisplayJSON() ([]byte, error) { 102 | return config.DisplayJSON(cfg.toJSONConfig()) 103 | } 104 | -------------------------------------------------------------------------------- /monitor/pubsubmon/config_test.go: -------------------------------------------------------------------------------- 1 | package pubsubmon 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | var cfgJSON = []byte(` 11 | { 12 | "check_interval": "15s" 13 | } 14 | `) 15 | 16 | func TestLoadJSON(t *testing.T) { 17 | cfg := &Config{} 18 | err := cfg.LoadJSON(cfgJSON) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | j := &jsonConfig{} 24 | 25 | json.Unmarshal(cfgJSON, j) 26 | j.CheckInterval = "-10" 27 | tst, _ := json.Marshal(j) 28 | err = cfg.LoadJSON(tst) 29 | if err == nil { 30 | t.Error("expected error decoding check_interval") 31 | } 32 | } 33 | 34 | func TestToJSON(t *testing.T) { 35 | cfg := &Config{} 36 | cfg.LoadJSON(cfgJSON) 37 | newjson, err := cfg.ToJSON() 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | cfg = &Config{} 42 | err = cfg.LoadJSON(newjson) 43 | if err != nil { 44 | t.Fatal(err) 45 | } 46 | } 47 | 48 | func TestDefault(t *testing.T) { 49 | cfg := &Config{} 50 | cfg.Default() 51 | if cfg.Validate() != nil { 52 | t.Fatal("error validating") 53 | } 54 | 55 | cfg.CheckInterval = 0 56 | if cfg.Validate() == nil { 57 | t.Fatal("expected error validating") 58 | } 59 | } 60 | 61 | func TestApplyEnvVars(t *testing.T) { 62 | os.Setenv("CLUSTER_PUBSUBMON_CHECKINTERVAL", "22s") 63 | cfg := &Config{} 64 | cfg.ApplyEnvVars() 65 | 66 | if cfg.CheckInterval != 22*time.Second { 67 | t.Fatal("failed to override check_interval with env var") 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /observations/config_test.go: -------------------------------------------------------------------------------- 1 | package observations 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestApplyEnvVars(t *testing.T) { 9 | os.Setenv("CLUSTER_METRICS_ENABLESTATS", "true") 10 | mcfg := &MetricsConfig{} 11 | mcfg.Default() 12 | mcfg.ApplyEnvVars() 13 | 14 | if !mcfg.EnableStats { 15 | t.Fatal("failed to override enable_stats with env var") 16 | } 17 | 18 | os.Setenv("CLUSTER_TRACING_ENABLETRACING", "true") 19 | tcfg := &TracingConfig{} 20 | tcfg.Default() 21 | tcfg.ApplyEnvVars() 22 | 23 | if !tcfg.EnableTracing { 24 | t.Fatal("failed to override enable_tracing with env var") 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /pintracker/optracker/operation_test.go: -------------------------------------------------------------------------------- 1 | package optracker 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | "time" 8 | 9 | "github.com/ipfs-cluster/ipfs-cluster/api" 10 | "github.com/ipfs-cluster/ipfs-cluster/test" 11 | ) 12 | 13 | func TestOperation(t *testing.T) { 14 | tim := time.Now().Add(-2 * time.Second) 15 | op := newOperation(context.Background(), api.PinCid(test.Cid1), OperationUnpin, PhaseQueued, nil) 16 | if !op.Cid().Equals(test.Cid1) { 17 | t.Error("bad cid") 18 | } 19 | if op.Phase() != PhaseQueued { 20 | t.Error("bad phase") 21 | } 22 | 23 | op.SetError(errors.New("fake error")) 24 | if op.Error() != "fake error" { 25 | t.Error("bad error") 26 | } 27 | 28 | op.SetPhase(PhaseInProgress) 29 | if op.Phase() != PhaseInProgress { 30 | t.Error("bad phase") 31 | } 32 | 33 | if op.Type() != OperationUnpin { 34 | t.Error("bad type") 35 | } 36 | 37 | if !op.Timestamp().After(tim) { 38 | t.Error("bad timestamp") 39 | } 40 | 41 | if op.Canceled() { 42 | t.Error("should not be canceled") 43 | } 44 | 45 | op.Cancel() 46 | if !op.Canceled() { 47 | t.Error("should be canceled") 48 | } 49 | 50 | if op.ToTrackerStatus() != api.TrackerStatusUnpinning { 51 | t.Error("should be in unpin error") 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /pintracker/optracker/operationtype_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=OperationType"; DO NOT EDIT. 2 | 3 | package optracker 4 | 5 | import "strconv" 6 | 7 | const _OperationType_name = "OperationUnknownOperationPinOperationUnpinOperationRemoteOperationShard" 8 | 9 | var _OperationType_index = [...]uint8{0, 16, 28, 42, 57, 71} 10 | 11 | func (i OperationType) String() string { 12 | if i < 0 || i >= OperationType(len(_OperationType_index)-1) { 13 | return "OperationType(" + strconv.FormatInt(int64(i), 10) + ")" 14 | } 15 | return _OperationType_name[_OperationType_index[i]:_OperationType_index[i+1]] 16 | } 17 | -------------------------------------------------------------------------------- /pintracker/optracker/phase_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=Phase"; DO NOT EDIT. 2 | 3 | package optracker 4 | 5 | import "strconv" 6 | 7 | const _Phase_name = "PhaseErrorPhaseQueuedPhaseInProgressPhaseDone" 8 | 9 | var _Phase_index = [...]uint8{0, 10, 21, 36, 45} 10 | 11 | func (i Phase) String() string { 12 | if i < 0 || i >= Phase(len(_Phase_index)-1) { 13 | return "Phase(" + strconv.FormatInt(int64(i), 10) + ")" 14 | } 15 | return _Phase_name[_Phase_index[i]:_Phase_index[i+1]] 16 | } 17 | -------------------------------------------------------------------------------- /pintracker/stateless/config_test.go: -------------------------------------------------------------------------------- 1 | package stateless 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | var cfgJSON = []byte(` 11 | { 12 | "max_pin_queue_size": 4092, 13 | "concurrent_pins": 2, 14 | "priority_pin_max_age": "240h", 15 | "priority_pin_max_retries": 4 16 | } 17 | `) 18 | 19 | func TestLoadJSON(t *testing.T) { 20 | cfg := &Config{} 21 | err := cfg.LoadJSON(cfgJSON) 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | 26 | j := &jsonConfig{} 27 | 28 | json.Unmarshal(cfgJSON, j) 29 | j.ConcurrentPins = 10 30 | j.PriorityPinMaxAge = "216h" 31 | j.PriorityPinMaxRetries = 2 32 | tst, _ := json.Marshal(j) 33 | err = cfg.LoadJSON(tst) 34 | if err != nil { 35 | t.Error("did not expect an error") 36 | } 37 | if cfg.ConcurrentPins != 10 { 38 | t.Error("expected 10 concurrent pins") 39 | } 40 | if cfg.PriorityPinMaxAge != 9*24*time.Hour { 41 | t.Error("expected 9 days max age") 42 | } 43 | if cfg.PriorityPinMaxRetries != 2 { 44 | t.Error("expected 2 max retries") 45 | } 46 | } 47 | 48 | func TestToJSON(t *testing.T) { 49 | cfg := &Config{} 50 | cfg.LoadJSON(cfgJSON) 51 | newjson, err := cfg.ToJSON() 52 | if err != nil { 53 | t.Fatal(err) 54 | } 55 | cfg = &Config{} 56 | err = cfg.LoadJSON(newjson) 57 | if err != nil { 58 | t.Fatal(err) 59 | } 60 | } 61 | 62 | func TestDefault(t *testing.T) { 63 | cfg := &Config{} 64 | cfg.Default() 65 | if cfg.Validate() != nil { 66 | t.Fatal("error validating") 67 | } 68 | 69 | cfg.ConcurrentPins = -2 70 | if cfg.Validate() == nil { 71 | t.Fatal("expected error validating") 72 | } 73 | cfg.ConcurrentPins = 3 74 | cfg.PriorityPinMaxRetries = -1 75 | if cfg.Validate() == nil { 76 | t.Fatal("expected error validating") 77 | } 78 | } 79 | 80 | func TestApplyEnvVars(t *testing.T) { 81 | os.Setenv("CLUSTER_STATELESS_CONCURRENTPINS", "22") 82 | os.Setenv("CLUSTER_STATELESS_PRIORITYPINMAXAGE", "72h") 83 | cfg := &Config{} 84 | cfg.ApplyEnvVars() 85 | 86 | if cfg.ConcurrentPins != 22 { 87 | t.Fatal("failed to override concurrent_pins with env var") 88 | } 89 | 90 | if cfg.PriorityPinMaxAge != 3*24*time.Hour { 91 | t.Fatal("failed to override priority_pin_max_age with env var") 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /pnet_test.go: -------------------------------------------------------------------------------- 1 | package ipfscluster 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | func TestClusterSecretFormat(t *testing.T) { 9 | goodSecret := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" 10 | emptySecret := "" 11 | tooShort := "0123456789abcdef" 12 | tooLong := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0" 13 | unsupportedChars := "0123456789abcdef0123456789!!!!!!0123456789abcdef0123456789abcdef" 14 | 15 | _, err := DecodeClusterSecret(goodSecret) 16 | if err != nil { 17 | t.Fatal("Failed to decode well-formatted secret.") 18 | } 19 | decodedEmptySecret, err := DecodeClusterSecret(emptySecret) 20 | if decodedEmptySecret != nil || err != nil { 21 | t.Fatal("Unsuspected output of decoding empty secret.") 22 | } 23 | _, err = DecodeClusterSecret(tooShort) 24 | if err == nil { 25 | t.Fatal("Successfully decoded secret that should haved failed (too short).") 26 | } 27 | _, err = DecodeClusterSecret(tooLong) 28 | if err == nil { 29 | t.Fatal("Successfully decoded secret that should haved failed (too long).") 30 | } 31 | _, err = DecodeClusterSecret(unsupportedChars) 32 | if err == nil { 33 | t.Fatal("Successfully decoded secret that should haved failed (unsupported chars).") 34 | } 35 | } 36 | 37 | func TestSimplePNet(t *testing.T) { 38 | ctx := context.Background() 39 | clusters, mocks, boot := peerManagerClusters(t) 40 | defer cleanState() 41 | defer shutdownClusters(t, clusters, mocks) 42 | defer boot.Close() 43 | 44 | if len(clusters) < 2 { 45 | t.Skip("need at least 2 nodes for this test") 46 | } 47 | 48 | _, err := clusters[0].PeerAdd(ctx, clusters[1].id) 49 | if err != nil { 50 | t.Fatal(err) 51 | } 52 | ttlDelay() 53 | 54 | if len(peers(ctx, t, clusters[0])) != len(peers(ctx, t, clusters[1])) { 55 | t.Fatal("Expected same number of peers") 56 | } 57 | if len(peers(ctx, t, clusters[0])) < 2 { 58 | // crdt mode has auto discovered all peers at this point. 59 | // Raft mode has 2 peers only. 60 | t.Fatal("Expected at least 2 peers") 61 | } 62 | } 63 | 64 | // // Adds one minute to tests. Disabled for the moment. 65 | // func TestClusterSecretRequired(t *testing.T) { 66 | // cl1Secret, err := pnet.GenerateV1Bytes() 67 | // if err != nil { 68 | // t.Fatal("Unable to generate cluster secret.") 69 | // } 70 | // cl1, _ := createOnePeerCluster(t, 1, (*cl1Secret)[:]) 71 | // cl2, _ := createOnePeerCluster(t, 2, testingClusterSecret) 72 | // defer cleanState() 73 | // defer cl1.Shutdown() 74 | // defer cl2.Shutdown() 75 | // peers1 := cl1.Peers() 76 | // peers2 := cl2.Peers() 77 | // 78 | // _, err = cl1.PeerAdd(clusterAddr(cl2)) 79 | // if err == nil { 80 | // t.Fatal("Peer entered private cluster without key.") 81 | // } 82 | 83 | // if len(peers1) != len(peers2) { 84 | // t.Fatal("Expected same number of peers") 85 | // } 86 | // if len(peers1) != 1 { 87 | // t.Fatal("Expected no peers other than self") 88 | // } 89 | // } 90 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Updates the Version variables, commits, tags and signs 4 | 5 | set -eux 6 | 7 | version="$1" 8 | 9 | if [ -z $version ]; then 10 | echo "Need a version!" 11 | exit 1 12 | fi 13 | 14 | make clean 15 | sed -i "s/Version = semver\.MustParse.*$/Version = semver.MustParse(\"$version\")/" version/version.go 16 | sed -i "s/const Version.*$/const Version = \"$version\"/" cmd/ipfs-cluster-ctl/main.go 17 | git add version/version.go cmd/ipfs-cluster-ctl/main.go 18 | 19 | # Next versions, just commit 20 | if [[ "$version" == *"-next" ]]; then 21 | git commit -S -m "Set development version v${version}" 22 | exit 0 23 | fi 24 | 25 | # RC versions, commit and make a tag without history. 26 | if [[ "$version" == *"-rc"* ]]; then 27 | git commit -S -m "Release candidate v${version}" 28 | git tag -s "v${version}" 29 | exit 0 30 | fi 31 | 32 | # Actual releases, commit and make an annotated tag with all the commits 33 | # since the last. 34 | git commit -S -m "Release v${version}" 35 | lastver=`git describe --abbrev=0 --exclude '*rc*'` 36 | echo "Tag for Release ${version}" > tag_annotation 37 | echo >> tag_annotation 38 | git log --pretty=oneline ${lastver}..HEAD >> tag_annotation 39 | git tag -a -s -F tag_annotation "v${version}" 40 | rm tag_annotation 41 | -------------------------------------------------------------------------------- /rpc_policy.go: -------------------------------------------------------------------------------- 1 | package ipfscluster 2 | 3 | // This file can be generated with rpcutil/policygen. 4 | 5 | // DefaultRPCPolicy associates all rpc endpoints offered by cluster peers to an 6 | // endpoint type. See rpcutil/policygen.go as a quick way to generate this 7 | // without missing any endpoint. 8 | var DefaultRPCPolicy = map[string]RPCEndpointType{ 9 | // Cluster methods 10 | "Cluster.Alerts": RPCClosed, 11 | "Cluster.BandwidthByProtocol": RPCClosed, 12 | "Cluster.BlockAllocate": RPCClosed, 13 | "Cluster.ConnectGraph": RPCClosed, 14 | "Cluster.ID": RPCOpen, 15 | "Cluster.IDStream": RPCOpen, 16 | "Cluster.IPFSID": RPCClosed, 17 | "Cluster.Join": RPCClosed, 18 | "Cluster.PeerAdd": RPCOpen, // Used by Join() 19 | "Cluster.PeerRemove": RPCTrusted, 20 | "Cluster.Peers": RPCTrusted, // Used by ConnectGraph() 21 | "Cluster.PeersWithFilter": RPCClosed, 22 | "Cluster.Pin": RPCClosed, 23 | "Cluster.PinGet": RPCClosed, 24 | "Cluster.PinPath": RPCClosed, 25 | "Cluster.Pins": RPCClosed, // Used in stateless tracker, ipfsproxy, restapi 26 | "Cluster.Recover": RPCClosed, 27 | "Cluster.RecoverAll": RPCClosed, 28 | "Cluster.RecoverAllLocal": RPCTrusted, 29 | "Cluster.RecoverLocal": RPCTrusted, 30 | "Cluster.RepoGC": RPCClosed, 31 | "Cluster.RepoGCLocal": RPCTrusted, 32 | "Cluster.SendInformerMetrics": RPCClosed, 33 | "Cluster.SendInformersMetrics": RPCClosed, 34 | "Cluster.Status": RPCClosed, 35 | "Cluster.StatusAll": RPCClosed, 36 | "Cluster.StatusAllLocal": RPCClosed, 37 | "Cluster.StatusLocal": RPCClosed, 38 | "Cluster.Unpin": RPCClosed, 39 | "Cluster.UnpinPath": RPCClosed, 40 | "Cluster.Version": RPCOpen, 41 | 42 | // PinTracker methods 43 | "PinTracker.PinQueueSize": RPCClosed, 44 | "PinTracker.Recover": RPCTrusted, // Called in broadcast from Recover() 45 | "PinTracker.RecoverAll": RPCClosed, // Broadcast in RecoverAll unimplemented 46 | "PinTracker.Status": RPCTrusted, 47 | "PinTracker.StatusAll": RPCTrusted, 48 | "PinTracker.Track": RPCClosed, 49 | "PinTracker.Untrack": RPCClosed, 50 | 51 | // IPFSConnector methods 52 | "IPFSConnector.BlockGet": RPCClosed, 53 | "IPFSConnector.BlockStream": RPCTrusted, // Called by adders 54 | "IPFSConnector.ConfigKey": RPCClosed, 55 | "IPFSConnector.Pin": RPCClosed, 56 | "IPFSConnector.PinLs": RPCClosed, 57 | "IPFSConnector.PinLsCid": RPCClosed, 58 | "IPFSConnector.RepoStat": RPCTrusted, // Called in broadcast from proxy/repo/stat 59 | "IPFSConnector.Resolve": RPCClosed, 60 | "IPFSConnector.SwarmPeers": RPCTrusted, // Called in ConnectGraph 61 | "IPFSConnector.Unpin": RPCClosed, 62 | 63 | // Consensus methods 64 | "Consensus.AddPeer": RPCTrusted, // Called by Raft/redirect to leader 65 | "Consensus.LogPin": RPCTrusted, // Called by Raft/redirect to leader 66 | "Consensus.LogUnpin": RPCTrusted, // Called by Raft/redirect to leader 67 | "Consensus.Peers": RPCClosed, 68 | "Consensus.RmPeer": RPCTrusted, // Called by Raft/redirect to leader 69 | 70 | // PeerMonitor methods 71 | "PeerMonitor.LatestMetrics": RPCClosed, 72 | "PeerMonitor.MetricNames": RPCClosed, 73 | } 74 | 75 | -------------------------------------------------------------------------------- /rpcutil/policygen/policygen.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "go/format" 6 | "os" 7 | "reflect" 8 | "strings" 9 | 10 | cluster "github.com/ipfs-cluster/ipfs-cluster" 11 | ) 12 | 13 | func rpcTypeStr(t cluster.RPCEndpointType) string { 14 | switch t { 15 | case cluster.RPCClosed: 16 | return "RPCClosed" 17 | case cluster.RPCTrusted: 18 | return "RPCTrusted" 19 | case cluster.RPCOpen: 20 | return "RPCOpen" 21 | default: 22 | return "ERROR" 23 | } 24 | } 25 | 26 | var comments = map[string]string{ 27 | "Cluster.PeerAdd": "Used by Join()", 28 | "Cluster.Peers": "Used by ConnectGraph()", 29 | "Cluster.Pins": "Used in stateless tracker, ipfsproxy, restapi", 30 | "PinTracker.Recover": "Called in broadcast from Recover()", 31 | "PinTracker.RecoverAll": "Broadcast in RecoverAll unimplemented", 32 | "Pintracker.Status": "Called in broadcast from Status()", 33 | "Pintracker.StatusAll": "Called in broadcast from StatusAll()", 34 | "IPFSConnector.BlockStream": "Called by adders", 35 | "IPFSConnector.RepoStat": "Called in broadcast from proxy/repo/stat", 36 | "IPFSConnector.SwarmPeers": "Called in ConnectGraph", 37 | "Consensus.AddPeer": "Called by Raft/redirect to leader", 38 | "Consensus.LogPin": "Called by Raft/redirect to leader", 39 | "Consensus.LogUnpin": "Called by Raft/redirect to leader", 40 | "Consensus.RmPeer": "Called by Raft/redirect to leader", 41 | } 42 | 43 | func main() { 44 | rpcComponents := []interface{}{ 45 | &cluster.ClusterRPCAPI{}, 46 | &cluster.PinTrackerRPCAPI{}, 47 | &cluster.IPFSConnectorRPCAPI{}, 48 | &cluster.ConsensusRPCAPI{}, 49 | &cluster.PeerMonitorRPCAPI{}, 50 | } 51 | 52 | fmt.Fprintln(os.Stderr, ` 53 | // The below generated policy keeps the endpoint types 54 | // from the existing one, marking new endpoints as NEW. Redirect stdout 55 | // into ../../rpc_policy.go and set the NEW endpoints to their correct 56 | // type (make sure you have recompiled this binary with the current version 57 | // of the code). If you are redirecting already, and things went fine, you 58 | // should only see this message. 59 | ============================================================================`) 60 | fmt.Fprintln(os.Stderr) 61 | 62 | var rpcPolicyDotGo strings.Builder 63 | 64 | rpcPolicyDotGo.WriteString("package ipfscluster\n\n") 65 | rpcPolicyDotGo.WriteString("// This file can be generated with rpcutil/policygen.\n\n") 66 | rpcPolicyDotGo.WriteString(` 67 | // DefaultRPCPolicy associates all rpc endpoints offered by cluster peers to an 68 | // endpoint type. See rpcutil/policygen.go as a quick way to generate this 69 | // without missing any endpoint.`) 70 | rpcPolicyDotGo.WriteString("\nvar DefaultRPCPolicy = map[string]RPCEndpointType{\n") 71 | 72 | for _, c := range rpcComponents { 73 | t := reflect.TypeOf(c) 74 | 75 | rpcPolicyDotGo.WriteString("// " + cluster.RPCServiceID(c) + " methods\n") 76 | for i := 0; i < t.NumMethod(); i++ { 77 | method := t.Method(i) 78 | name := cluster.RPCServiceID(c) + "." + method.Name 79 | rpcT, ok := cluster.DefaultRPCPolicy[name] 80 | rpcTStr := "NEW" 81 | if ok { 82 | rpcTStr = rpcTypeStr(rpcT) 83 | } 84 | comment, ok := comments[name] 85 | if ok { 86 | comment = "// " + comment 87 | } 88 | 89 | fmt.Fprintf(&rpcPolicyDotGo, "\"%s\": %s, %s\n", name, rpcTStr, comment) 90 | } 91 | rpcPolicyDotGo.WriteString("\n") 92 | } 93 | 94 | rpcPolicyDotGo.WriteString("}\n") 95 | src, err := format.Source([]byte(rpcPolicyDotGo.String())) 96 | if err != nil { 97 | fmt.Fprintln(os.Stderr, err) 98 | os.Exit(1) 99 | } 100 | fmt.Println(string(src)) 101 | } 102 | -------------------------------------------------------------------------------- /sharness/config/basic_auth/identity.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "QmdEtBsfumeH2V6dnx1fgn8zuW7XYjWdgJF4NEYpEBcTsg", 3 | "private_key": "CAASqAkwggSkAgEAAoIBAQC/ZmfWDbwyI0nJdRxgHcTdEaBFQo8sky9E+OOvtwZa5WKoLdHyHOLWxCAdpIHUBbhxz5rkMEWLwPI6ykqLIJToMPO8lJbKVzphOjv4JwpiAPdmeSiYMKLjx5V8MpqU2rwj/Uf3sRL8Gg9/Tei3PZ8cftxN1rkQQeeaOtk0CBxUFZSHEsyut1fbgIeL7TAY+4vCmXW0DBr4wh9fnoES/YivOvSiN9rScgWg6N65LfkI78hzaOJ4Nok2S4vYFCxjTAI9NWFUbhP5eJIFzTU+bZuQZxOn2qsoyw8pNZwuF+JClA/RcgBcCvVZcDH2ueVq/zT++bGCN+EWsAEdvJqJ5bsjAgMBAAECggEAaGDUZ6t94mnUJ4UyQEh7v4OJP7wYkFqEAL0qjfzl/lPyBX1XbQ3Ltwul6AR6uMGV4JszARZCFwDWGLGRDWZrTmTDxyfRQ+9l6vfzFFVWGDQmtz+Dn9uGOWnyX5TJMDxJNec+hBmRHOKpaOd37dYxGz0jr19V9UO7piRJp1J1AHUCypUGv5x1IekioSCu5fEyc7dyWwnmITHBjD08st+bCcjrIUFeXSdJKC8SymYeXdaVE3xH3zVEISKnrfT7bhuKZY1iibZIlXbVLNpyX36LkYJOiCqsMum3u70LH0VvTypkqiDbD4S6qfJ4vvUakpmKpOPutikiP7jkSP+AkaO0AQKBgQDkTuhnDK6+Y0a/HgpHJisji0coO+g2gsIszargHk8nNY2AB8t+EUn7C+Qu8cmrem5V8EXcdxS6z7iAXpJmY1Xepnsz+JP7Q91Lgt3OoqK5EybzUXXKkmNCD65n70Xxn2fEFzm6+GJP3c/HymlDKU2KBCYIyuUeaREjT0Fu3v6tgQKBgQDWnXppJwn4LJHhzFOCeO4zomDJDbLTZCabdKZoFP9r+vtEHAnclDDKx4AYbomSqgERe+DX6HR/tPHRVizP63RYPf7al2mJmPzt1nTkoc1/q5hQoD+oE154dADsW1pUp7AQjwCtys4iq5S0qAwIDpuY8M8bOHwZ+QmBvHYAigJCowKBgQC3HH6TX/2rH463bE2MARXqXSPGJj45sigwrQfW1xhe9zm1LQtN4mn2mvP5nt1D1l82OA6gIzYSGtX8x10eF5/ggqAf78goZ6bOkHh76b8fNzgvQO97eGt5qYAVRjhP8azU/lfEGMEpE1s5/6LrRe41utwSg0C+YkBnlIKDfQDAgQKBgDoBTCF5hK9H1JHzuKpt5uubuo78ndWWnvyrNYKyEirsJddNwLiWcO2NqChyT8qNGkbQdX/Fex89F5KduPTlTYfAEc6g18xxxgK+UM+uj60vArbf6PSTb5gculcnha2VuPdwvx050Cb8uu9s7/uJfzKB+2f/B0O51ID1H+ubYWsDAoGBAKrwGKHyqFTHSPg3XuRA1FgDAoOsfzP9ZJvMEXUWyu/VxjNt+0mRlyGeZ5qb9UZG+K/In4FbC/ux2P/PucCUIbgy/XGPtPXVavMwNbx0MquAcU0FihKXP0CUpi8zwiYc42MF7n/SztQnismxigBMSuJEDurcXXazjfcSRTypduNn" 4 | } -------------------------------------------------------------------------------- /sharness/config/basic_auth/service.json: -------------------------------------------------------------------------------- 1 | { 2 | "cluster": { 3 | "peername": "testname", 4 | "secret": "84399cd0be811c2ca372d6ca473ffd73c09034f991c5e306fe9ada6c5fcfb641", 5 | "leave_on_shutdown": false, 6 | "listen_multiaddress": [ 7 | "/ip4/0.0.0.0/tcp/9096", 8 | "/ip6/::/tcp/9096" 9 | ], 10 | "state_sync_interval": "1m0s", 11 | "replication_factor": -1, 12 | "monitor_ping_interval": "15s" 13 | }, 14 | "consensus": { 15 | "raft": { 16 | "heartbeat_timeout": "1s", 17 | "election_timeout": "1s", 18 | "commit_timeout": "50ms", 19 | "max_append_entries": 64, 20 | "trailing_logs": 10240, 21 | "snapshot_interval": "2m0s", 22 | "snapshot_threshold": 8192, 23 | "leader_lease_timeout": "500ms" 24 | } 25 | }, 26 | "api": { 27 | "ipfsproxy": { 28 | "listen_multiaddress": "/ip4/127.0.0.1/tcp/9095", 29 | "node_multiaddress": "/ip4/127.0.0.1/tcp/5001", 30 | "read_timeout": "10m0s", 31 | "read_header_timeout": "5s", 32 | "write_timeout": "10m0s", 33 | "idle_timeout": "1m0s" 34 | }, 35 | "restapi": { 36 | "ssl_cert_file": "", 37 | "ssl_key_file": "", 38 | "http_listen_multiaddress": "/ip4/127.0.0.1/tcp/9094", 39 | "read_timeout": "30s", 40 | "read_header_timeout": "5s", 41 | "write_timeout": "1m0s", 42 | "idle_timeout": "2m0s", 43 | "basic_auth_credentials": { 44 | "testuser": "testpass" 45 | }, 46 | "cors_allowed_origins": [ 47 | "*" 48 | ], 49 | "cors_allowed_methods": [ 50 | "GET" 51 | ], 52 | "cors_allowed_headers": [], 53 | "cors_exposed_headers": [ 54 | "Content-Type", 55 | "X-Stream-Output", 56 | "X-Chunked-Output", 57 | "X-Content-Length" 58 | ], 59 | "cors_allow_credentials": true, 60 | "cors_max_age": "0s" 61 | } 62 | }, 63 | "ipfs_connector": { 64 | "ipfshttp": { 65 | "node_multiaddress": "/ip4/127.0.0.1/tcp/5001", 66 | "connect_swarms_delay": "30s", 67 | "ipfs_request_timeout": "5m0s", 68 | "pin_timeout": "0h2m0s", 69 | "unpin_timeout": "3h0m0s" 70 | } 71 | }, 72 | "pin_tracker": { 73 | "stateless": { 74 | "max_pin_queue_size": 50000, 75 | "concurrent_pins": 10 76 | } 77 | }, 78 | "monitor": { 79 | "monbasic": { 80 | "check_interval": "15s" 81 | }, 82 | "pubsubmon": { 83 | "check_interval": "15s" 84 | } 85 | }, 86 | "informer": { 87 | "disk": { 88 | "metric_ttl": "30s", 89 | "metric_type": "reposize" 90 | }, 91 | "numpin": { 92 | "metric_ttl": "10s" 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /sharness/config/ssl-basic_auth/identity.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "QmdEtBsfumeH2V6dnx1fgn8zuW7XYjWdgJF4NEYpEBcTsg", 3 | "private_key": "CAASqAkwggSkAgEAAoIBAQC/ZmfWDbwyI0nJdRxgHcTdEaBFQo8sky9E+OOvtwZa5WKoLdHyHOLWxCAdpIHUBbhxz5rkMEWLwPI6ykqLIJToMPO8lJbKVzphOjv4JwpiAPdmeSiYMKLjx5V8MpqU2rwj/Uf3sRL8Gg9/Tei3PZ8cftxN1rkQQeeaOtk0CBxUFZSHEsyut1fbgIeL7TAY+4vCmXW0DBr4wh9fnoES/YivOvSiN9rScgWg6N65LfkI78hzaOJ4Nok2S4vYFCxjTAI9NWFUbhP5eJIFzTU+bZuQZxOn2qsoyw8pNZwuF+JClA/RcgBcCvVZcDH2ueVq/zT++bGCN+EWsAEdvJqJ5bsjAgMBAAECggEAaGDUZ6t94mnUJ4UyQEh7v4OJP7wYkFqEAL0qjfzl/lPyBX1XbQ3Ltwul6AR6uMGV4JszARZCFwDWGLGRDWZrTmTDxyfRQ+9l6vfzFFVWGDQmtz+Dn9uGOWnyX5TJMDxJNec+hBmRHOKpaOd37dYxGz0jr19V9UO7piRJp1J1AHUCypUGv5x1IekioSCu5fEyc7dyWwnmITHBjD08st+bCcjrIUFeXSdJKC8SymYeXdaVE3xH3zVEISKnrfT7bhuKZY1iibZIlXbVLNpyX36LkYJOiCqsMum3u70LH0VvTypkqiDbD4S6qfJ4vvUakpmKpOPutikiP7jkSP+AkaO0AQKBgQDkTuhnDK6+Y0a/HgpHJisji0coO+g2gsIszargHk8nNY2AB8t+EUn7C+Qu8cmrem5V8EXcdxS6z7iAXpJmY1Xepnsz+JP7Q91Lgt3OoqK5EybzUXXKkmNCD65n70Xxn2fEFzm6+GJP3c/HymlDKU2KBCYIyuUeaREjT0Fu3v6tgQKBgQDWnXppJwn4LJHhzFOCeO4zomDJDbLTZCabdKZoFP9r+vtEHAnclDDKx4AYbomSqgERe+DX6HR/tPHRVizP63RYPf7al2mJmPzt1nTkoc1/q5hQoD+oE154dADsW1pUp7AQjwCtys4iq5S0qAwIDpuY8M8bOHwZ+QmBvHYAigJCowKBgQC3HH6TX/2rH463bE2MARXqXSPGJj45sigwrQfW1xhe9zm1LQtN4mn2mvP5nt1D1l82OA6gIzYSGtX8x10eF5/ggqAf78goZ6bOkHh76b8fNzgvQO97eGt5qYAVRjhP8azU/lfEGMEpE1s5/6LrRe41utwSg0C+YkBnlIKDfQDAgQKBgDoBTCF5hK9H1JHzuKpt5uubuo78ndWWnvyrNYKyEirsJddNwLiWcO2NqChyT8qNGkbQdX/Fex89F5KduPTlTYfAEc6g18xxxgK+UM+uj60vArbf6PSTb5gculcnha2VuPdwvx050Cb8uu9s7/uJfzKB+2f/B0O51ID1H+ubYWsDAoGBAKrwGKHyqFTHSPg3XuRA1FgDAoOsfzP9ZJvMEXUWyu/VxjNt+0mRlyGeZ5qb9UZG+K/In4FbC/ux2P/PucCUIbgy/XGPtPXVavMwNbx0MquAcU0FihKXP0CUpi8zwiYc42MF7n/SztQnismxigBMSuJEDurcXXazjfcSRTypduNn" 4 | } -------------------------------------------------------------------------------- /sharness/config/ssl-basic_auth/server.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID7TCCAtWgAwIBAgIJAMqpHdKRMzMLMA0GCSqGSIb3DQEBCwUAMIGCMQswCQYD 3 | VQQGEwJVUzERMA8GA1UECAwIQ29sb3JhZG8xDzANBgNVBAcMBmdvbGRlbjEMMAoG 4 | A1UECgwDQ1NNMREwDwYDVQQLDAhTZWN0b3IgNzEMMAoGA1UEAwwDQm9iMSAwHgYJ 5 | KoZIhvcNAQkBFhFtaW5pc3RlckBtb3N3Lm9yZzAeFw0xNzA3MjExNjA5NTlaFw0y 6 | NzA3MTkxNjA5NTlaMIGCMQswCQYDVQQGEwJVUzERMA8GA1UECAwIQ29sb3JhZG8x 7 | DzANBgNVBAcMBmdvbGRlbjEMMAoGA1UECgwDQ1NNMREwDwYDVQQLDAhTZWN0b3Ig 8 | NzEMMAoGA1UEAwwDQm9iMSAwHgYJKoZIhvcNAQkBFhFtaW5pc3RlckBtb3N3Lm9y 9 | ZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALuoP8PehGItmKPi3+8S 10 | IV1qz8C3FiK85X/INxYLjyuzvpmDROtlkOvdmPCJrveKDZF7ECQpwIGApFbnKCCW 11 | 3zdOPQmAVzm4N8bvnzFtM9mTm8qKb9SwRi6ZLZ/qXo98t8C7CV6FaNKUkIw0lUes 12 | ZiXEcmknrlPy3svaDQVoSOH8L38d0g4geqiNrMmZDaGe8FAYdpCoeYDIm/u0Ag9y 13 | G3+XAbETxWhkfTyH3XcQ/Izg0wG9zFY8y/fyYwC+C7+xF75x4gbIzHAY2iFS2ua7 14 | GTKa2GZhOXtMuzJ6cf+TZW460Z+O+PkA1aH01WrGL7iCW/6Cn9gPRKL+IP6iyDnh 15 | 9HMCAwEAAaNkMGIwDwYDVR0RBAgwBocEfwAAATAdBgNVHQ4EFgQU9mXv8mv/LlAa 16 | jwr8X9hzk52cBagwHwYDVR0jBBgwFoAU9mXv8mv/LlAajwr8X9hzk52cBagwDwYD 17 | VR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAIxqpKYzF6A9RlLso0lkF 18 | nYfcyeVAvi03IBdiTNnpOe6ROa4gNwKH/JUJMCRDPzm/x78+srCmrcCCAJJTcqgi 19 | b84vq3DegGPg2NXbn9qVUA1SdiXFelqMFwLitDn2KKizihEN4L5PEArHuDaNvLI+ 20 | kMr+yZSALWTdtfydj211c7hTBvFqO8l5MYDXCmfoS9sqniorlNHIaBim/SNfDsi6 21 | 8hAhvfRvk3e6dPjAPrIZYdQR5ROGewtD4F/anXgKY2BmBtWwd6gbGeMnnVi1SGRP 22 | 0UHc4O9aq9HrAOFL/72WVk/kyyPyJ/GtSaPYL1OFS12R/l0hNi+pER7xDtLOVHO2 23 | iw== 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /sharness/config/ssl-basic_auth/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpQIBAAKCAQEAu6g/w96EYi2Yo+Lf7xIhXWrPwLcWIrzlf8g3FguPK7O+mYNE 3 | 62WQ692Y8Imu94oNkXsQJCnAgYCkVucoIJbfN049CYBXObg3xu+fMW0z2ZObyopv 4 | 1LBGLpktn+pej3y3wLsJXoVo0pSQjDSVR6xmJcRyaSeuU/Ley9oNBWhI4fwvfx3S 5 | DiB6qI2syZkNoZ7wUBh2kKh5gMib+7QCD3Ibf5cBsRPFaGR9PIfddxD8jODTAb3M 6 | VjzL9/JjAL4Lv7EXvnHiBsjMcBjaIVLa5rsZMprYZmE5e0y7Mnpx/5NlbjrRn474 7 | +QDVofTVasYvuIJb/oKf2A9Eov4g/qLIOeH0cwIDAQABAoIBAAOYreArG45mIU7C 8 | wlfqmQkZSvH+kEYKKLvSMnwRrKTBxR1cDq4UPDrI/G1ftiK4Wpo3KZAH3NCejoe7 9 | 1mEJgy2kKjdMZl+M0ETXws1Hsn6w/YNcM9h3qGCsPtuZukY1ta/T5dIR7HhcsIh/ 10 | WX0OKMcAhNDPGeAx/2MYwrcf0IXELx0+eP1fuBllkajH14J8+ZkVrBMDhqppn8Iq 11 | f9poVNQliJtN7VkL6lJ60HwoVNGEhFaOYphn3CR/sCc6xl+/CzV4h6c5X/RIUfDs 12 | kjgl9mlPFuWq9S19Z+XVfLSE+sYd6LDrh0IZEx9s0OfOjucH2bUAuKNDnCq0wW70 13 | FzH6KoECgYEA4ZOcAMgujk8goL8nleNjuEq7d8pThAsuAy5vq9oyol8oe+p1pXHR 14 | SHP6wHyhXeTS5g1Ej+QV6f0v9gVFS2pFqTXymc9Gxald3trcnheodZXx63YbxHm2 15 | H7mYWyZvq05A0qRLmmqCoSRJHUOkH2wVqgj9KsVYP1anIhdykbycansCgYEA1Pdp 16 | uAfWt/GLZ7B0q3JPlVvusf97wBIUcoaxLHGKopvfsaFp0EY3NRxLSTaZ0NPOxTHh 17 | W6xaIlBmKllyt6q8W609A8hrXayV1yYnVE44b5UEMhVlfRFeEdf9Sp4YdQJ8r1J0 18 | QA89jHCjf8VocP5pSJz5tXvWHhmaotXBthFgWGkCgYEAiy7dwenCOBKAqk5n6Wb9 19 | X3fVBguzzjRrtpDPXHTsax1VyGeZIXUB0bemD2CW3G1U55dmJ3ZvQwnyrtT/tZGj 20 | 280qnFa1bz6aaegW2gD082CKfWNJrMgAZMDKTeuAWW2WN6Ih9+wiH7VY25Kh0LWL 21 | BHg5ZUuQsLwRscpP6bY7uMMCgYEAwY23hK2DJZyfEXcbIjL7R4jNMPM82nzUHp5x 22 | 6i2rTUyTitJj5Anc5SU4+2pnc5b9RtWltva22Jbvs6+mBm1jUYLqgESn5/QSHv8r 23 | IYER47+wl4BAw+GD+H2wVB/JpJbFEWbEBvCTBM/emSKmYIOo1njsrlfFa4fjtfjG 24 | XJ4ATXkCgYEAzeSrCCVrfPMLCmOijIYD1F7TMFthosW2JJie3bcHZMu2QEM8EIif 25 | YzkUvMaDAXJ4VniTHkDf3ubRoUi3DwLbvJIPnoOlx3jmzz6KYiEd+uXx40Yrebb0 26 | V9GB2S2q1RY7wsFoCqT/mq8usQkjr3ulYMJqeIWnCTWgajXWqAHH/Mw= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /sharness/config/ssl-basic_auth/service.json: -------------------------------------------------------------------------------- 1 | { 2 | "cluster": { 3 | "peername": "testname", 4 | "secret": "84399cd0be811c2ca372d6ca473ffd73c09034f991c5e306fe9ada6c5fcfb641", 5 | "leave_on_shutdown": false, 6 | "listen_multiaddress": [ 7 | "/ip4/0.0.0.0/tcp/9096", 8 | "/ip6/::/tcp/9096" 9 | ], 10 | "state_sync_interval": "1m0s", 11 | "replication_factor": -1, 12 | "monitor_ping_interval": "15s" 13 | }, 14 | "consensus": { 15 | "raft": { 16 | "heartbeat_timeout": "1s", 17 | "election_timeout": "1s", 18 | "commit_timeout": "50ms", 19 | "max_append_entries": 64, 20 | "trailing_logs": 10240, 21 | "snapshot_interval": "2m0s", 22 | "snapshot_threshold": 8192, 23 | "leader_lease_timeout": "500ms" 24 | } 25 | }, 26 | "api": { 27 | "ipfsproxy": { 28 | "listen_multiaddress": "/ip4/127.0.0.1/tcp/9095", 29 | "node_multiaddress": "/ip4/127.0.0.1/tcp/5001", 30 | "read_timeout": "10m0s", 31 | "read_header_timeout": "5s", 32 | "write_timeout": "10m0s", 33 | "idle_timeout": "1m0s" 34 | }, 35 | "restapi": { 36 | "ssl_cert_file": "server.crt", 37 | "ssl_key_file": "server.key", 38 | "http_listen_multiaddress": "/ip4/127.0.0.1/tcp/9094", 39 | "read_timeout": "30s", 40 | "read_header_timeout": "5s", 41 | "write_timeout": "1m0s", 42 | "idle_timeout": "2m0s", 43 | "basic_auth_credentials": { 44 | "testuser": "testpass", 45 | "userwithoutpass": "" 46 | }, 47 | "cors_allowed_origins": [ 48 | "*" 49 | ], 50 | "cors_allowed_methods": [ 51 | "GET" 52 | ], 53 | "cors_allowed_headers": [], 54 | "cors_exposed_headers": [ 55 | "Content-Type", 56 | "X-Stream-Output", 57 | "X-Chunked-Output", 58 | "X-Content-Length" 59 | ], 60 | "cors_allow_credentials": true, 61 | "cors_max_age": "0s" 62 | } 63 | }, 64 | "ipfs_connector": { 65 | "ipfshttp": { 66 | "node_multiaddress": "/ip4/127.0.0.1/tcp/5001", 67 | "connect_swarms_delay": "30s", 68 | "ipfs_request_timeout": "5m0s", 69 | "pin_timeout": "0h2m0s", 70 | "unpin_timeout": "3h0m0s" 71 | } 72 | }, 73 | "pin_tracker": { 74 | "stateless": { 75 | "max_pin_queue_size": 50000, 76 | "concurrent_pins": 10 77 | } 78 | }, 79 | "monitor": { 80 | "pubsubmon": { 81 | "check_interval": "15s" 82 | } 83 | }, 84 | "informer": { 85 | "disk": { 86 | "metric_ttl": "30s", 87 | "metric_type": "reposize" 88 | }, 89 | "numpin": { 90 | "metric_ttl": "10s" 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /sharness/config/ssl/identity.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "QmdEtBsfumeH2V6dnx1fgn8zuW7XYjWdgJF4NEYpEBcTsg", 3 | "private_key": "CAASqAkwggSkAgEAAoIBAQC/ZmfWDbwyI0nJdRxgHcTdEaBFQo8sky9E+OOvtwZa5WKoLdHyHOLWxCAdpIHUBbhxz5rkMEWLwPI6ykqLIJToMPO8lJbKVzphOjv4JwpiAPdmeSiYMKLjx5V8MpqU2rwj/Uf3sRL8Gg9/Tei3PZ8cftxN1rkQQeeaOtk0CBxUFZSHEsyut1fbgIeL7TAY+4vCmXW0DBr4wh9fnoES/YivOvSiN9rScgWg6N65LfkI78hzaOJ4Nok2S4vYFCxjTAI9NWFUbhP5eJIFzTU+bZuQZxOn2qsoyw8pNZwuF+JClA/RcgBcCvVZcDH2ueVq/zT++bGCN+EWsAEdvJqJ5bsjAgMBAAECggEAaGDUZ6t94mnUJ4UyQEh7v4OJP7wYkFqEAL0qjfzl/lPyBX1XbQ3Ltwul6AR6uMGV4JszARZCFwDWGLGRDWZrTmTDxyfRQ+9l6vfzFFVWGDQmtz+Dn9uGOWnyX5TJMDxJNec+hBmRHOKpaOd37dYxGz0jr19V9UO7piRJp1J1AHUCypUGv5x1IekioSCu5fEyc7dyWwnmITHBjD08st+bCcjrIUFeXSdJKC8SymYeXdaVE3xH3zVEISKnrfT7bhuKZY1iibZIlXbVLNpyX36LkYJOiCqsMum3u70LH0VvTypkqiDbD4S6qfJ4vvUakpmKpOPutikiP7jkSP+AkaO0AQKBgQDkTuhnDK6+Y0a/HgpHJisji0coO+g2gsIszargHk8nNY2AB8t+EUn7C+Qu8cmrem5V8EXcdxS6z7iAXpJmY1Xepnsz+JP7Q91Lgt3OoqK5EybzUXXKkmNCD65n70Xxn2fEFzm6+GJP3c/HymlDKU2KBCYIyuUeaREjT0Fu3v6tgQKBgQDWnXppJwn4LJHhzFOCeO4zomDJDbLTZCabdKZoFP9r+vtEHAnclDDKx4AYbomSqgERe+DX6HR/tPHRVizP63RYPf7al2mJmPzt1nTkoc1/q5hQoD+oE154dADsW1pUp7AQjwCtys4iq5S0qAwIDpuY8M8bOHwZ+QmBvHYAigJCowKBgQC3HH6TX/2rH463bE2MARXqXSPGJj45sigwrQfW1xhe9zm1LQtN4mn2mvP5nt1D1l82OA6gIzYSGtX8x10eF5/ggqAf78goZ6bOkHh76b8fNzgvQO97eGt5qYAVRjhP8azU/lfEGMEpE1s5/6LrRe41utwSg0C+YkBnlIKDfQDAgQKBgDoBTCF5hK9H1JHzuKpt5uubuo78ndWWnvyrNYKyEirsJddNwLiWcO2NqChyT8qNGkbQdX/Fex89F5KduPTlTYfAEc6g18xxxgK+UM+uj60vArbf6PSTb5gculcnha2VuPdwvx050Cb8uu9s7/uJfzKB+2f/B0O51ID1H+ubYWsDAoGBAKrwGKHyqFTHSPg3XuRA1FgDAoOsfzP9ZJvMEXUWyu/VxjNt+0mRlyGeZ5qb9UZG+K/In4FbC/ux2P/PucCUIbgy/XGPtPXVavMwNbx0MquAcU0FihKXP0CUpi8zwiYc42MF7n/SztQnismxigBMSuJEDurcXXazjfcSRTypduNn" 4 | } -------------------------------------------------------------------------------- /sharness/config/ssl/server.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID7TCCAtWgAwIBAgIJAMqpHdKRMzMLMA0GCSqGSIb3DQEBCwUAMIGCMQswCQYD 3 | VQQGEwJVUzERMA8GA1UECAwIQ29sb3JhZG8xDzANBgNVBAcMBmdvbGRlbjEMMAoG 4 | A1UECgwDQ1NNMREwDwYDVQQLDAhTZWN0b3IgNzEMMAoGA1UEAwwDQm9iMSAwHgYJ 5 | KoZIhvcNAQkBFhFtaW5pc3RlckBtb3N3Lm9yZzAeFw0xNzA3MjExNjA5NTlaFw0y 6 | NzA3MTkxNjA5NTlaMIGCMQswCQYDVQQGEwJVUzERMA8GA1UECAwIQ29sb3JhZG8x 7 | DzANBgNVBAcMBmdvbGRlbjEMMAoGA1UECgwDQ1NNMREwDwYDVQQLDAhTZWN0b3Ig 8 | NzEMMAoGA1UEAwwDQm9iMSAwHgYJKoZIhvcNAQkBFhFtaW5pc3RlckBtb3N3Lm9y 9 | ZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALuoP8PehGItmKPi3+8S 10 | IV1qz8C3FiK85X/INxYLjyuzvpmDROtlkOvdmPCJrveKDZF7ECQpwIGApFbnKCCW 11 | 3zdOPQmAVzm4N8bvnzFtM9mTm8qKb9SwRi6ZLZ/qXo98t8C7CV6FaNKUkIw0lUes 12 | ZiXEcmknrlPy3svaDQVoSOH8L38d0g4geqiNrMmZDaGe8FAYdpCoeYDIm/u0Ag9y 13 | G3+XAbETxWhkfTyH3XcQ/Izg0wG9zFY8y/fyYwC+C7+xF75x4gbIzHAY2iFS2ua7 14 | GTKa2GZhOXtMuzJ6cf+TZW460Z+O+PkA1aH01WrGL7iCW/6Cn9gPRKL+IP6iyDnh 15 | 9HMCAwEAAaNkMGIwDwYDVR0RBAgwBocEfwAAATAdBgNVHQ4EFgQU9mXv8mv/LlAa 16 | jwr8X9hzk52cBagwHwYDVR0jBBgwFoAU9mXv8mv/LlAajwr8X9hzk52cBagwDwYD 17 | VR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAIxqpKYzF6A9RlLso0lkF 18 | nYfcyeVAvi03IBdiTNnpOe6ROa4gNwKH/JUJMCRDPzm/x78+srCmrcCCAJJTcqgi 19 | b84vq3DegGPg2NXbn9qVUA1SdiXFelqMFwLitDn2KKizihEN4L5PEArHuDaNvLI+ 20 | kMr+yZSALWTdtfydj211c7hTBvFqO8l5MYDXCmfoS9sqniorlNHIaBim/SNfDsi6 21 | 8hAhvfRvk3e6dPjAPrIZYdQR5ROGewtD4F/anXgKY2BmBtWwd6gbGeMnnVi1SGRP 22 | 0UHc4O9aq9HrAOFL/72WVk/kyyPyJ/GtSaPYL1OFS12R/l0hNi+pER7xDtLOVHO2 23 | iw== 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /sharness/config/ssl/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpQIBAAKCAQEAu6g/w96EYi2Yo+Lf7xIhXWrPwLcWIrzlf8g3FguPK7O+mYNE 3 | 62WQ692Y8Imu94oNkXsQJCnAgYCkVucoIJbfN049CYBXObg3xu+fMW0z2ZObyopv 4 | 1LBGLpktn+pej3y3wLsJXoVo0pSQjDSVR6xmJcRyaSeuU/Ley9oNBWhI4fwvfx3S 5 | DiB6qI2syZkNoZ7wUBh2kKh5gMib+7QCD3Ibf5cBsRPFaGR9PIfddxD8jODTAb3M 6 | VjzL9/JjAL4Lv7EXvnHiBsjMcBjaIVLa5rsZMprYZmE5e0y7Mnpx/5NlbjrRn474 7 | +QDVofTVasYvuIJb/oKf2A9Eov4g/qLIOeH0cwIDAQABAoIBAAOYreArG45mIU7C 8 | wlfqmQkZSvH+kEYKKLvSMnwRrKTBxR1cDq4UPDrI/G1ftiK4Wpo3KZAH3NCejoe7 9 | 1mEJgy2kKjdMZl+M0ETXws1Hsn6w/YNcM9h3qGCsPtuZukY1ta/T5dIR7HhcsIh/ 10 | WX0OKMcAhNDPGeAx/2MYwrcf0IXELx0+eP1fuBllkajH14J8+ZkVrBMDhqppn8Iq 11 | f9poVNQliJtN7VkL6lJ60HwoVNGEhFaOYphn3CR/sCc6xl+/CzV4h6c5X/RIUfDs 12 | kjgl9mlPFuWq9S19Z+XVfLSE+sYd6LDrh0IZEx9s0OfOjucH2bUAuKNDnCq0wW70 13 | FzH6KoECgYEA4ZOcAMgujk8goL8nleNjuEq7d8pThAsuAy5vq9oyol8oe+p1pXHR 14 | SHP6wHyhXeTS5g1Ej+QV6f0v9gVFS2pFqTXymc9Gxald3trcnheodZXx63YbxHm2 15 | H7mYWyZvq05A0qRLmmqCoSRJHUOkH2wVqgj9KsVYP1anIhdykbycansCgYEA1Pdp 16 | uAfWt/GLZ7B0q3JPlVvusf97wBIUcoaxLHGKopvfsaFp0EY3NRxLSTaZ0NPOxTHh 17 | W6xaIlBmKllyt6q8W609A8hrXayV1yYnVE44b5UEMhVlfRFeEdf9Sp4YdQJ8r1J0 18 | QA89jHCjf8VocP5pSJz5tXvWHhmaotXBthFgWGkCgYEAiy7dwenCOBKAqk5n6Wb9 19 | X3fVBguzzjRrtpDPXHTsax1VyGeZIXUB0bemD2CW3G1U55dmJ3ZvQwnyrtT/tZGj 20 | 280qnFa1bz6aaegW2gD082CKfWNJrMgAZMDKTeuAWW2WN6Ih9+wiH7VY25Kh0LWL 21 | BHg5ZUuQsLwRscpP6bY7uMMCgYEAwY23hK2DJZyfEXcbIjL7R4jNMPM82nzUHp5x 22 | 6i2rTUyTitJj5Anc5SU4+2pnc5b9RtWltva22Jbvs6+mBm1jUYLqgESn5/QSHv8r 23 | IYER47+wl4BAw+GD+H2wVB/JpJbFEWbEBvCTBM/emSKmYIOo1njsrlfFa4fjtfjG 24 | XJ4ATXkCgYEAzeSrCCVrfPMLCmOijIYD1F7TMFthosW2JJie3bcHZMu2QEM8EIif 25 | YzkUvMaDAXJ4VniTHkDf3ubRoUi3DwLbvJIPnoOlx3jmzz6KYiEd+uXx40Yrebb0 26 | V9GB2S2q1RY7wsFoCqT/mq8usQkjr3ulYMJqeIWnCTWgajXWqAHH/Mw= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /sharness/run-sharness-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Run tests 4 | cd "$(dirname "$0")" 5 | statuses=0 6 | for i in t0*.sh; 7 | do 8 | echo "*** $i ***" 9 | ./$i 10 | status=$? 11 | statuses=$((statuses + $status)) 12 | if [ $status -ne 0 ]; then 13 | echo "Test $i failed" 14 | fi 15 | done 16 | 17 | # Aggregate Results 18 | echo "Aggregating..." 19 | for f in test-results/*.counts; do 20 | echo "$f"; 21 | done | bash lib/sharness/aggregate-results.sh 22 | 23 | # Cleanup results 24 | rm -rf test-results 25 | 26 | # Exit with error if any test has failed 27 | if [ $statuses -gt 0 ]; then 28 | echo $statuses 29 | exit 1 30 | fi 31 | exit 0 32 | -------------------------------------------------------------------------------- /sharness/t0010-ctl-basic-commands.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_description="Test ctl installation and some basic commands" 4 | 5 | . lib/test-lib.sh 6 | 7 | 8 | test_expect_success "current dir is writeable" ' 9 | echo "Writability check" >test.txt && 10 | test_when_finished "rm test.txt" 11 | ' 12 | 13 | test_expect_success "cluster-ctl --version succeeds" ' 14 | ipfs-cluster-ctl --version 15 | ' 16 | 17 | test_expect_success "cluster-ctl help commands succeed" ' 18 | ipfs-cluster-ctl --help && 19 | ipfs-cluster-ctl -h && 20 | ipfs-cluster-ctl h && 21 | ipfs-cluster-ctl help 22 | ' 23 | 24 | test_expect_success "cluster-ctl help has 120 char limits" ' 25 | ipfs-cluster-ctl --help >help.txt && 26 | test_when_finished "rm help.txt" && 27 | LENGTH="$(cat help.txt | awk '"'"'{print length }'"'"' | sort -nr | head -n 1)" && 28 | [ ! "$LENGTH" -gt 120 ] 29 | ' 30 | 31 | test_expect_success "cluster-ctl help output looks good" ' 32 | ipfs-cluster-ctl --help | egrep -q -i "^(Usage|Commands|Global options)" 33 | ' 34 | 35 | test_expect_success "cluster-ctl commands output looks good" ' 36 | ipfs-cluster-ctl commands > commands.txt && 37 | test_when_finished "rm commands.txt" && 38 | egrep -q "ipfs-cluster-ctl id" commands.txt && 39 | egrep -q "ipfs-cluster-ctl peers" commands.txt && 40 | egrep -q "ipfs-cluster-ctl pin" commands.txt && 41 | egrep -q "ipfs-cluster-ctl status" commands.txt && 42 | egrep -q "ipfs-cluster-ctl recover" commands.txt && 43 | egrep -q "ipfs-cluster-ctl version" commands.txt && 44 | egrep -q "ipfs-cluster-ctl commands" commands.txt 45 | ' 46 | 47 | test_expect_success "All cluster-ctl command docs are 120 columns or less" ' 48 | export failure="0" && 49 | ipfs-cluster-ctl commands | awk "NF" >commands.txt && 50 | test_when_finished "rm commands.txt" && 51 | while read cmd 52 | do 53 | LENGTH="$($cmd --help | awk "{ print length }" | sort -nr | head -n 1)" 54 | [ "$LENGTH" -gt 120 ] && 55 | { echo "$cmd" help text is longer than 119 chars "($LENGTH)"; export failure="1"; } 56 | done I0" 12 | ' 13 | 14 | test_expect_success IPFS,CLUSTER "health metrics with metric name must succeed" ' 15 | ipfs-cluster-ctl health metrics ping && 16 | ipfs-cluster-ctl health metrics freespace 17 | ' 18 | 19 | test_expect_success IPFS,CLUSTER "health metrics without metric name doesn't fail" ' 20 | ipfs-cluster-ctl health metrics 21 | ' 22 | 23 | test_expect_success IPFS,CLUSTER "list latest metrics logged by this peer" ' 24 | pid=`ipfs-cluster-ctl --enc=json id | jq -r ".id"` 25 | ipfs-cluster-ctl health metrics freespace | grep -q -E "(^$pid \| freespace: [0-9]+ (G|M|K)B \| Expires in: [0-9]+ seconds from now)" 26 | ' 27 | 28 | test_expect_success IPFS,CLUSTER "alerts must succeed" ' 29 | ipfs-cluster-ctl health alerts 30 | ' 31 | 32 | test_clean_ipfs 33 | test_clean_cluster 34 | 35 | test_done 36 | -------------------------------------------------------------------------------- /sharness/t0040-ssl-simple-exchange.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_description="Test service + ctl SSL interaction" 4 | 5 | ssl_config="`pwd`/config/ssl" 6 | 7 | . lib/test-lib.sh 8 | 9 | test_ipfs_init 10 | 11 | test_cluster_init "$ssl_config" 12 | cleanup test_clean_cluster 13 | 14 | test_expect_success "prerequisites" ' 15 | test_have_prereq IPFS && test_have_prereq CLUSTER 16 | ' 17 | 18 | test_expect_success "ssl interaction succeeds" ' 19 | id=`cluster_id` 20 | ipfs-cluster-ctl --https --no-check-certificate id | egrep -q "$id" 21 | ' 22 | 23 | test_clean_ipfs 24 | 25 | test_done 26 | -------------------------------------------------------------------------------- /sharness/t0041-ssl-enforcement.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_description="Test failure when server not using SSL but client requests it" 4 | 5 | . lib/test-lib.sh 6 | 7 | test_ipfs_init 8 | test_cluster_init 9 | 10 | test_expect_success "prerequisites" ' 11 | test_have_prereq IPFS && test_have_prereq CLUSTER 12 | ' 13 | 14 | test_expect_success "ssl enforced by client" ' 15 | id=`cluster_id` 16 | test_must_fail ipfs-cluster-ctl --https --no-check-certificate id 17 | ' 18 | 19 | test_clean_ipfs 20 | test_clean_cluster 21 | 22 | test_done 23 | -------------------------------------------------------------------------------- /sharness/t0042-basic-auth.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_description="Test service + ctl SSL interaction" 4 | 5 | config="`pwd`/config/basic_auth" 6 | 7 | . lib/test-lib.sh 8 | 9 | test_ipfs_init 10 | test_cluster_init "$config" 11 | 12 | test_expect_success "prerequisites" ' 13 | test_have_prereq IPFS && test_have_prereq CLUSTER 14 | ' 15 | 16 | test_expect_success "BasicAuth fails without credentials" ' 17 | id=`cluster_id` 18 | { test_must_fail ipfs-cluster-ctl id; } | grep -A1 "401" | grep -i "unauthorized" 19 | ' 20 | 21 | test_expect_success "BasicAuth fails with bad credentials" ' 22 | id=`cluster_id` 23 | { test_must_fail ipfs-cluster-ctl --basic-auth "testuser" --force-http id; } | grep -A1 "401" | grep -i "unauthorized" && 24 | { test_must_fail ipfs-cluster-ctl --basic-auth "testuser:badpass" --force-http id; } | grep -A1 "401" | grep -i "unauthorized" && 25 | { test_must_fail ipfs-cluster-ctl --basic-auth "baduser:testpass" --force-http id; } | grep -A1 "401" | grep -i "unauthorized" && 26 | { test_must_fail ipfs-cluster-ctl --basic-auth "baduser:badpass" --force-http id; } | grep -A1 "401" | grep -i "unauthorized" 27 | ' 28 | 29 | test_expect_success "BasicAuth over HTTP succeeds with CLI flag credentials" ' 30 | id=`cluster_id` 31 | ipfs-cluster-ctl --basic-auth "testuser:testpass" --force-http id | grep -q "$id" 32 | ' 33 | 34 | test_expect_success "BasicAuth succeeds with env var credentials" ' 35 | id=`cluster_id` 36 | export CLUSTER_CREDENTIALS="testuser:testpass" 37 | ipfs-cluster-ctl --force-http id | egrep -q "$id" 38 | ' 39 | 40 | test_clean_ipfs 41 | test_clean_cluster 42 | 43 | test_done 44 | -------------------------------------------------------------------------------- /sharness/t0043-ssl-basic-auth.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_description="Test service + ctl SSL interaction" 4 | 5 | config="`pwd`/config/ssl-basic_auth" 6 | 7 | . lib/test-lib.sh 8 | 9 | test_ipfs_init 10 | test_cluster_init "$config" 11 | 12 | test_expect_success "prerequisites" ' 13 | test_have_prereq IPFS && test_have_prereq CLUSTER 14 | ' 15 | 16 | test_expect_success "ssl interaction fails with bad credentials" ' 17 | id=`cluster_id` 18 | { test_must_fail ipfs-cluster-ctl --no-check-certificate --basic-auth "testuser:badpass" id; } | grep -A1 "401" | grep -i "unauthorized" 19 | ' 20 | 21 | test_expect_success "ssl interaction succeeds" ' 22 | id=`cluster_id` 23 | ipfs-cluster-ctl --no-check-certificate --basic-auth "testuser:testpass" id | egrep -q "$id" 24 | ' 25 | 26 | test_clean_ipfs 27 | test_clean_cluster 28 | 29 | test_done 30 | -------------------------------------------------------------------------------- /sharness/t0052-service-state-export.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_description="Test service state export" 4 | 5 | . lib/test-lib.sh 6 | 7 | test_ipfs_init 8 | test_cluster_init "" crdt 9 | 10 | test_expect_success IPFS,CLUSTER,JQ "state export saves the correct state to expected file (crdt)" ' 11 | cid=`docker exec ipfs sh -c "echo test_52-1 | ipfs add -q"` && 12 | ipfs-cluster-ctl pin add "$cid" && 13 | sleep 5 && 14 | cluster_kill && sleep 5 && 15 | ipfs-cluster-service --debug --config "test-config" state export -f export.json && 16 | [ -f export.json ] && 17 | jq -r ".cid" export.json | grep -q "$cid" 18 | ' 19 | 20 | cluster_kill 21 | sleep 5 22 | test_cluster_init "" raft 23 | 24 | test_expect_success IPFS,CLUSTER,JQ "state export saves the correct state to expected file (raft)" ' 25 | cid=`docker exec ipfs sh -c "echo test_52-2 | ipfs add -q"` && 26 | ipfs-cluster-ctl pin add "$cid" && 27 | sleep 5 && 28 | cluster_kill && sleep 5 && 29 | ipfs-cluster-service --debug --config "test-config" state export -f export.json && 30 | [ -f export.json ] && 31 | jq -r ".cid" export.json | grep -q "$cid" 32 | ' 33 | 34 | test_clean_ipfs 35 | test_clean_cluster 36 | 37 | test_done 38 | -------------------------------------------------------------------------------- /sharness/t0053-service-state-import.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_description="Test service state import" 4 | 5 | . lib/test-lib.sh 6 | 7 | test_ipfs_init 8 | test_cluster_init 9 | test_confirm_importState 10 | 11 | # Kill cluster daemon but keep data folder 12 | cluster_kill 13 | 14 | 15 | # WARNING: Updating the added content needs updating the importState file. 16 | 17 | test_expect_success IPFS,CLUSTER "state import fails on incorrect format (crdt)" ' 18 | sleep 5 && 19 | echo "not exactly json" > badImportFile && 20 | test_expect_code 1 ipfs-cluster-service --config "test-config" state import -f badImportFile 21 | ' 22 | 23 | test_expect_success IPFS,CLUSTER,IMPORTSTATE "state import succeeds on correct format (crdt)" ' 24 | sleep 5 25 | cid=`docker exec ipfs sh -c "echo test_53 | ipfs add -q"` && 26 | ipfs-cluster-service --config "test-config" state import -f importState && 27 | cluster_start && 28 | sleep 5 && 29 | ipfs-cluster-ctl pin ls "$cid" | grep -q "$cid" && 30 | ipfs-cluster-ctl status "$cid" | grep -q -i "PINNED" 31 | ' 32 | 33 | # Kill cluster daemon but keep data folder 34 | cluster_kill 35 | sleep 5 36 | 37 | test_expect_success IPFS,CLUSTER "state import fails on incorrect format (raft)" ' 38 | ipfs-cluster-service --config "test-config" init --force --consensus raft && 39 | echo "not exactly json" > badImportFile && 40 | test_expect_code 1 ipfs-cluster-service --config "test-config" state import -f badImportFile 41 | ' 42 | 43 | test_expect_success IPFS,CLUSTER,IMPORTSTATE "state import succeeds on correct format (raft)" ' 44 | sleep 5 45 | cid=`docker exec ipfs sh -c "echo test_53 | ipfs add -q"` && 46 | ipfs-cluster-service --config "test-config" state import -f importState && 47 | cluster_start && 48 | sleep 5 && 49 | ipfs-cluster-ctl pin ls "$cid" | grep -q "$cid" && 50 | ipfs-cluster-ctl status "$cid" | grep -q -i "PINNED" 51 | ' 52 | 53 | test_clean_ipfs 54 | test_clean_cluster 55 | 56 | test_done 57 | -------------------------------------------------------------------------------- /sharness/t0054-service-state-clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_description="Test service state import" 4 | 5 | . lib/test-lib.sh 6 | 7 | test_ipfs_init 8 | test_cluster_init 9 | 10 | test_expect_success IPFS,CLUSTER "state cleanup refreshes state on restart (crdt)" ' 11 | cid=`docker exec ipfs sh -c "echo test_54 | ipfs add -q"` && 12 | ipfs-cluster-ctl pin add "$cid" && sleep 5 && 13 | ipfs-cluster-ctl pin ls "$cid" | grep -q "$cid" && 14 | ipfs-cluster-ctl status "$cid" | grep -q -i "PINNED" && 15 | [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] && 16 | cluster_kill && sleep 5 && 17 | ipfs-cluster-service --config "test-config" state cleanup -f && 18 | cluster_start && sleep 5 && 19 | [ 0 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] 20 | ' 21 | 22 | test_expect_success IPFS,CLUSTER "export + cleanup + import == noop (crdt)" ' 23 | cid=`docker exec ipfs sh -c "echo test_54 | ipfs add -q"` && 24 | ipfs-cluster-ctl pin add "$cid" && sleep 5 && 25 | [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] && 26 | cluster_kill && sleep 5 && 27 | ipfs-cluster-service --config "test-config" state export -f import.json && 28 | ipfs-cluster-service --config "test-config" state cleanup -f && 29 | ipfs-cluster-service --config "test-config" state import -f import.json && 30 | cluster_start && sleep 5 && 31 | ipfs-cluster-ctl pin ls "$cid" | grep -q "$cid" && 32 | ipfs-cluster-ctl status "$cid" | grep -q -i "PINNED" && 33 | [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] 34 | ' 35 | 36 | cluster_kill 37 | sleep 5 38 | test_cluster_init "" raft 39 | 40 | test_expect_success IPFS,CLUSTER "state cleanup refreshes state on restart (raft)" ' 41 | cid=`docker exec ipfs sh -c "echo test_54 | ipfs add -q"` && 42 | ipfs-cluster-ctl pin add "$cid" && sleep 5 && 43 | ipfs-cluster-ctl pin ls "$cid" | grep -q "$cid" && 44 | ipfs-cluster-ctl status "$cid" | grep -q -i "PINNED" && 45 | [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] && 46 | cluster_kill && sleep 5 && 47 | ipfs-cluster-service --config "test-config" state cleanup -f && 48 | cluster_start && sleep 5 && 49 | [ 0 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] 50 | ' 51 | 52 | test_expect_success IPFS,CLUSTER "export + cleanup + import == noop (raft)" ' 53 | cid=`docker exec ipfs sh -c "echo test_54 | ipfs add -q"` && 54 | ipfs-cluster-ctl pin add "$cid" && sleep 5 && 55 | [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] && 56 | cluster_kill && sleep 5 && 57 | ipfs-cluster-service --config "test-config" state export -f import.json && 58 | ipfs-cluster-service --config "test-config" state cleanup -f && 59 | ipfs-cluster-service --config "test-config" state import -f import.json && 60 | cluster_start && sleep 5 && 61 | ipfs-cluster-ctl pin ls "$cid" | grep -q "$cid" && 62 | ipfs-cluster-ctl status "$cid" | grep -q -i "PINNED" && 63 | [ 1 -eq "$(ipfs-cluster-ctl --enc=json status | jq -n "[inputs] | length")" ] 64 | ' 65 | 66 | 67 | test_clean_ipfs 68 | test_clean_cluster 69 | 70 | test_done 71 | -------------------------------------------------------------------------------- /sharness/test_data/importState: -------------------------------------------------------------------------------- 1 | { 2 | "cid": "QmbrCtydGyPeHiLURSPMqrvE5mCgMCwFYq3UD4XLCeAYw6", 3 | "name": "", 4 | "allocations": [], 5 | "replication_factor_min": -1, 6 | "replication_factor_max": -1 7 | } 8 | 9 | -------------------------------------------------------------------------------- /sharness/test_data/small_file: -------------------------------------------------------------------------------- 1 | small file 2 | -------------------------------------------------------------------------------- /sharness/test_data/v1Crc: -------------------------------------------------------------------------------- 1 | y8SrOIoXJo4= 2 | -------------------------------------------------------------------------------- /state/dsstate/datastore_test.go: -------------------------------------------------------------------------------- 1 | package dsstate 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "testing" 7 | "time" 8 | 9 | "github.com/ipfs-cluster/ipfs-cluster/api" 10 | "github.com/ipfs-cluster/ipfs-cluster/datastore/inmem" 11 | 12 | peer "github.com/libp2p/go-libp2p/core/peer" 13 | ) 14 | 15 | var testCid1, _ = api.DecodeCid("QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmq") 16 | var testPeerID1, _ = peer.Decode("QmXZrtE5jQwXNqCJMfHUTQkvhQ4ZAnqMnmzFMJfLewuabc") 17 | 18 | var c = api.Pin{ 19 | Cid: testCid1, 20 | Type: api.DataType, 21 | Allocations: []peer.ID{testPeerID1}, 22 | MaxDepth: -1, 23 | PinOptions: api.PinOptions{ 24 | ReplicationFactorMax: -1, 25 | ReplicationFactorMin: -1, 26 | Name: "test", 27 | }, 28 | } 29 | 30 | func newState(t *testing.T) *State { 31 | store := inmem.New() 32 | ds, err := New(context.Background(), store, "", DefaultHandle()) 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | return ds 37 | } 38 | 39 | func TestAdd(t *testing.T) { 40 | ctx := context.Background() 41 | st := newState(t) 42 | st.Add(ctx, c) 43 | if ok, err := st.Has(ctx, c.Cid); !ok || err != nil { 44 | t.Error("should have added it") 45 | } 46 | } 47 | 48 | func TestRm(t *testing.T) { 49 | ctx := context.Background() 50 | st := newState(t) 51 | st.Add(ctx, c) 52 | st.Rm(ctx, c.Cid) 53 | if ok, err := st.Has(ctx, c.Cid); ok || err != nil { 54 | t.Error("should have removed it") 55 | } 56 | } 57 | 58 | func TestGet(t *testing.T) { 59 | ctx := context.Background() 60 | defer func() { 61 | if r := recover(); r != nil { 62 | t.Fatal("paniced") 63 | } 64 | }() 65 | st := newState(t) 66 | st.Add(ctx, c) 67 | get, err := st.Get(ctx, c.Cid) 68 | if err != nil { 69 | t.Fatal(err) 70 | } 71 | 72 | if get.Cid.String() != c.Cid.String() { 73 | t.Error("bad cid decoding: ", get.Cid) 74 | } 75 | 76 | if get.Allocations[0] != c.Allocations[0] { 77 | t.Error("bad allocations decoding:", get.Allocations) 78 | } 79 | 80 | if get.ReplicationFactorMax != c.ReplicationFactorMax || 81 | get.ReplicationFactorMin != c.ReplicationFactorMin { 82 | t.Error("bad replication factors decoding") 83 | } 84 | } 85 | 86 | func TestList(t *testing.T) { 87 | ctx := context.Background() 88 | defer func() { 89 | if r := recover(); r != nil { 90 | t.Fatal("paniced") 91 | } 92 | }() 93 | st := newState(t) 94 | st.Add(ctx, c) 95 | out := make(chan api.Pin) 96 | go func() { 97 | err := st.List(ctx, out) 98 | if err != nil { 99 | t.Error(err) 100 | } 101 | }() 102 | 103 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) 104 | defer cancel() 105 | 106 | var list0 api.Pin 107 | for { 108 | select { 109 | case p, ok := <-out: 110 | if !ok && !list0.Cid.Defined() { 111 | t.Fatal("should have read list0 first") 112 | } 113 | if !ok { 114 | return 115 | } 116 | list0 = p 117 | if !p.Equals(c) { 118 | t.Error("returned something different") 119 | } 120 | case <-ctx.Done(): 121 | t.Error("should have read from channel") 122 | return 123 | } 124 | } 125 | 126 | } 127 | 128 | func TestMarshalUnmarshal(t *testing.T) { 129 | ctx := context.Background() 130 | st := newState(t) 131 | st.Add(ctx, c) 132 | buf := new(bytes.Buffer) 133 | err := st.Marshal(buf) 134 | if err != nil { 135 | t.Fatal(err) 136 | } 137 | st2 := newState(t) 138 | err = st2.Unmarshal(buf) 139 | if err != nil { 140 | t.Fatal(err) 141 | } 142 | 143 | get, err := st2.Get(ctx, c.Cid) 144 | if err != nil { 145 | t.Fatal(err) 146 | } 147 | if get.Allocations[0] != testPeerID1 { 148 | t.Error("expected different peer id") 149 | } 150 | if !get.Cid.Equals(c.Cid) { 151 | t.Error("expected different cid") 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /state/empty.go: -------------------------------------------------------------------------------- 1 | package state 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ipfs-cluster/ipfs-cluster/api" 7 | ) 8 | 9 | type empty struct{} 10 | 11 | func (e *empty) List(ctx context.Context, out chan<- api.Pin) error { 12 | close(out) 13 | return nil 14 | } 15 | 16 | func (e *empty) Has(ctx context.Context, c api.Cid) (bool, error) { 17 | return false, nil 18 | } 19 | 20 | func (e *empty) Get(ctx context.Context, c api.Cid) (api.Pin, error) { 21 | return api.Pin{}, ErrNotFound 22 | } 23 | 24 | // Empty returns an empty read-only state. 25 | func Empty() ReadOnly { 26 | return &empty{} 27 | } 28 | -------------------------------------------------------------------------------- /state/interface.go: -------------------------------------------------------------------------------- 1 | // Package state holds the interface that any state implementation for 2 | // IPFS Cluster must satisfy. 3 | package state 4 | 5 | // State represents the shared state of the cluster 6 | import ( 7 | "context" 8 | "errors" 9 | "io" 10 | 11 | "github.com/ipfs-cluster/ipfs-cluster/api" 12 | ) 13 | 14 | // ErrNotFound should be returned when a pin is not part of the state. 15 | var ErrNotFound = errors.New("pin is not part of the pinset") 16 | 17 | // State is a wrapper to the Cluster shared state so that Pin objects can 18 | // be easily read, written and queried. The state can be marshaled and 19 | // unmarshaled. Implementation should be thread-safe. 20 | type State interface { 21 | ReadOnly 22 | WriteOnly 23 | // Migrate restores the serialized format of an outdated state to the 24 | // current version. 25 | Migrate(ctx context.Context, r io.Reader) error 26 | // Marshal serializes the state to a byte slice. 27 | Marshal(io.Writer) error 28 | // Unmarshal deserializes the state from marshaled bytes. 29 | Unmarshal(io.Reader) error 30 | } 31 | 32 | // ReadOnly represents the read side of a State. 33 | type ReadOnly interface { 34 | // List lists all the pins in the state. 35 | List(context.Context, chan<- api.Pin) error 36 | // Has returns true if the state is holding information for a Cid. 37 | Has(context.Context, api.Cid) (bool, error) 38 | // Get returns the information attacthed to this pin, if any. If the 39 | // pin is not part of the state, it should return ErrNotFound. 40 | Get(context.Context, api.Cid) (api.Pin, error) 41 | } 42 | 43 | // WriteOnly represents the write side of a State. 44 | type WriteOnly interface { 45 | // Add adds a pin to the State 46 | Add(context.Context, api.Pin) error 47 | // Rm removes a pin from the State. 48 | Rm(context.Context, api.Cid) error 49 | } 50 | 51 | // BatchingState represents a state which batches write operations. 52 | type BatchingState interface { 53 | State 54 | // Commit writes any batched operations. 55 | Commit(context.Context) error 56 | } 57 | -------------------------------------------------------------------------------- /test/cids.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "github.com/ipfs-cluster/ipfs-cluster/api" 5 | peer "github.com/libp2p/go-libp2p/core/peer" 6 | ) 7 | 8 | // Common variables used all around tests. 9 | var ( 10 | Cid1, _ = api.DecodeCid("QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmq") 11 | Cid2, _ = api.DecodeCid("QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmma") 12 | Cid3, _ = api.DecodeCid("QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmb") 13 | Cid4Data = "Cid4Data" 14 | // Cid resulting from block put using blake2b-256 and raw format 15 | Cid4, _ = api.DecodeCid("bafk2bzaceawsyhsnrwwy5mtit2emnjfalkxsyq2p2ptd6fuliolzwwjbs42fq") 16 | 17 | // Cid resulting from block put using format "v0" defaults 18 | Cid5, _ = api.DecodeCid("QmbgmXgsFjxAJ7cEaziL2NDSptHAkPwkEGMmKMpfyYeFXL") 19 | Cid5Data = "Cid5Data" 20 | SlowCid1, _ = api.DecodeCid("QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmd") 21 | CidResolved, _ = api.DecodeCid("zb2rhiKhUepkTMw7oFfBUnChAN7ABAvg2hXUwmTBtZ6yxuabc") 22 | // ErrorCid is meant to be used as a Cid which causes errors. i.e. the 23 | // ipfs mock fails when pinning this CID. 24 | ErrorCid, _ = api.DecodeCid("QmP63DkAFEnDYNjDYBpyNDfttu1fvUw99x1brscPzpqmmc") 25 | // NotFoundCid is meant to be used as a CID that doesn't exist in the 26 | // pinset. 27 | NotFoundCid, _ = api.DecodeCid("bafyreiay3jpjk74dkckv2r74eyvf3lfnxujefay2rtuluintasq2zlapv4") 28 | PeerID1, _ = peer.Decode("QmXZrtE5jQwXNqCJMfHUTQkvhQ4ZAnqMnmzFMJfLewuabc") 29 | PeerID2, _ = peer.Decode("QmUZ13osndQ5uL4tPWHXe3iBgBgq9gfewcBMSCAuMBsDJ6") 30 | PeerID3, _ = peer.Decode("QmPGDFvBkgWhvzEK9qaTWrWurSwqXNmhnK3hgELPdZZNPa") 31 | PeerID4, _ = peer.Decode("QmZ8naDy5mEz4GLuQwjWt9MPYqHTBbsm8tQBrNSjiq6zBc") 32 | PeerID5, _ = peer.Decode("QmZVAo3wd8s5eTTy2kPYs34J9PvfxpKPuYsePPYGjgRRjg") 33 | PeerID6, _ = peer.Decode("QmR8Vu6kZk7JvAN2rWVWgiduHatgBq2bb15Yyq8RRhYSbx") 34 | PeerID7, _ = peer.Decode("12D3KooWGHTKzeT4KaLGLrbKKyT8zKrBPXAUBRzCAN6ZMDMo4M6M") 35 | PeerID8, _ = peer.Decode("12D3KooWFBFCDQzAkQSwPZLV883pKdsmb6urQ3sMjfJHUxn5GCVv") 36 | PeerID9, _ = peer.Decode("12D3KooWKuJ8LPTyHbyX4nt4C7uWmUobzFsiceTVoFw7HpmoNakM") 37 | 38 | PeerName1 = "TestPeer1" 39 | PeerName2 = "TestPeer2" 40 | PeerName3 = "TestPeer3" 41 | PeerName4 = "TestPeer4" 42 | PeerName5 = "TestPeer5" 43 | PeerName6 = "TestPeer6" 44 | 45 | PathIPFS1 = "/ipfs/QmaNJ5acV31sx8jq626qTpAWW4DXKw34aGhx53dECLvXbY" 46 | PathIPFS2 = "/ipfs/QmbUNM297ZwxB8CfFAznK7H9YMesDoY6Tt5bPgt5MSCB2u/im.gif" 47 | PathIPFS3 = "/ipfs/QmbUNM297ZwxB8CfFAznK7H9YMesDoY6Tt5bPgt5MSCB2u/im.gif/" 48 | PathIPNS1 = "/ipns/QmbmSAQNnfGcBAB8M8AsSPxd1TY7cpT9hZ398kXAScn2Ka" 49 | PathIPNS2 = "/ipns/QmbmSAQNnfGcBAB8M8AsSPxd1TY7cpT9hZ398kXAScn2Ka/" 50 | PathIPLD1 = "/ipld/QmaNJ5acV31sx8jq626qTpAWW4DXKw34aGhx53dECLvXbY" 51 | PathIPLD2 = "/ipld/QmaNJ5acV31sx8jq626qTpAWW4DXKw34aGhx53dECLvXbY/" 52 | 53 | // NotFoundPath is meant to be used as a path that resolves into a CID that doesn't exist in the 54 | // pinset. 55 | NotFoundPath = "/ipfs/bafyreiay3jpjk74dkckv2r74eyvf3lfnxujefay2rtuluintasq2zlapv4" 56 | InvalidPath1 = "/invalidkeytype/QmaNJ5acV31sx8jq626qTpAWW4DXKw34aGhx53dECLvXbY/" 57 | InvalidPath2 = "/ipfs/invalidhash" 58 | InvalidPath3 = "/ipfs/" 59 | ) 60 | -------------------------------------------------------------------------------- /test/test.go: -------------------------------------------------------------------------------- 1 | // Package test offers testing utilities for all the IPFS Cluster 2 | // codebase, like IPFS daemon and RPC mocks and pre-defined testing CIDs. 3 | package test 4 | -------------------------------------------------------------------------------- /test/test_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | ipfscluster "github.com/ipfs-cluster/ipfs-cluster" 8 | ) 9 | 10 | func TestIpfsMock(t *testing.T) { 11 | ipfsmock := NewIpfsMock(t) 12 | defer ipfsmock.Close() 13 | } 14 | 15 | // Test that our RPC mock resembles the original 16 | func TestRPCMockValid(t *testing.T) { 17 | type tc struct { 18 | mock reflect.Type 19 | real reflect.Type 20 | } 21 | 22 | tcs := []tc{ 23 | { 24 | real: reflect.TypeOf(&ipfscluster.ClusterRPCAPI{}), 25 | mock: reflect.TypeOf(&mockCluster{}), 26 | }, 27 | { 28 | real: reflect.TypeOf(&ipfscluster.PinTrackerRPCAPI{}), 29 | mock: reflect.TypeOf(&mockPinTracker{}), 30 | }, 31 | { 32 | real: reflect.TypeOf(&ipfscluster.IPFSConnectorRPCAPI{}), 33 | mock: reflect.TypeOf(&mockIPFSConnector{}), 34 | }, 35 | { 36 | real: reflect.TypeOf(&ipfscluster.ConsensusRPCAPI{}), 37 | mock: reflect.TypeOf(&mockConsensus{}), 38 | }, 39 | { 40 | real: reflect.TypeOf(&ipfscluster.PeerMonitorRPCAPI{}), 41 | mock: reflect.TypeOf(&mockPeerMonitor{}), 42 | }, 43 | } 44 | 45 | for _, tc := range tcs { 46 | realT := tc.real 47 | mockT := tc.mock 48 | 49 | // Make sure all the methods we have match the original 50 | for i := 0; i < mockT.NumMethod(); i++ { 51 | method := mockT.Method(i) 52 | name := method.Name 53 | origMethod, ok := realT.MethodByName(name) 54 | if !ok { 55 | t.Fatalf("%s method not found in real RPC", name) 56 | } 57 | 58 | mType := method.Type 59 | oType := origMethod.Type 60 | 61 | if nout := mType.NumOut(); nout != 1 || nout != oType.NumOut() { 62 | t.Errorf("%s: more than 1 out parameter", name) 63 | } 64 | 65 | if mType.Out(0).Name() != "error" { 66 | t.Errorf("%s out param should be an error", name) 67 | } 68 | 69 | if nin := mType.NumIn(); nin != oType.NumIn() || nin != 4 { 70 | t.Fatalf("%s: num in parameter mismatch: %d vs. %d", name, nin, oType.NumIn()) 71 | } 72 | 73 | for j := 1; j < 4; j++ { 74 | mn := mType.In(j).String() 75 | on := oType.In(j).String() 76 | if mn != on { 77 | t.Errorf("%s: name mismatch: %s vs %s", name, mn, on) 78 | } 79 | } 80 | } 81 | 82 | for i := 0; i < realT.NumMethod(); i++ { 83 | name := realT.Method(i).Name 84 | _, ok := mockT.MethodByName(name) 85 | if !ok { 86 | t.Logf("Warning: %s: unimplemented in mock rpc", name) 87 | } 88 | } 89 | } 90 | } 91 | 92 | // Test that testing directory is created without error 93 | func TestGenerateTestDirs(t *testing.T) { 94 | sth := NewShardingTestHelper() 95 | defer sth.Clean(t) 96 | _, closer := sth.GetTreeMultiReader(t) 97 | closer.Close() 98 | _, closer = sth.GetRandFileMultiReader(t, 2) 99 | closer.Close() 100 | } 101 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | // Package version stores version information for IPFS Cluster. 2 | package version 3 | 4 | import ( 5 | semver "github.com/blang/semver" 6 | protocol "github.com/libp2p/go-libp2p/core/protocol" 7 | ) 8 | 9 | // Version is the current cluster version. 10 | var Version = semver.MustParse("1.1.2") 11 | 12 | // RPCProtocol is protocol handler used to send libp2p-rpc messages between 13 | // cluster peers. All peers in the cluster need to speak the same protocol 14 | // version. 15 | // 16 | // The RPC Protocol is not linked to the IPFS Cluster version (though it once 17 | // was). The protocol version will be updated as needed when breaking changes 18 | // are introduced, though at this point we aim to minimize those as much as 19 | // possible. 20 | var RPCProtocol = protocol.ID("/ipfscluster/1.0/rpc") 21 | --------------------------------------------------------------------------------