├── .dockerignore ├── .github ├── actions │ ├── go-check-setup │ │ └── action.yml │ └── go-test-setup │ │ └── action.yml └── workflows │ ├── ecr-publisher.yml │ ├── go-check.yml │ └── go-test.yml ├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── Makefile ├── README.md ├── autoretrieve.go ├── bitswap ├── provider.go └── util.go ├── blocks ├── manager.go ├── manager_test.go └── randompruner.go ├── config.go ├── docker-compose.yml ├── docker-env ├── docs └── updating-docker-image-on-ec2.md ├── endpoint └── estuary.go ├── go.mod ├── go.sum ├── keystore └── keystore.go ├── main.go ├── messagepusher └── msgpusher.go ├── metrics ├── opentelemetry.go └── server.go ├── minerpeergetter └── minerpeergetter.go ├── paychannelapi.go ├── paychannelmanager └── lassiepaychannelmanager.go └── scripts └── autoretrieve-service ├── autoretrieve-register.service ├── autoretrieve.service └── config.env /.dockerignore: -------------------------------------------------------------------------------- 1 | data/ 2 | extern/ 3 | .git/ -------------------------------------------------------------------------------- /.github/actions/go-check-setup/action.yml: -------------------------------------------------------------------------------- 1 | runs: 2 | using: "composite" 3 | steps: 4 | - name: Prepare for FFI build 5 | shell: bash 6 | run: | 7 | sudo apt-get update 8 | sudo apt-get install ocl-icd-opencl-dev libhwloc-dev -y 9 | - name: Install & Build FFI 10 | shell: bash 11 | run: make all -------------------------------------------------------------------------------- /.github/actions/go-test-setup/action.yml: -------------------------------------------------------------------------------- 1 | runs: 2 | using: "composite" 3 | steps: 4 | - name: Prepare for FFI build 5 | shell: bash 6 | run: make ffi_install_dependencies 7 | env: 8 | GITHUB_TOKEN: ${{ github.token }} 9 | - name: Install & Build FFI 10 | shell: bash 11 | run: make all 12 | env: 13 | GITHUB_TOKEN: ${{ github.token }} 14 | -------------------------------------------------------------------------------- /.github/workflows/ecr-publisher.yml: -------------------------------------------------------------------------------- 1 | name: ECR 2 | 3 | on: 4 | release: 5 | types: 6 | - published 7 | push: 8 | paths-ignore: 9 | - 'deploy/**' 10 | - 'docs/**' 11 | branches: 12 | - master 13 | - staging 14 | 15 | jobs: 16 | publisher: 17 | if: ${{ github.event.pusher.name != 'sti-bot' }} 18 | name: Publish 19 | runs-on: ubuntu-latest 20 | permissions: 21 | id-token: write 22 | contents: read 23 | env: 24 | ECR_REGISTRY: 407967248065.dkr.ecr.us-east-2.amazonaws.com/autoretrieve 25 | steps: 26 | - name: Checkout 27 | uses: actions/checkout@v2 28 | - name: Determine Container Tag 29 | run: | 30 | IMAGE_TAG="${GITHUB_REF#refs/tags/v}" 31 | if test "${IMAGE_TAG}" = "${GITHUB_REF}"; then 32 | IMAGE_TAG="$(date '+%Y%m%d%H%M%S')-${GITHUB_REF##*/}-${GITHUB_SHA}" 33 | fi 34 | echo "Using image tag: ${IMAGE_TAG}" 35 | echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_ENV 36 | - name: AWS Login 37 | uses: aws-actions/configure-aws-credentials@v1 38 | with: 39 | aws-region: us-east-2 40 | role-to-assume: "arn:aws:iam::407967248065:role/common/github_actions" 41 | role-duration-seconds: 1200 42 | - name: Login to Amazon ECR 43 | run: aws ecr get-login-password | docker login --username AWS --password-stdin ${ECR_REGISTRY} 44 | - name: Publish Container Image 45 | env: 46 | DOCKER_BUILDKIT: '1' 47 | run: | 48 | IMAGE_NAME="${ECR_REGISTRY}/autoretrieve:${IMAGE_TAG}" 49 | docker build -t "${IMAGE_NAME}" . 50 | docker push "${IMAGE_NAME}" 51 | echo "Published image ${IMAGE_NAME}" 52 | -------------------------------------------------------------------------------- /.github/workflows/go-check.yml: -------------------------------------------------------------------------------- 1 | # Based on https://github.com/protocol/.github/ templates 2 | 3 | on: [push, pull_request] 4 | name: Go Checks 5 | 6 | jobs: 7 | unit: 8 | runs-on: ubuntu-latest 9 | name: All 10 | env: 11 | RUNGOGENERATE: false 12 | steps: 13 | - uses: actions/checkout@v2 14 | with: 15 | submodules: recursive 16 | - uses: actions/setup-go@v2 17 | with: 18 | go-version: "1.19.x" 19 | - name: Run repo-specific setup 20 | uses: ./.github/actions/go-check-setup 21 | if: hashFiles('./.github/actions/go-check-setup') != '' 22 | - name: Read config 23 | if: hashFiles('./.github/workflows/go-check-config.json') != '' 24 | run: | 25 | if jq -re .gogenerate ./.github/workflows/go-check-config.json; then 26 | echo "RUNGOGENERATE=true" >> $GITHUB_ENV 27 | fi 28 | - name: Install staticcheck 29 | run: go install honnef.co/go/tools/cmd/staticcheck@376210a89477dedbe6fdc4484b233998650d7b3c # 2022.1.3 (v0.3.3) 30 | - name: Check that go.mod is tidy 31 | run: | 32 | go mod tidy 33 | if [[ -n $(git ls-files --other --exclude-standard --directory -- go.sum) ]]; then 34 | echo "go.sum was added by go mod tidy" 35 | exit 1 36 | fi 37 | git diff --exit-code -- go.sum go.mod 38 | - name: gofmt 39 | if: ${{ success() || failure() }} # run this step even if the previous one failed 40 | run: | 41 | out=$(find . -path '*/extern/*' -prune -o -name '*.go' -type f -exec gofmt -s -l {} \;) 42 | if [[ -n "$out" ]]; then 43 | echo $out | awk '{print "::error file=" $0 ",line=0,col=0::File is not gofmt-ed."}' 44 | exit 1 45 | fi 46 | - name: go vet 47 | if: ${{ success() || failure() }} # run this step even if the previous one failed 48 | run: go vet ./... 49 | - name: staticcheck 50 | if: ${{ success() || failure() }} # run this step even if the previous one failed 51 | run: | 52 | set -o pipefail 53 | staticcheck ./... | sed -e 's@\(.*\)\.go@./\1.go@g' 54 | - name: go generate 55 | if: (success() || failure()) && env.RUNGOGENERATE == 'true' 56 | run: | 57 | git clean -fd # make sure there aren't untracked files / directories 58 | go generate ./... 59 | # check if go generate modified or added any files 60 | if ! $(git add . && git diff-index HEAD --exit-code --quiet); then 61 | echo "go generated caused changes to the repository:" 62 | git status --short 63 | exit 1 64 | fi 65 | -------------------------------------------------------------------------------- /.github/workflows/go-test.yml: -------------------------------------------------------------------------------- 1 | # Based on https://github.com/protocol/.github/ templates 2 | 3 | on: [push, pull_request] 4 | name: Go Test 5 | 6 | jobs: 7 | unit: 8 | strategy: 9 | fail-fast: false 10 | matrix: 11 | os: [ "ubuntu", "macos" ] 12 | go: [ "1.19.x" ] 13 | env: 14 | COVERAGES: "" 15 | runs-on: ${{ format('{0}-latest', matrix.os) }} 16 | name: ${{ matrix.os }} (go ${{ matrix.go }}) 17 | steps: 18 | - uses: actions/checkout@v2 19 | with: 20 | submodules: recursive 21 | - uses: actions/setup-go@v2 22 | with: 23 | go-version: ${{ matrix.go }} 24 | - name: Go information 25 | run: | 26 | go version 27 | go env 28 | - name: Run repo-specific setup 29 | uses: ./.github/actions/go-test-setup 30 | if: hashFiles('./.github/actions/go-test-setup') != '' 31 | - name: Run tests 32 | run: go test -v -coverprofile=module-coverage.txt -coverpkg=./... ./... 33 | - name: Run tests with race detector 34 | if: ${{ matrix.os == 'ubuntu' }} 35 | run: go test -v -race ./... 36 | - name: Collect coverage files 37 | shell: bash 38 | run: echo "COVERAGES=$(find . -type f -name 'module-coverage.txt' | tr -s '\n' ',' | sed 's/,$//')" >> $GITHUB_ENV 39 | - name: Upload coverage to Codecov 40 | uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # v2.1.0 41 | with: 42 | files: '${{ env.COVERAGES }}' 43 | env_vars: OS=${{ matrix.os }}, GO=${{ matrix.go }} 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /data 2 | /extern 3 | /autoretrieve 4 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). 3 | 4 | ## 2022-07-10 5 | ### Added 6 | - YAML option `advertise-endpoint-url` 7 | - YAML option `advertise-token` 8 | 9 | ### Changed 10 | - Use config defaults for missing config.yaml properties instead of zero-values 11 | - Flag `--endpoint-type` to `--lookup-endpoint-type` 12 | - Flag `--endpoint-url` to `--lookup-endpoint-url` 13 | - YAML option `endpoint-type` to `lookup-endpoint-type` 14 | - YAML option `endpoint-url` to `lookup-endpoint-url` 15 | - Env var `AUTORETRIEVE_ENDPOINT_TYPE` to `AUTORETRIEVE_LOOKUP_ENDPOINT_TYPE` 16 | - Env var `AUTORETRIEVE_ENDPOINT_URL` to `AUTORETRIEVE_LOOKUP_ENDPOINT_URL` 17 | 18 | ## 2022-07-01 19 | ### Changed 20 | - Flag `--use-fullrt [true|false]` to `--routing-table-type [dht|full|disabled]` 21 | - YAML option `use-fullrt: [true|false]` to `routing-table-type: [dht|full|disabled]` 22 | - Env var `AUTORETRIEVE_USE_FULLRT=[true|false]` to `AUTORETRIEVE_ROUTING_TABLE_TYPE=[dht|full|disabled]` 23 | 24 | ## feat/reorg-and-config - 2022-04-?? 25 | ### Added 26 | - Self-contained Autoretrieve and accompanying Config types, and separate from CLI code 27 | - Config file `config.yaml` 28 | - Subcommand `print-config` 29 | - Config file generation on startup if none present 30 | - Can now use either peer ID or storage provider address in config.yaml 31 | 32 | ### Changed 33 | - Flag `--datadir` to `--data-dir` 34 | - Flag `--endpoint` to `--endpoint-url` 35 | - Default endpoint type from `estuary` to `indexer` 36 | - Default endpoint URL from `https://api.estuary.tech/retrieval-candidates` to `https://cid.contact` 37 | 38 | ### Removed 39 | - Flag `--max-send-workers` 40 | - Flag `--per-miner-retrieval-timeout` 41 | - Flag `--timeout` 42 | - Flag `--miner-whitelist` 43 | - Flag `--miner-blacklist` 44 | - Flag `--cid-blacklist` 45 | - Subcommand `check-miner-whitelist` 46 | - Subcommand `check-miner-blacklist` 47 | - Subcommand `check-cid-blacklist` 48 | - Config file `miner-blacklist.txt` 49 | - Config file `miner-whitelist.txt` 50 | - Config file `cid-blacklist.txt` 51 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Autoretrieve uses Docker Buildkit to speed up builds, make sure 2 | # DOCKER_BUILDKIT=1 is set 3 | 4 | FROM golang:1.19 AS builder 5 | 6 | # Install rustup 7 | ENV RUSTUP_HOME=/usr/local/rustup \ 8 | CARGO_HOME=/usr/local/cargo \ 9 | PATH=/usr/local/cargo/bin:$PATH 10 | RUN wget "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init"; \ 11 | chmod +x rustup-init && \ 12 | ./rustup-init -y --no-modify-path --profile minimal --default-toolchain nightly 13 | 14 | # Install build dependencies 15 | RUN --mount=type=cache,target=/var/cache/apt \ 16 | apt update && \ 17 | apt install -y --no-install-recommends jq libhwloc-dev ocl-icd-opencl-dev 18 | 19 | # Build app 20 | WORKDIR /app 21 | COPY . . 22 | RUN --mount=type=cache,target=/root/.cache/go-build \ 23 | --mount=type=cache,target=/go/pkg/mod \ 24 | --mount=type=cache,target=/app/extern \ 25 | make 26 | 27 | FROM ubuntu:22.04 28 | RUN --mount=type=cache,target=/var/cache/apt \ 29 | apt-get update && \ 30 | apt-get install -y --no-install-recommends jq libhwloc-dev ocl-icd-opencl-dev ca-certificates 31 | WORKDIR /app 32 | COPY --from=builder /app/autoretrieve autoretrieve 33 | 34 | # Create the volume for the autoretrieve data directory 35 | # This is the default directory for autoretrieve configuration for this docker container 36 | VOLUME /root/.autoretrieve 37 | 38 | # Libp2p port 39 | EXPOSE 6746 40 | 41 | # Http(s) ports 42 | EXPOSE 80 43 | EXPOSE 443 44 | 45 | # This env var is required deep in lotus 46 | # Hardcoding for now for convenience until we have a reason to make it easily configurable 47 | ENV FULLNODE_API_INFO="wss://api.chain.love" 48 | 49 | ENTRYPOINT [ "./autoretrieve" ] -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # commit or branch for the extern/filecoin-ffi dependency 2 | filecoin_ffi_branch = 32afd6e1f1419b6b 3 | 4 | ffi_remote = https://github.com/filecoin-project/filecoin-ffi 5 | ffi_dir = extern/filecoin-ffi 6 | ffi_update = $(ffi_dir)-$(ffi_dir) # ffi exists 7 | ffi_checkout = $(ffi_dir)- # ffi doesn't yet exist 8 | ffi_target = $(ffi_dir)-$(wildcard $(ffi_dir)) # switch between update & checkout depending on status 9 | # if ffi is cloned, check the ref we have and the ref we want so we can compare in ffi_update 10 | ffi_expected_ref = $(shell [ -d "${ffi_dir}" ] && cd ${ffi_dir} && git rev-parse -q HEAD) 11 | ffi_current_ref = $(shell [ -d "${ffi_dir}" ] && cd ${ffi_dir} && git rev-parse -q ${filecoin_ffi_branch}) 12 | os_uname = $(shell uname) 13 | 14 | all: autoretrieve 15 | .PHONY: all 16 | 17 | autoretrieve: | $(ffi_target) 18 | go build 19 | 20 | # we have FFI checked out, make sure it's running the commit we want, update it if not 21 | $(ffi_update): 22 | ifneq ($(ffi_expected_ref), $(ffi_current_ref)) 23 | @echo Commit changed, updating and rebuilding FFI ... 24 | cd $(ffi_dir) && git fetch $(ffi_remote) && git checkout $(filecoin_ffi_branch) && make 25 | endif 26 | 27 | # we don't have FFI checked out, clone and build 28 | $(ffi_checkout): ffi_checkout 29 | cd $(ffi_dir) && make 30 | 31 | ffi_checkout: 32 | @echo Cloning and rebuilding FFI ... 33 | git clone $(ffi_remote) -b $(filecoin_ffi_branch) $(ffi_dir) 34 | 35 | # intended to be used in CI to prepare the environemnt to install FFI 36 | ffi_install_dependencies: 37 | ifeq ($(os_uname),Darwin) 38 | # assumes availability of Homebrew 39 | brew install hwloc 40 | endif 41 | ifeq ($(os_uname),Linux) 42 | # assumes an APT based Linux 43 | sudo apt-get update 44 | sudo apt-get install ocl-icd-opencl-dev libhwloc-dev -y 45 | endif 46 | 47 | .PHONY: install 48 | install: autoretrieve 49 | install -C autoretrieve /usr/local/bin/autoretrieve 50 | 51 | .PHONY: install-autoretrieve-service 52 | install-autoretrieve-service: 53 | cp scripts/autoretrieve-service/autoretrieve-register.service /etc/systemd/system/autoretrieve-register.service 54 | cp scripts/autoretrieve-service/autoretrieve.service /etc/systemd/system/autoretrieve.service 55 | mkdir -p /etc/autoretrieve 56 | cp scripts/autoretrieve-service/config.env /etc/autoretrieve/config.env 57 | 58 | #TODO: if service changes to autoretrieve user/group, need to chown the /etc/autoretrieve dir and contents 59 | 60 | systemctl daemon-reload 61 | 62 | #Edit config values in /etc/autoretrieve/config.env before running any autoretrieve service files 63 | #Run 'sudo systemctl start autoretrieve-setup.service' to complete setup 64 | #Run 'sudo systemctl enable --now autoretrieve.service' once ready to enable and start autoretrieve service 65 | 66 | clean: 67 | rm -rf extern autoretrieve 68 | .PHONY: clean 69 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Autoretrieve 2 | 3 | Autoretrieve is a standalone Graphsync-to-Bitswap proxy server, which allows IPFS clients to retrieve data which may be available on the Filecoin network but not on IPFS (such as data that has become unpinned, or data that was uploaded to a Filecoin storage provider but was never sent to an IPFS provider). 4 | 5 | ## What problem does Autoretrieve solve? 6 | 7 | Protocol Labs develops two decentralized data transfer protocols - Bitswap and GraphSync, which back the IPFS and Filecoin networks, respectively. Although built on similar principles, these networks are fundamentally incompatible and separate - a client of one network cannot retrieve data from a provider of the other. [TODO: link more info about differences.] This raises the following issue: what if there is data that exists on Filecoin, but not on IPFS? 8 | 9 | Autoretrieve is a "translational proxy" that allows data to be transferred from Filecoin to IPFS in an automated fashion. The existing alternatives for Autoretrieve's Filecoin-to-IPFS flow are the Boost IPFS node that providers may optionally enable, and manual transfer. 10 | 11 | [Boost IPFS node](https://github.com/filecoin-project/boost/issues/709) is not always a feasible option for several reasons: 12 | - Providers are not incentivized to enable this feature 13 | - Only free retrievals are supported 14 | 15 | In comparison, Autoretrieve: 16 | - Is a dedicated node, and operational burden/cost does not fall on storage provider operators 17 | - Supports paid retrievals (the Autoretrieve node operator covers the payment) 18 | 19 | ## How does Autoretrieve work? 20 | 21 | Autoretrieve is at its core a Bitswap server. When a Bitswap request comes in, Autoretrieve queries an indexer for Filecoin storage providers that have the requested CID. The providers are sorted and retrieval is attempted sequentially until a successful retrieval is opened. As the GraphSync data lands into Autoretrieve from the storage provider, the data is streamed live back to the IPFS client. 22 | 23 | In order for IPFS clients to be able to retrieve Filecoin data using Autoretrieve, they must be connected to Autoretrieve. Currently, Autoretrieve can be advertised to the indexer (and by extension the DHT) by Estuary. Autoretrieve does not currently have an independent way to advertise its own data. 24 | 25 | If an Autoretrieve node is not advertised, clients may still download data from it if a connection is established either manually, or by chance while walking through the DHT searching for other providers. 26 | 27 | ## Usage 28 | 29 | Autoretrieve uses Docker with Buildkit for build caching. Docker rebuilds are 30 | quite fast, and it is usable for local development. Check the docker-compose 31 | documentation for more help. 32 | 33 | ```console 34 | $ DOCKER_BUILDKIT=1 docker-compose up 35 | ``` 36 | 37 | You may optionally set `FULLNODE_API_INFO` to a custom fullnode's WebSocket 38 | address. The default is `FULLNODE_API_INFO=wss://api.chain.love`. 39 | 40 | By default, config files and cache are stored at `~/.autoretrieve`. When using 41 | docker-compose, a binding is created to this directory. This location can be 42 | configured by setting `AUTORETRIEVE_DATA_DIR`. 43 | 44 | Internally, the Docker volume's path on the image is `/root/.autoretrieve`. Keep 45 | this in mind when using the Docker image directly. 46 | 47 | ## Configuration 48 | 49 | Some CLI flags and corresponding environment variables are available for basic configuration. 50 | 51 | For more advanced configuration, `config.yaml` may be used. It lives in the autoretrieve data directory, and will be automatically generated by running autoretrieve. It may also be manually generated using the `gen-config` subcommand. 52 | 53 | Configurations are applied in the following order, from least to most important: 54 | - YAML config 55 | - Environment variables 56 | - CLI flags 57 | 58 | ### YAML Example 59 | 60 | ```yaml 61 | advertise-endpoint-url: # leave blank to disable, example https://api.estuary.tech/autoretrieve/heartbeat (must be registered) 62 | advertise-endpoint-token: # leave blank to disable 63 | lookup-endpoint-type: indexer # indexer | estuary 64 | lookup-endpoint-url: https://cid.contact # for estuary endpoint-type: https://api.estuary.tech/retrieval-candidates 65 | max-bitswap-workers: 1 66 | routing-table-type: dht 67 | prune-threshold: 1GiB # 1000000000, 1 GB, etc. Uses go-humanize for parsing. Table of valid byte sizes can be found here: https://github.com/dustin/go-humanize/blob/v1.0.0/bytes.go#L34-L62 68 | pin-duration: 1h # 1h30m, etc. 69 | log-resource-manager: false 70 | log-retrieval-stats: false 71 | disable-retrieval: false 72 | cid-blacklist: 73 | - QmCID01234 74 | - QmCID56789 75 | - QmCIDabcde 76 | miner-blacklist: 77 | - f01234 78 | - f05678 79 | miner-whitelist: 80 | - f01234 81 | default-miner-config: 82 | retrieval-timeout: 1m 83 | max-concurrent-retrievals: 1 84 | miner-configs: 85 | f01234: 86 | retrieval-timeout: 2m30s 87 | max-concurrent-retrievals: 2 88 | f05678: 89 | max-concurrent-retrievals: 10 90 | ``` 91 | -------------------------------------------------------------------------------- /autoretrieve.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "crypto/rand" 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "io" 11 | "net/http" 12 | "net/url" 13 | "os" 14 | "path/filepath" 15 | "strings" 16 | "time" 17 | 18 | "github.com/application-research/autoretrieve/bitswap" 19 | "github.com/application-research/autoretrieve/blocks" 20 | "github.com/application-research/autoretrieve/endpoint" 21 | "github.com/application-research/autoretrieve/keystore" 22 | "github.com/application-research/autoretrieve/messagepusher" 23 | "github.com/application-research/autoretrieve/minerpeergetter" 24 | "github.com/application-research/autoretrieve/paychannelmanager" 25 | lassieclient "github.com/filecoin-project/lassie/pkg/client" 26 | lassieeventrecorder "github.com/filecoin-project/lassie/pkg/eventrecorder" 27 | "github.com/filecoin-project/lassie/pkg/indexerlookup" 28 | lassieretriever "github.com/filecoin-project/lassie/pkg/retriever" 29 | rpcstmgr "github.com/filecoin-project/lotus/chain/stmgr/rpc" 30 | "github.com/filecoin-project/lotus/chain/wallet" 31 | lcli "github.com/filecoin-project/lotus/cli" 32 | "github.com/filecoin-project/lotus/paychmgr" 33 | ipfsdatastore "github.com/ipfs/go-datastore" 34 | "github.com/ipfs/go-datastore/namespace" 35 | flatfs "github.com/ipfs/go-ds-flatfs" 36 | leveldb "github.com/ipfs/go-ds-leveldb" 37 | blockstore "github.com/ipfs/go-ipfs-blockstore" 38 | "github.com/libp2p/go-libp2p" 39 | "github.com/libp2p/go-libp2p/core/crypto" 40 | "github.com/libp2p/go-libp2p/core/host" 41 | "github.com/libp2p/go-libp2p/core/network" 42 | peer "github.com/libp2p/go-libp2p/core/peer" 43 | "github.com/multiformats/go-multiaddr" 44 | "github.com/urfave/cli/v2" 45 | ) 46 | 47 | var statFmtString = `global conn stats: 48 | memory: %d, 49 | number of inbound conns: %d, 50 | number of outbound conns: %d, 51 | number of file descriptors: %d, 52 | number of inbound streams: %d, 53 | number of outbound streams: %d, 54 | ` 55 | 56 | type Autoretrieve struct { 57 | host host.Host 58 | retriever *lassieretriever.Retriever 59 | provider *bitswap.Provider 60 | apiCloser func() 61 | } 62 | 63 | func New(cctx *cli.Context, dataDir string, cfg Config) (*Autoretrieve, error) { 64 | 65 | if dataDir == "" { 66 | return nil, fmt.Errorf("dataDir must be set") 67 | } 68 | 69 | if err := os.MkdirAll(dataDir, 0744); err != nil { 70 | return nil, err 71 | } 72 | 73 | // Initialize P2P host 74 | host, err := initHost(cctx.Context, dataDir, cfg.LogResourceManager, multiaddr.StringCast("/ip4/0.0.0.0/tcp/6746")) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | // Open Lotus API 80 | api, apiCloser, err := lcli.GetGatewayAPI(cctx) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | // Initialize blockstore manager 86 | parseShardFunc, err := flatfs.ParseShardFunc("/repo/flatfs/shard/v1/next-to-last/3") 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | blockstoreDatastore, err := flatfs.CreateOrOpen(filepath.Join(dataDir, blockstoreSubdir), parseShardFunc, false) 92 | if err != nil { 93 | return nil, err 94 | } 95 | 96 | blockstore := blockstore.NewBlockstoreNoPrefix(blockstoreDatastore) 97 | 98 | // Only wrap blockstore with pruner when a prune threshold is specified 99 | if cfg.PruneThreshold != 0 { 100 | blockstore, err = blocks.NewRandomPruner(cctx.Context, blockstore, blockstoreDatastore, blocks.RandomPrunerConfig{ 101 | Threshold: uint64(cfg.PruneThreshold), 102 | PinDuration: cfg.PinDuration, 103 | }) 104 | 105 | if err != nil { 106 | return nil, err 107 | } 108 | } else { 109 | logger.Warnf("No prune threshold provided, blockstore garbage collection will not be performed") 110 | } 111 | 112 | // TODO: make configurable 113 | blockManager := blocks.NewManager(blockstore, time.Minute*10) 114 | if err != nil { 115 | return nil, err 116 | } 117 | 118 | // Open datastore 119 | datastore, err := leveldb.NewDatastore(filepath.Join(dataDir, datastoreSubdir), nil) 120 | if err != nil { 121 | return nil, err 122 | } 123 | 124 | // Set up wallet 125 | keystore, err := keystore.OpenOrInitKeystore(filepath.Join(dataDir, walletSubdir)) 126 | if err != nil { 127 | return nil, fmt.Errorf("keystore initialization failed: %w", err) 128 | } 129 | 130 | wallet, err := wallet.NewWallet(keystore) 131 | if err != nil { 132 | return nil, fmt.Errorf("wallet initialization failed: %w", err) 133 | } 134 | 135 | // Set up the PayChannelManager 136 | ctx, shutdown := context.WithCancel(context.Background()) 137 | mpusher := messagepusher.NewMsgPusher(api, wallet) 138 | rpcStateMgr := rpcstmgr.NewRPCStateManager(api) 139 | pchds := namespace.Wrap(datastore, ipfsdatastore.NewKey("paych")) 140 | payChanStore := paychmgr.NewStore(pchds) 141 | payChanApi := &payChannelApiProvider{ 142 | Gateway: api, 143 | wallet: wallet, 144 | mp: mpusher, 145 | } 146 | 147 | payChanMgr := paychannelmanager.NewLassiePayChannelManager(ctx, shutdown, rpcStateMgr, payChanStore, payChanApi) 148 | if err := payChanMgr.Start(); err != nil { 149 | return nil, err 150 | } 151 | 152 | minerPeerGetter := minerpeergetter.NewMinerPeerGetter(api) 153 | 154 | // Initialize Filecoin retriever 155 | var retriever *lassieretriever.Retriever 156 | if !cfg.DisableRetrieval { 157 | var candidateFinder lassieretriever.CandidateFinder 158 | switch cfg.LookupEndpointType { 159 | case EndpointTypeEstuary: 160 | logger.Infof("Using Estuary candidate finder type") 161 | candidateFinder = endpoint.NewEstuaryEndpoint(cfg.LookupEndpointURL, minerPeerGetter) 162 | case EndpointTypeIndexer: 163 | logger.Infof("Using indexer candidate finder type") 164 | candidateFinder = indexerlookup.NewCandidateFinder(cfg.LookupEndpointURL) 165 | default: 166 | return nil, errors.New("unrecognized candidate finder type") 167 | } 168 | 169 | retrieverCfg, err := cfg.ExtractFilecoinRetrieverConfig(cctx.Context, minerPeerGetter) 170 | if err != nil { 171 | return nil, err 172 | } 173 | 174 | // Instantiate client 175 | retrievalClient, err := lassieclient.NewClient( 176 | datastore, 177 | host, 178 | payChanMgr, 179 | ) 180 | if err != nil { 181 | return nil, err 182 | } 183 | 184 | if err := retrievalClient.AwaitReady(); err != nil { 185 | return nil, err 186 | } 187 | 188 | retriever, err = lassieretriever.NewRetriever(cctx.Context, retrieverCfg, retrievalClient, candidateFinder) 189 | if err != nil { 190 | return nil, err 191 | } 192 | retriever.Start() 193 | if cfg.EventRecorderEndpointURL != "" { 194 | logger.Infof("Reporting retrieval events to %v", cfg.EventRecorderEndpointURL) 195 | eventRecorderEndpointAuthorization, err := loadEventRecorderAuth(dataDirPath(cctx)) 196 | if err != nil { 197 | return nil, err 198 | } 199 | eventRecorder := lassieeventrecorder.NewEventRecorder(cctx.Context, cfg.InstanceId, cfg.EventRecorderEndpointURL, eventRecorderEndpointAuthorization) 200 | retriever.RegisterSubscriber(eventRecorder.RecordEvent) 201 | } 202 | } 203 | 204 | // Initialize Bitswap provider 205 | provider, err := bitswap.NewProvider( 206 | cctx.Context, 207 | cfg.ExtractBitswapProviderConfig(cctx.Context), 208 | host, 209 | datastore, 210 | blockManager, 211 | retriever, // This will be nil if --disable-retrieval is passed 212 | ) 213 | if err != nil { 214 | return nil, err 215 | } 216 | 217 | // Start Estuary heartbeat goroutine if an endpoint was specified 218 | if cfg.EstuaryURL != "" { 219 | _, err := url.Parse(cfg.EstuaryURL) 220 | if err != nil { 221 | return nil, fmt.Errorf("could not parse Estuary URL: %w", err) 222 | } 223 | 224 | go func() { 225 | logger.Infof("Starting estuary heartbeat ticker with heartbeat interval %s", cfg.HeartbeatInterval) 226 | 227 | ticker := time.NewTicker(cfg.HeartbeatInterval) 228 | if ticker == nil { 229 | logger.Infof("Error setting ticker") 230 | } 231 | for ; true; <-ticker.C { 232 | logger.Infof("Sending Estuary heartbeat message") 233 | interval, err := sendEstuaryHeartbeat(&cfg) 234 | if err != nil { 235 | logger.Errorf("Failed to send Estuary heartbeat message - resetting to default heartbeat interval of %s: %v", cfg.HeartbeatInterval, err) 236 | ticker.Reset(cfg.HeartbeatInterval) 237 | } else { 238 | logger.Infof("Next Estuary heartbeat in %s", interval) 239 | ticker.Reset(interval) 240 | } 241 | } 242 | }() 243 | } 244 | 245 | logger.Infof("Using peer ID: %s", host.ID()) 246 | 247 | return &Autoretrieve{ 248 | host: host, 249 | retriever: retriever, 250 | provider: provider, 251 | apiCloser: apiCloser, 252 | }, nil 253 | } 254 | 255 | func sendEstuaryHeartbeat(cfg *Config) (time.Duration, error) { 256 | 257 | // Set ticker 258 | 259 | req, err := http.NewRequest("POST", cfg.EstuaryURL+"/autoretrieve/heartbeat", bytes.NewBuffer(nil)) 260 | if err != nil { 261 | return 0, err 262 | } 263 | 264 | req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.AdvertiseToken)) 265 | res, err := http.DefaultClient.Do(req) 266 | if err != nil { 267 | return 0, err 268 | } 269 | defer res.Body.Close() 270 | 271 | // If we got a non-success status code, warn about the error 272 | if res.StatusCode/100 != 2 { 273 | resBytes, err := io.ReadAll(res.Body) 274 | resString := string(resBytes) 275 | 276 | if err != nil { 277 | resString = "could not read response" 278 | } 279 | return 0, fmt.Errorf("%s", resString) 280 | } 281 | 282 | var output struct { 283 | Handle string 284 | LastConnection time.Time 285 | LastAdvertisement time.Time 286 | AddrInfo peer.AddrInfo 287 | AdvertiseInterval string 288 | Error interface{} 289 | } 290 | if err := json.NewDecoder(res.Body).Decode(&output); err != nil { 291 | return 0, fmt.Errorf("could not decode response: %v", err) 292 | } 293 | 294 | if output.Error != nil && output.Error != "" { 295 | return 0, fmt.Errorf("%v", output.Error) 296 | } 297 | 298 | advInterval, err := time.ParseDuration(output.AdvertiseInterval) 299 | if err != nil { 300 | return 0, fmt.Errorf("could not parse advertise interval: %s", err) 301 | } 302 | 303 | return advInterval / 2, nil 304 | } 305 | 306 | // TODO: this function should do a lot more to clean up resources and come to 307 | // safe stop 308 | func (autoretrieve *Autoretrieve) Close() { 309 | autoretrieve.apiCloser() 310 | autoretrieve.host.Close() 311 | } 312 | 313 | func loadEventRecorderAuth(dataDir string) (string, error) { 314 | authPath := filepath.Join(dataDir, "eventrecorderauth") 315 | eventRecorderAuth, err := os.ReadFile(authPath) 316 | if err != nil { 317 | if !os.IsNotExist(err) { 318 | return "", err 319 | } 320 | return "", nil 321 | } 322 | return strings.TrimSuffix(string(eventRecorderAuth), "\n"), nil 323 | } 324 | 325 | func loadPeerKey(dataDir string) (crypto.PrivKey, error) { 326 | var peerkey crypto.PrivKey 327 | keyPath := filepath.Join(dataDir, "peerkey") 328 | keyFile, err := os.ReadFile(keyPath) 329 | if err != nil { 330 | if !os.IsNotExist(err) { 331 | return nil, err 332 | } 333 | 334 | logger.Infof("Generating new peer key...") 335 | 336 | key, _, err := crypto.GenerateEd25519Key(rand.Reader) 337 | if err != nil { 338 | return nil, err 339 | } 340 | peerkey = key 341 | 342 | data, err := crypto.MarshalPrivateKey(key) 343 | if err != nil { 344 | return nil, err 345 | } 346 | 347 | if err := os.WriteFile(keyPath, data, 0600); err != nil { 348 | return nil, err 349 | } 350 | } else { 351 | key, err := crypto.UnmarshalPrivateKey(keyFile) 352 | if err != nil { 353 | return nil, err 354 | } 355 | 356 | peerkey = key 357 | } 358 | 359 | if peerkey == nil { 360 | panic("sanity check: peer key is uninitialized") 361 | } 362 | 363 | return peerkey, nil 364 | } 365 | 366 | func initHost(ctx context.Context, dataDir string, resourceManagerStats bool, listenAddrs ...multiaddr.Multiaddr) (host.Host, error) { 367 | 368 | peerkey, err := loadPeerKey(dataDir) 369 | if err != nil { 370 | return nil, err 371 | } 372 | 373 | host, err := libp2p.New(libp2p.ListenAddrs(listenAddrs...), libp2p.Identity(peerkey), libp2p.ResourceManager(network.NullResourceManager)) 374 | if err != nil { 375 | return nil, err 376 | } 377 | 378 | if resourceManagerStats { 379 | go func() { 380 | for range time.Tick(time.Second * 10) { 381 | select { 382 | case <-ctx.Done(): 383 | return 384 | default: 385 | } 386 | err := host.Network().ResourceManager().ViewSystem(func(scope network.ResourceScope) error { 387 | stat := scope.Stat() 388 | logger.Infof(statFmtString, stat.Memory, stat.NumConnsInbound, stat.NumConnsOutbound, stat.NumFD, stat.NumStreamsInbound, stat.NumStreamsOutbound) 389 | return nil 390 | }) 391 | if err != nil { 392 | logger.Errorf("unable to fetch global resource manager scope: %s", err.Error()) 393 | return 394 | } 395 | } 396 | }() 397 | } 398 | return host, nil 399 | } 400 | -------------------------------------------------------------------------------- /bitswap/provider.go: -------------------------------------------------------------------------------- 1 | package bitswap 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "time" 7 | 8 | "github.com/application-research/autoretrieve/blocks" 9 | "github.com/application-research/autoretrieve/metrics" 10 | "github.com/dustin/go-humanize" 11 | lassieretriever "github.com/filecoin-project/lassie/pkg/retriever" 12 | "github.com/filecoin-project/lassie/pkg/types" 13 | "github.com/ipfs/go-bitswap/message" 14 | bitswap_message_pb "github.com/ipfs/go-bitswap/message/pb" 15 | "github.com/ipfs/go-bitswap/network" 16 | "github.com/ipfs/go-cid" 17 | "github.com/ipfs/go-datastore" 18 | "github.com/ipfs/go-graphsync/storeutil" 19 | format "github.com/ipfs/go-ipld-format" 20 | "github.com/ipfs/go-log/v2" 21 | "github.com/ipfs/go-peertaskqueue" 22 | "github.com/ipfs/go-peertaskqueue/peertask" 23 | "github.com/ipld/go-ipld-prime" 24 | dht "github.com/libp2p/go-libp2p-kad-dht" 25 | "github.com/libp2p/go-libp2p-kad-dht/fullrt" 26 | "github.com/libp2p/go-libp2p/core/host" 27 | "github.com/libp2p/go-libp2p/core/peer" 28 | "github.com/libp2p/go-libp2p/core/routing" 29 | "go.opencensus.io/stats" 30 | "go.opencensus.io/tag" 31 | ) 32 | 33 | var logger = log.Logger("autoretrieve") 34 | 35 | // Wantlist want type redeclarations 36 | const ( 37 | wantTypeHave = bitswap_message_pb.Message_Wantlist_Have 38 | wantTypeBlock = bitswap_message_pb.Message_Wantlist_Block 39 | ) 40 | 41 | // Task queue topics 42 | type ( 43 | topicHave cid.Cid 44 | topicDontHave cid.Cid 45 | topicBlock cid.Cid 46 | ) 47 | 48 | const targetMessageSize = 1 << 10 49 | 50 | type ProviderConfig struct { 51 | CidBlacklist map[cid.Cid]bool 52 | MaxBitswapWorkers uint 53 | RoutingTableType RoutingTableType 54 | } 55 | 56 | type Provider struct { 57 | config ProviderConfig 58 | network network.BitSwapNetwork 59 | blockManager *blocks.Manager 60 | linkSystem ipld.LinkSystem 61 | retriever *lassieretriever.Retriever 62 | taskQueue *peertaskqueue.PeerTaskQueue 63 | workReady chan struct{} 64 | } 65 | 66 | func NewProvider( 67 | ctx context.Context, 68 | config ProviderConfig, 69 | host host.Host, 70 | datastore datastore.Batching, 71 | blockManager *blocks.Manager, 72 | retriever *lassieretriever.Retriever, 73 | ) (*Provider, error) { 74 | 75 | var routing routing.ContentRouting 76 | 77 | rtCfg := []dht.Option{ 78 | dht.Datastore(datastore), 79 | dht.BucketSize(20), 80 | dht.BootstrapPeers(dht.GetDefaultBootstrapPeerAddrInfos()...), 81 | } 82 | 83 | switch config.RoutingTableType { 84 | case RoutingTableTypeDisabled: 85 | case RoutingTableTypeFull: 86 | fullRT, err := fullrt.NewFullRT(host, dht.DefaultPrefix, fullrt.DHTOption(rtCfg...)) 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | routing = fullRT 92 | case RoutingTableTypeDHT: 93 | dht, err := dht.New(ctx, host, rtCfg...) 94 | if err != nil { 95 | return nil, err 96 | } 97 | 98 | routing = dht 99 | } 100 | 101 | provider := &Provider{ 102 | config: config, 103 | network: network.NewFromIpfsHost(host, routing), 104 | blockManager: blockManager, 105 | linkSystem: storeutil.LinkSystemForBlockstore(blockManager.Blockstore), 106 | retriever: retriever, 107 | taskQueue: peertaskqueue.New(), 108 | workReady: make(chan struct{}, config.MaxBitswapWorkers), 109 | } 110 | 111 | provider.network.Start(provider) 112 | 113 | for i := uint(0); i < config.MaxBitswapWorkers; i++ { 114 | go provider.runWorker() 115 | } 116 | 117 | return provider, nil 118 | } 119 | 120 | // Upon receiving a message, provider will iterate over the requested CIDs. For 121 | // each CID, it'll check if it's present in the blockstore. If it is, it will 122 | // respond with that block, and if it isn't, it'll start a retrieval and 123 | // register the block to be sent later. 124 | func (provider *Provider) ReceiveMessage(ctx context.Context, sender peer.ID, incoming message.BitSwapMessage) { 125 | for _, entry := range incoming.Wantlist() { 126 | 127 | // Immediately, if this is a cancel message, just ignore it (TODO) 128 | if entry.Cancel { 129 | continue 130 | } 131 | 132 | stats.Record(ctx, metrics.BitswapRequestCount.M(1)) 133 | 134 | // We want to skip CIDs in the blacklist, queue DONT_HAVE 135 | if provider.config.CidBlacklist[entry.Cid] { 136 | logger.Debugf("Replying DONT_HAVE for blacklisted CID: %s", entry.Cid) 137 | provider.queueDontHave(ctx, sender, entry, "blacklisted_cid") 138 | continue 139 | } 140 | 141 | switch entry.WantType { 142 | case wantTypeHave: 143 | // For WANT_HAVE, just confirm whether it's in the blockstore 144 | has, err := provider.blockManager.Has(ctx, entry.Cid) 145 | 146 | // If there was a problem, log and move on 147 | if err != nil { 148 | logger.Warnf("Failed to check blockstore for bitswap entry: %s", entry.Cid) 149 | continue 150 | } 151 | 152 | // If the block was found, queue HAVE and move on... 153 | if has { 154 | stats.Record(ctx, metrics.BlockstoreCacheHitCount.M(1)) 155 | provider.queueHave(ctx, sender, entry) 156 | continue 157 | } 158 | 159 | // ...otherwise, check retrieval candidates for it 160 | case wantTypeBlock: 161 | // For WANT_BLOCK, try to get the block 162 | size, err := provider.blockManager.GetSize(ctx, entry.Cid) 163 | 164 | // If there was a problem (aside from block not found), log and move 165 | // on 166 | if err != nil && !format.IsNotFound(err) { 167 | logger.Warnf("Failed to get block for bitswap entry: %s", entry.Cid) 168 | continue 169 | } 170 | 171 | // As long as no not found error was hit, queue the block and move 172 | // on... 173 | if !format.IsNotFound(err) { 174 | stats.Record(ctx, metrics.BlockstoreCacheHitCount.M(1)) 175 | provider.queueBlock(ctx, sender, entry, size) 176 | continue 177 | } 178 | 179 | // ...otherwise, check retrieval candidates for it 180 | } 181 | 182 | // If retriever is disabled, nothing we can do, send DONT_HAVE and move 183 | // on 184 | if provider.retriever == nil { 185 | provider.queueDontHave(ctx, sender, entry, "disabled_retriever") 186 | continue 187 | } 188 | 189 | // At this point, the blockstore did not have the requested block, so a 190 | // retrieval is attempted 191 | 192 | // Record that a retrieval is required 193 | stats.Record(ctx, metrics.BitswapRetrieverRequestCount.M(1)) 194 | 195 | switch entry.WantType { 196 | case wantTypeHave: 197 | provider.retrieveForPeer(ctx, entry, sender, false) 198 | case wantTypeBlock: 199 | provider.retrieveForPeer(ctx, entry, sender, true) 200 | } 201 | } 202 | 203 | provider.signalWork() 204 | } 205 | 206 | func (provider *Provider) ReceiveError(err error) { 207 | logger.Errorf("Error receiving bitswap message: %s", err.Error()) 208 | } 209 | 210 | func (provider *Provider) PeerConnected(peer peer.ID) {} 211 | 212 | func (provider *Provider) PeerDisconnected(peer peer.ID) {} 213 | 214 | func (provider *Provider) signalWork() { 215 | // Request work if any worker isn't running or do nothing otherwise 216 | select { 217 | case provider.workReady <- struct{}{}: 218 | default: 219 | } 220 | } 221 | 222 | func (provider *Provider) runWorker() { 223 | for { 224 | peer, tasks, pending := provider.taskQueue.PopTasks(targetMessageSize) 225 | 226 | // If there's nothing to do, wait for something to happen 227 | if len(tasks) == 0 { 228 | select { 229 | case _, ok := <-provider.workReady: 230 | // If the workReady channel closed, stop the worker 231 | if !ok { 232 | logger.Infof("Shutting down worker") 233 | return 234 | } 235 | case <-time.After(250 * time.Millisecond): 236 | } 237 | continue 238 | } 239 | 240 | msg := message.New(false) 241 | 242 | for _, task := range tasks { 243 | switch topic := task.Topic.(type) { 244 | case topicHave: 245 | have, err := provider.blockManager.Has(context.Background(), cid.Cid(topic)) 246 | if err != nil { 247 | logger.Errorf("Block load error: %s", err.Error()) 248 | msg.AddDontHave(cid.Cid(topic)) 249 | } else if !have { 250 | logger.Debugf("Had a block but lost it for want_have: %s", cid.Cid(topic)) 251 | msg.AddDontHave(cid.Cid(topic)) 252 | } else { 253 | logger.Debugf("Sending want_have to peer: %s", cid.Cid(topic)) 254 | msg.AddHave(cid.Cid(topic)) 255 | } 256 | case topicDontHave: 257 | msg.AddDontHave(cid.Cid(topic)) 258 | case topicBlock: 259 | blk, err := provider.blockManager.Get(context.Background(), cid.Cid(topic)) 260 | if err != nil { 261 | logger.Debugf("Had a block but lost it for want_block: %s (%s)", cid.Cid(topic), err.Error()) 262 | msg.AddDontHave(cid.Cid(topic)) 263 | } else { 264 | logger.Debugf("Sending want_block to peer: %s", cid.Cid(topic)) 265 | msg.AddBlock(blk) 266 | } 267 | } 268 | } 269 | msg.SetPendingBytes(int32(pending)) 270 | 271 | if err := provider.network.SendMessage(context.Background(), peer, msg); err != nil { 272 | logger.Errorf("Failed to send message %#v: %s", msg, err.Error()) 273 | } 274 | 275 | provider.taskQueue.TasksDone(peer, tasks...) 276 | } 277 | } 278 | 279 | // Adds a BLOCK task to the task queue 280 | func (provider *Provider) queueBlock(ctx context.Context, sender peer.ID, entry message.Entry, size int) { 281 | ctx, _ = tag.New(ctx, tag.Insert(metrics.BitswapTopic, "BLOCK")) 282 | 283 | provider.taskQueue.PushTasks(sender, peertask.Task{ 284 | Topic: topicBlock(entry.Cid), 285 | Priority: int(entry.Priority), 286 | Work: size, 287 | }) 288 | 289 | // Record response metric 290 | stats.Record(ctx, metrics.BitswapResponseCount.M(1)) 291 | } 292 | 293 | // Adds a DONT_HAVE task to the task queue 294 | func (provider *Provider) queueDontHave(ctx context.Context, sender peer.ID, entry message.Entry, reason string) { 295 | ctx, _ = tag.New(ctx, tag.Insert(metrics.BitswapTopic, "DONT_HAVE"), tag.Insert(metrics.BitswapDontHaveReason, reason)) 296 | 297 | provider.taskQueue.PushTasks(sender, peertask.Task{ 298 | Topic: topicDontHave(entry.Cid), 299 | Priority: int(entry.Priority), 300 | Work: message.BlockPresenceSize(entry.Cid), 301 | }) 302 | 303 | // Record response metric 304 | stats.Record(ctx, metrics.BitswapResponseCount.M(1)) 305 | } 306 | 307 | // Adds a HAVE task to the task queue 308 | func (provider *Provider) queueHave(ctx context.Context, sender peer.ID, entry message.Entry) { 309 | ctx, _ = tag.New(ctx, tag.Insert(metrics.BitswapTopic, "HAVE")) 310 | 311 | provider.taskQueue.PushTasks(sender, peertask.Task{ 312 | Topic: topicHave(entry.Cid), 313 | Priority: int(entry.Priority), 314 | Work: message.BlockPresenceSize(entry.Cid), 315 | }) 316 | 317 | // Record response metric 318 | stats.Record(ctx, metrics.BitswapResponseCount.M(1)) 319 | } 320 | 321 | func (provider *Provider) retrieveForPeer(ctx context.Context, entry message.Entry, sender peer.ID, sendBlock bool) { 322 | retrievalId, err := types.NewRetrievalID() 323 | if err != nil { 324 | logger.Errorf("Failed to create retrieval ID: %s", err.Error()) 325 | return 326 | } 327 | 328 | logger.Debugf("Starting retrieval for %s (%s)", entry.Cid, retrievalId) 329 | 330 | // Start a background blockstore fetch with a callback to send the block 331 | // to the peer once it's available. 332 | blockCtx, blockCancel := context.WithCancel(ctx) 333 | if provider.blockManager.AwaitBlock(blockCtx, entry.Cid, func(block blocks.Block, err error) { 334 | if err != nil { 335 | logger.Debugf("Failed to load block: %s", err.Error()) 336 | provider.queueDontHave(ctx, sender, entry, "failed_block_load") 337 | } else { 338 | if sendBlock { 339 | logger.Debugf("Successfully awaited block (want_block): %s", entry.Cid) 340 | provider.queueBlock(context.Background(), sender, entry, block.Size) 341 | } else { 342 | logger.Debugf("Successfully awaited block (want_have): %s", entry.Cid) 343 | provider.queueHave(context.Background(), sender, entry) 344 | } 345 | } 346 | provider.signalWork() 347 | blockCancel() 348 | }) { 349 | // If the block was already in the blockstore then we don't need to 350 | // start a retrieval. 351 | return 352 | } 353 | 354 | // Try to start a new retrieval (if it's already running then no 355 | // need to error, just continue on to await block) 356 | go func() { 357 | result, err := provider.retriever.Retrieve(ctx, types.RetrievalRequest{ 358 | LinkSystem: provider.linkSystem, 359 | RetrievalID: retrievalId, 360 | Cid: entry.Cid, 361 | }, func(types.RetrievalEvent) {}) 362 | if err != nil { 363 | if errors.Is(err, lassieretriever.ErrRetrievalAlreadyRunning) { 364 | logger.Debugf("Retrieval already running for %s, no new one will be started", entry.Cid) 365 | return // Don't send dont_have or run blockCancel(), let it async load 366 | } else if errors.Is(err, lassieretriever.ErrNoCandidates) { 367 | // Just do a debug print if there were no candidates because this happens a lot 368 | logger.Debugf("No candidates for %s (%s)", entry.Cid, retrievalId) 369 | provider.queueDontHave(ctx, sender, entry, "no_candidates") 370 | } else { 371 | // Otherwise, there was a real failure, print with more importance 372 | logger.Errorf("Retrieval for %s (%s) failed: %v", entry.Cid, retrievalId, err) 373 | provider.queueDontHave(ctx, sender, entry, "retrieval_failed") 374 | } 375 | } else { 376 | logger.Infof("Retrieval for %s (%s) completed (duration: %s, bytes: %s, blocks: %d)", 377 | entry.Cid, 378 | retrievalId, 379 | result.Duration, 380 | humanize.IBytes(result.Size), 381 | result.Blocks, 382 | ) 383 | } 384 | blockCancel() 385 | }() 386 | } 387 | -------------------------------------------------------------------------------- /bitswap/util.go: -------------------------------------------------------------------------------- 1 | package bitswap 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | type RoutingTableType int 11 | 12 | const ( 13 | RoutingTableTypeDisabled RoutingTableType = iota 14 | RoutingTableTypeDHT 15 | RoutingTableTypeFull 16 | ) 17 | 18 | func ParseRoutingTableType(routingTableType string) (RoutingTableType, error) { 19 | switch strings.ToLower(strings.TrimSpace(routingTableType)) { 20 | case RoutingTableTypeDisabled.String(): 21 | return RoutingTableTypeDisabled, nil 22 | case RoutingTableTypeDHT.String(): 23 | return RoutingTableTypeDHT, nil 24 | case RoutingTableTypeFull.String(): 25 | return RoutingTableTypeFull, nil 26 | default: 27 | return 0, fmt.Errorf("unknown routing table type %s", routingTableType) 28 | } 29 | } 30 | 31 | func (routingTableType RoutingTableType) String() string { 32 | switch routingTableType { 33 | case RoutingTableTypeDisabled: 34 | return "disabled" 35 | case RoutingTableTypeDHT: 36 | return "dht" 37 | case RoutingTableTypeFull: 38 | return "full" 39 | default: 40 | return fmt.Sprintf("unknown routing table type %d", routingTableType) 41 | } 42 | } 43 | 44 | func (routingTableType *RoutingTableType) UnmarshalYAML(value *yaml.Node) error { 45 | var str string 46 | if err := value.Decode(&str); err != nil { 47 | return err 48 | } 49 | 50 | _routingTableType, err := ParseRoutingTableType(str) 51 | if err != nil { 52 | return &yaml.TypeError{ 53 | Errors: []string{fmt.Sprintf("line %d: %v", value.Line, err.Error())}, 54 | } 55 | } 56 | 57 | *routingTableType = _routingTableType 58 | 59 | return nil 60 | } 61 | 62 | func (routingTableType RoutingTableType) MarshalYAML() (interface{}, error) { 63 | return routingTableType.String(), nil 64 | } 65 | -------------------------------------------------------------------------------- /blocks/manager.go: -------------------------------------------------------------------------------- 1 | package blocks 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "sync" 7 | "time" 8 | 9 | blocks "github.com/ipfs/go-block-format" 10 | "github.com/ipfs/go-cid" 11 | blockstore "github.com/ipfs/go-ipfs-blockstore" 12 | ipld "github.com/ipfs/go-ipld-format" 13 | logging "github.com/ipfs/go-log/v2" 14 | ) 15 | 16 | var log = logging.Logger("blockstore") 17 | 18 | var ErrWaitTimeout = errors.New("wait timeout for block") 19 | 20 | type Block struct { 21 | Cid cid.Cid 22 | Size int 23 | } 24 | 25 | // Manager is a blockstore with thread safe notification hooking for put 26 | // events. 27 | type Manager struct { 28 | blockstore.Blockstore 29 | readyBlocks chan Block 30 | waitList map[cid.Cid][]waitListEntry 31 | waitListLk sync.Mutex 32 | getAwaitTimeout time.Duration 33 | } 34 | 35 | type waitListEntry struct { 36 | ctx context.Context 37 | callback func(Block, error) 38 | registeredAt time.Time 39 | } 40 | 41 | func NewManager(inner blockstore.Blockstore, getAwaitTimeout time.Duration) *Manager { 42 | mgr := &Manager{ 43 | Blockstore: inner, 44 | readyBlocks: make(chan Block, 10), 45 | waitList: make(map[cid.Cid][]waitListEntry), 46 | getAwaitTimeout: getAwaitTimeout, 47 | } 48 | 49 | go mgr.startPollReadyBlocks() 50 | 51 | // Only do cleanups if a timeout is actually set 52 | if getAwaitTimeout != 0 { 53 | go mgr.startPollCleanup() 54 | } 55 | 56 | return mgr 57 | } 58 | 59 | // AwaitBlock will wait for a block to be added to the blockstore and then 60 | // call the callback with the block. If the block is already in the blockstore, 61 | // the callback will be called immediately. If the block is not in the blockstore 62 | // or the context is cancelled, the callback will not be called. 63 | // Returns true if the block was already in the blockstore, allowing the 64 | // callback to be called, or false otherwise. 65 | func (mgr *Manager) AwaitBlock(ctx context.Context, cid cid.Cid, callback func(Block, error)) bool { 66 | // We need to lock the blockstore here to make sure the requested block 67 | // doesn't get added while being added to the waitlist 68 | mgr.waitListLk.Lock() 69 | 70 | size, err := mgr.GetSize(ctx, cid) 71 | 72 | // If we couldn't get the block, we add it to the waitlist - the block will 73 | // be populated later during a Put or PutMany event 74 | if err != nil { 75 | if !ipld.IsNotFound(err) { 76 | mgr.waitListLk.Unlock() 77 | callback(Block{}, err) 78 | return false 79 | } 80 | 81 | mgr.waitList[cid] = append(mgr.waitList[cid], waitListEntry{ 82 | ctx: ctx, 83 | callback: callback, 84 | registeredAt: time.Now(), 85 | }) 86 | 87 | mgr.waitListLk.Unlock() 88 | return false 89 | } 90 | 91 | mgr.waitListLk.Unlock() 92 | 93 | // Otherwise, we can immediately run the callback and notify the caller of 94 | // success 95 | callback(Block{cid, size}, nil) 96 | return true 97 | } 98 | 99 | func (mgr *Manager) Put(ctx context.Context, block blocks.Block) error { 100 | 101 | // We do this first since it should catch any errors with block being nil 102 | if err := mgr.Blockstore.Put(ctx, block); err != nil { 103 | log.Debugw("err save block", "cid", block.Cid(), "error", err) 104 | return err 105 | } 106 | 107 | select { 108 | case mgr.readyBlocks <- Block{block.Cid(), len(block.RawData())}: 109 | case <-ctx.Done(): 110 | return ctx.Err() 111 | } 112 | 113 | log.Debugw("finish save block", "cid", block.Cid()) 114 | 115 | return nil 116 | } 117 | 118 | func (mgr *Manager) PutMany(ctx context.Context, blocks []blocks.Block) error { 119 | 120 | if err := mgr.Blockstore.PutMany(ctx, blocks); err != nil { 121 | return err 122 | } 123 | 124 | for _, block := range blocks { 125 | select { 126 | case mgr.readyBlocks <- Block{block.Cid(), len(block.RawData())}: 127 | case <-ctx.Done(): 128 | return ctx.Err() 129 | } 130 | } 131 | 132 | return nil 133 | } 134 | 135 | func (mgr *Manager) startPollReadyBlocks() { 136 | for block := range mgr.readyBlocks { 137 | mgr.notifyWaitCallbacks(block) 138 | } 139 | } 140 | 141 | func (mgr *Manager) notifyWaitCallbacks(block Block) { 142 | cid := block.Cid 143 | mgr.waitListLk.Lock() 144 | entries, ok := mgr.waitList[cid] 145 | if ok { 146 | delete(mgr.waitList, cid) 147 | } 148 | mgr.waitListLk.Unlock() 149 | if ok { 150 | for _, entry := range entries { 151 | entry.callback(block, nil) 152 | } 153 | } 154 | } 155 | 156 | func (mgr *Manager) startPollCleanup() { 157 | for range time.Tick(time.Second * 1) { 158 | mgr.waitListLk.Lock() 159 | for cid := range mgr.waitList { 160 | // For each element in the slice for this CID... 161 | for i := 0; i < len(mgr.waitList[cid]); i++ { 162 | // ...check whether the waiter context was cancelled or it's been in the 163 | // list too long... 164 | if mgr.waitList[cid][i].ctx.Err() != nil || time.Since(mgr.waitList[cid][i].registeredAt) > mgr.getAwaitTimeout { 165 | // ...and if so, delete this element by replacing it with 166 | // the last element of the slice and shrinking the length by 167 | // 1, and step the index back. 168 | if mgr.waitList[cid][i].ctx.Err() == nil { 169 | mgr.waitList[cid][i].callback(Block{}, ErrWaitTimeout) 170 | } 171 | mgr.waitList[cid][i] = mgr.waitList[cid][len(mgr.waitList[cid])-1] 172 | mgr.waitList[cid] = mgr.waitList[cid][:len(mgr.waitList[cid])-1] 173 | i-- 174 | } 175 | } 176 | 177 | // If the slice is empty now, remove it entirely from the waitList map 178 | if len(mgr.waitList[cid]) == 0 { 179 | delete(mgr.waitList, cid) 180 | } 181 | } 182 | mgr.waitListLk.Unlock() 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /blocks/manager_test.go: -------------------------------------------------------------------------------- 1 | package blocks 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | blocks "github.com/ipfs/go-block-format" 9 | "github.com/ipfs/go-cid" 10 | "github.com/ipfs/go-datastore" 11 | "github.com/ipfs/go-datastore/sync" 12 | blockstore "github.com/ipfs/go-ipfs-blockstore" 13 | blocksutil "github.com/ipfs/go-ipfs-blocksutil" 14 | ) 15 | 16 | func TestPutBlock(t *testing.T) { 17 | ctx := context.Background() 18 | ds := sync.MutexWrap(datastore.NewMapDatastore()) 19 | bs := blockstore.NewBlockstore(ds) 20 | manager := NewManager(bs, 0) 21 | blockGenerator := blocksutil.NewBlockGenerator() 22 | blk := blockGenerator.Next() 23 | receivedBlk := make(chan Block) 24 | errChan := make(chan error, 1) 25 | go func() { 26 | manager.AwaitBlock(ctx, blk.Cid(), func(blk Block, err error) { 27 | if err != nil { 28 | errChan <- err 29 | } else { 30 | receivedBlk <- blk 31 | } 32 | }) 33 | }() 34 | err := manager.Put(ctx, blk) 35 | if err != nil { 36 | t.Fatal("wasn't able to put to blockstore") 37 | } 38 | select { 39 | case <-ctx.Done(): 40 | t.Fatal("not enough succcesful blocks fetched") 41 | case err := <-errChan: 42 | t.Fatalf("received error getting block: %s", err) 43 | case received := <-receivedBlk: 44 | if received.Cid != blk.Cid() && received.Size != len(blk.RawData()) { 45 | t.Fatalf("received block bit with incorrect parameters") 46 | } 47 | } 48 | } 49 | 50 | func TestPutMany(t *testing.T) { 51 | ctx := context.Background() 52 | ds := sync.MutexWrap(datastore.NewMapDatastore()) 53 | bs := blockstore.NewBlockstore(ds) 54 | manager := NewManager(bs, 0) 55 | blockGenerator := blocksutil.NewBlockGenerator() 56 | var blks []blocks.Block 57 | for i := 0; i < 20; i++ { 58 | blks = append(blks, blockGenerator.Next()) 59 | } 60 | receivedBlocks := make(chan cid.Cid, 20) 61 | errChan := make(chan error, 20) 62 | go func() { 63 | for _, blk := range blks { 64 | manager.AwaitBlock(ctx, blk.Cid(), func(blk Block, err error) { 65 | if err != nil { 66 | errChan <- err 67 | } else { 68 | receivedBlocks <- blk.Cid 69 | } 70 | }) 71 | } 72 | }() 73 | err := manager.PutMany(ctx, blks) 74 | if err != nil { 75 | t.Fatal("wasn't able to put to blockstore") 76 | } 77 | for i := 0; i < 20; i++ { 78 | select { 79 | case <-ctx.Done(): 80 | t.Fatal("not enough succcesful blocks fetched") 81 | case err := <-errChan: 82 | t.Fatalf("received error getting block: %s", err) 83 | case <-receivedBlocks: 84 | } 85 | } 86 | } 87 | 88 | func TestCleanup(t *testing.T) { 89 | ctx := context.Background() 90 | ds := sync.MutexWrap(datastore.NewMapDatastore()) 91 | bs := blockstore.NewBlockstore(ds) 92 | manager := NewManager(bs, time.Second) 93 | 94 | testCount := 1000000 95 | 96 | gen := blocksutil.NewBlockGenerator() 97 | for i := 0; i < testCount; i++ { 98 | cid := gen.Next().Cid() 99 | manager.AwaitBlock(ctx, cid, func(b Block, err error) { 100 | if err != ErrWaitTimeout { 101 | t.Fatalf("This should not happen") 102 | } 103 | }) 104 | } 105 | 106 | // manager.waitListLk.Lock() 107 | // addedCount := len(manager.waitList) 108 | // manager.waitListLk.Unlock() 109 | 110 | // if addedCount != testCount { 111 | // t.Fatalf("Only %d wait callbacks got registered", addedCount) 112 | // } 113 | 114 | time.Sleep(time.Second * 2) 115 | 116 | manager.waitListLk.Lock() 117 | cleanedCount := len(manager.waitList) 118 | manager.waitListLk.Unlock() 119 | 120 | if cleanedCount != 0 { 121 | t.Fatalf("%d block wait callbacks were still present after cleanup deadline", cleanedCount) 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /blocks/randompruner.go: -------------------------------------------------------------------------------- 1 | package blocks 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | "math/rand" 9 | "os" 10 | "path" 11 | "strings" 12 | "sync" 13 | "time" 14 | 15 | "github.com/dustin/go-humanize" 16 | blocks "github.com/ipfs/go-block-format" 17 | "github.com/ipfs/go-cid" 18 | flatfs "github.com/ipfs/go-ds-flatfs" 19 | blockstore "github.com/ipfs/go-ipfs-blockstore" 20 | ipld "github.com/ipfs/go-ipld-format" 21 | ) 22 | 23 | type RandomPrunerConfig struct { 24 | // The amount of bytes at which the blockstore will run a prune operation - 25 | // cannot be zero 26 | Threshold uint64 27 | 28 | // Min bytes to remove during each prune operation - a value greater than or 29 | // equal to MaxCapacityBytes shall prune all applicable blocks; a value of 0 30 | // shall default to half of MaxCapacityBytes 31 | PruneBytes uint64 32 | 33 | // How long a block should remain pinned after it is used, before it is 34 | // considered for pruning; defaults to one day 35 | PinDuration time.Duration 36 | } 37 | 38 | // RandomPruner is a blockstore wrapper which removes blocks at random when disk 39 | // space is exhausted 40 | type RandomPruner struct { 41 | blockstore.Blockstore 42 | datastore *flatfs.Datastore 43 | threshold uint64 44 | pruneBytes uint64 45 | pinDuration time.Duration 46 | size uint64 47 | lastSizeUpdate time.Time 48 | 49 | // A list of "hot" CIDs which should not be deleted, and when they were last 50 | // used 51 | pins map[cid.Cid]time.Time 52 | pinsLk sync.Mutex 53 | 54 | pruneLk sync.Mutex 55 | } 56 | 57 | // The datastore that was used to create the blockstore is a required parameter 58 | // used for calculating remaining disk space - the Blockstore interface itself 59 | // does not provide this information 60 | func NewRandomPruner( 61 | ctx context.Context, 62 | inner blockstore.Blockstore, 63 | datastore *flatfs.Datastore, 64 | cfg RandomPrunerConfig, 65 | ) (*RandomPruner, error) { 66 | if cfg.Threshold == 0 { 67 | log.Warnf("Zero is not a valid prune threshold - do not initialize RandomPruner when it is not intended to be used") 68 | } 69 | 70 | if cfg.PruneBytes == 0 { 71 | cfg.PruneBytes = cfg.Threshold / 2 72 | } 73 | 74 | if cfg.Threshold > cfg.PruneBytes { 75 | cfg.PruneBytes = cfg.Threshold 76 | } 77 | 78 | size, err := datastore.DiskUsage(ctx) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | log.Infof("Initialized pruner's tracked size as %s", humanize.IBytes(size)) 84 | 85 | pruner := &RandomPruner{ 86 | Blockstore: inner, 87 | datastore: datastore, 88 | threshold: cfg.Threshold, 89 | pruneBytes: cfg.PruneBytes, 90 | pinDuration: cfg.PinDuration, 91 | size: size, 92 | 93 | pins: make(map[cid.Cid]time.Time), 94 | } 95 | 96 | // Poll immediately on startup and then periodically 97 | pruner.Poll(ctx) 98 | go func() { 99 | for range time.Tick(time.Second * 10) { 100 | pruner.Poll(ctx) 101 | } 102 | }() 103 | 104 | return pruner, nil 105 | } 106 | 107 | func (pruner *RandomPruner) DeleteBlock(ctx context.Context, cid cid.Cid) error { 108 | blockSize, err := pruner.Blockstore.GetSize(ctx, cid) 109 | if err != nil { 110 | return err 111 | } 112 | 113 | if err := pruner.Blockstore.DeleteBlock(ctx, cid); err != nil { 114 | return err 115 | } 116 | 117 | pruner.size -= uint64(blockSize) 118 | 119 | return nil 120 | } 121 | 122 | func (pruner *RandomPruner) Put(ctx context.Context, block blocks.Block) error { 123 | pruner.updatePin(block.Cid()) 124 | pruner.Poll(ctx) 125 | if err := pruner.Blockstore.Put(ctx, block); err != nil { 126 | return err 127 | } 128 | 129 | pruner.size += uint64(len(block.RawData())) 130 | 131 | return nil 132 | } 133 | 134 | func (pruner *RandomPruner) PutMany(ctx context.Context, blocks []blocks.Block) error { 135 | blocksSize := 0 136 | for _, block := range blocks { 137 | pruner.updatePin(block.Cid()) 138 | blocksSize += len(block.RawData()) 139 | } 140 | pruner.Poll(ctx) 141 | if err := pruner.Blockstore.PutMany(ctx, blocks); err != nil { 142 | return err 143 | } 144 | 145 | pruner.size += uint64(blocksSize) 146 | 147 | return nil 148 | } 149 | 150 | // Checks remaining blockstore capacity and prunes if maximum capacity is hit 151 | func (pruner *RandomPruner) Poll(ctx context.Context) { 152 | 153 | if time.Since(pruner.lastSizeUpdate) > time.Minute { 154 | size, err := pruner.datastore.DiskUsage(ctx) 155 | if err != nil { 156 | log.Errorf("Pruner could not get blockstore disk usage: %v", err) 157 | } else { 158 | var diff uint64 159 | if size > pruner.size { 160 | diff = size - pruner.size 161 | } else { 162 | diff = pruner.size - size 163 | } 164 | if diff > 1<<30 { 165 | log.Warnf("Large mismatch between pruner's tracked size (%v) and datastore's reported disk usage (%v)", pruner.size, size) 166 | } 167 | 168 | pruner.size = size 169 | } 170 | } 171 | 172 | if pruner.size >= pruner.threshold { 173 | if pruner.pruneLk.TryLock() { 174 | go func() { 175 | defer pruner.pruneLk.Unlock() 176 | 177 | if err := pruner.prune(context.Background(), pruner.pruneBytes); err != nil { 178 | log.Errorf("Random pruner errored during prune: %v", err) 179 | } 180 | }() 181 | } 182 | } 183 | } 184 | 185 | // This is a potentially long-running operation, start it as a goroutine! 186 | // 187 | // TODO: definitely want to add a span here 188 | func (pruner *RandomPruner) prune(ctx context.Context, bytesToPrune uint64) error { 189 | 190 | log.Infof("Starting prune operation with original datastore size of %s", humanize.IBytes(pruner.size)) 191 | start := time.Now() 192 | 193 | // Get all the keys in the blockstore 194 | allCids, err := pruner.AllKeysChan(ctx) 195 | if err != nil { 196 | return err 197 | } 198 | 199 | tmpFile, err := os.Create(path.Join(os.TempDir(), "autoretrieve-prune.txt")) 200 | if err != nil { 201 | return err 202 | } 203 | defer tmpFile.Close() 204 | 205 | // Write every non-pinned block on a separate line to the temporary file 206 | const cidPadLength = 64 207 | writer := bufio.NewWriter(tmpFile) 208 | cidCount := 0 209 | for cid := range allCids { 210 | if ctx.Err() != nil { 211 | return ctx.Err() 212 | } 213 | 214 | // Don't consider pinned blocks for deletion 215 | if pruner.isPinned(cid) { 216 | continue 217 | } 218 | 219 | paddedCidStr := fmt.Sprintf("%-*s", cidPadLength, cid.String()) 220 | if _, err := writer.WriteString(paddedCidStr + "\n"); err != nil { 221 | return fmt.Errorf("failed to write cid to tmp file: %w", err) 222 | } 223 | cidCount++ 224 | } 225 | writer.Flush() 226 | 227 | // Choose random blocks and remove them until the requested amount of bytes 228 | // have been pruned 229 | bytesPruned := uint64(0) 230 | // notFoundCount is used to avoid infinitely spinning through here if no 231 | // blocks appear to be left to remove - concretely detecting that all blocks 232 | // have been covered actually seems to be a difficult problem 233 | notFoundCount := 0 234 | 235 | defer func() { 236 | duration := time.Since(start) 237 | log.Infof("Prune operation finished after %s with new datastore size of %s (removed %v)", duration, humanize.IBytes(pruner.size), humanize.IBytes(bytesPruned)) 238 | }() 239 | 240 | // If there aren't any non-pinned CIDs to remove, just finish now 241 | if cidCount == 0 { 242 | log.Warnf("No non-pinned blocks available to remove, consider lowering the pin duration (currently %s)", pruner.pinDuration) 243 | return nil 244 | } 245 | 246 | for bytesPruned < bytesToPrune { 247 | if ctx.Err() != nil { 248 | return ctx.Err() 249 | } 250 | 251 | // Seek to the line with the CID, read it and parse 252 | cidIndex := rand.Int() % cidCount 253 | if _, err := tmpFile.Seek((cidPadLength+1)*int64(cidIndex), io.SeekStart); err != nil { 254 | return fmt.Errorf("failed to seek back to start of tmp file: %w", err) 255 | } 256 | scanner := bufio.NewScanner(tmpFile) 257 | scanner.Scan() 258 | cidStr := scanner.Text() 259 | cid, err := cid.Parse(strings.TrimSpace(cidStr)) 260 | if err != nil { 261 | log.Errorf("Failed to parse cid for removal (%s): %v", cidStr, err) 262 | continue 263 | } 264 | 265 | blockSize, err := pruner.GetSize(ctx, cid) 266 | if err != nil { 267 | // If the block was already removed, ignore it up to a few times in 268 | // a row; this strategy might not work super effectively when 269 | // pruneBytes is close to the threshold, and could possibly be 270 | // remedied by deleting CIDs from the file after reading them 271 | if ipld.IsNotFound(err) { 272 | notFoundCount++ 273 | if notFoundCount > 10 { 274 | break 275 | } 276 | continue 277 | } 278 | return err 279 | } 280 | 281 | if err := pruner.DeleteBlock(ctx, cid); err != nil { 282 | return err 283 | } 284 | 285 | bytesPruned += uint64(blockSize) 286 | notFoundCount = 0 287 | } 288 | 289 | return nil 290 | } 291 | 292 | func (pruner *RandomPruner) updatePin(pin cid.Cid) { 293 | // clone the Cid to ensure we don't hang on to memory that may come from the network stack 294 | // where the Cid was decoded as part of a larger message 295 | emptyPinClone := make([]byte, 0, len(pin.Bytes())) 296 | pinClone := append(emptyPinClone, pin.Bytes()...) 297 | pin = cid.MustParse(pinClone) 298 | pruner.pinsLk.Lock() 299 | pruner.pins[pin] = time.Now() 300 | pruner.pinsLk.Unlock() 301 | } 302 | 303 | func (pruner *RandomPruner) isPinned(cid cid.Cid) bool { 304 | pruner.pinsLk.Lock() 305 | defer pruner.pinsLk.Unlock() 306 | 307 | lastUse, ok := pruner.pins[cid] 308 | if !ok { 309 | return false 310 | } 311 | 312 | if time.Since(lastUse) < pruner.pinDuration { 313 | return true 314 | } 315 | 316 | delete(pruner.pins, cid) 317 | return false 318 | 319 | } 320 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "strings" 8 | "time" 9 | 10 | "github.com/application-research/autoretrieve/bitswap" 11 | mpg "github.com/application-research/autoretrieve/minerpeergetter" 12 | "github.com/dustin/go-humanize" 13 | "github.com/filecoin-project/go-address" 14 | lassieretriever "github.com/filecoin-project/lassie/pkg/retriever" 15 | "github.com/ipfs/go-cid" 16 | peer "github.com/libp2p/go-libp2p/core/peer" 17 | "gopkg.in/yaml.v3" 18 | ) 19 | 20 | type EndpointType int 21 | 22 | const ( 23 | EndpointTypeIndexer EndpointType = iota 24 | EndpointTypeEstuary 25 | ) 26 | 27 | func ParseEndpointType(endpointType string) (EndpointType, error) { 28 | switch strings.ToLower(strings.TrimSpace(endpointType)) { 29 | case EndpointTypeIndexer.String(): 30 | return EndpointTypeIndexer, nil 31 | case EndpointTypeEstuary.String(): 32 | return EndpointTypeEstuary, nil 33 | default: 34 | return 0, fmt.Errorf("unknown endpoint type %s", endpointType) 35 | } 36 | } 37 | 38 | func (endpointType EndpointType) String() string { 39 | switch endpointType { 40 | case EndpointTypeIndexer: 41 | return "indexer" 42 | case EndpointTypeEstuary: 43 | return "estuary" 44 | default: 45 | return fmt.Sprintf("unknown endpoint type %d", endpointType) 46 | } 47 | } 48 | 49 | func (endpointType *EndpointType) UnmarshalYAML(value *yaml.Node) error { 50 | var str string 51 | if err := value.Decode(&str); err != nil { 52 | return err 53 | } 54 | 55 | _endpointType, err := ParseEndpointType(str) 56 | if err != nil { 57 | return &yaml.TypeError{ 58 | Errors: []string{fmt.Sprintf("line %d: %v", value.Line, err.Error())}, 59 | } 60 | } 61 | 62 | *endpointType = _endpointType 63 | 64 | return nil 65 | } 66 | 67 | func (endpointType EndpointType) MarshalYAML() (interface{}, error) { 68 | return endpointType.String(), nil 69 | } 70 | 71 | type ConfigByteCount uint64 72 | 73 | func (byteCount *ConfigByteCount) UnmarshalYAML(value *yaml.Node) error { 74 | var str string 75 | if err := value.Decode(&str); err != nil { 76 | return err 77 | } 78 | 79 | _byteCount, err := humanize.ParseBytes(str) 80 | if err != nil { 81 | return &yaml.TypeError{ 82 | Errors: []string{fmt.Sprintf("line %d: %v", value.Line, err.Error())}, 83 | } 84 | } 85 | 86 | *byteCount = ConfigByteCount(_byteCount) 87 | 88 | return nil 89 | } 90 | 91 | func (byteCount ConfigByteCount) MarshalYAML() (interface{}, error) { 92 | return humanize.IBytes(uint64(byteCount)), nil 93 | } 94 | 95 | type ConfigStorageProvider struct { 96 | maybeAddr address.Address 97 | maybePeer peer.ID 98 | } 99 | 100 | func NewConfigStorageProviderAddress(addr address.Address) ConfigStorageProvider { 101 | return ConfigStorageProvider{ 102 | maybeAddr: addr, 103 | } 104 | } 105 | 106 | func NewConfigStorageProviderPeer(peer peer.ID) ConfigStorageProvider { 107 | return ConfigStorageProvider{ 108 | maybePeer: peer, 109 | } 110 | } 111 | 112 | func (provider *ConfigStorageProvider) GetPeerID(ctx context.Context, minerPeerGetter *mpg.MinerPeerGetter) (peer.ID, error) { 113 | if provider.maybeAddr == address.Undef { 114 | return provider.maybePeer, nil 115 | } 116 | 117 | peer, err := minerPeerGetter.MinerPeer(ctx, provider.maybeAddr) 118 | if err != nil { 119 | return "", fmt.Errorf("could not get peer id from address %s: %w", provider.maybeAddr, err) 120 | } 121 | 122 | return peer.ID, nil 123 | } 124 | 125 | func (provider *ConfigStorageProvider) UnmarshalYAML(value *yaml.Node) error { 126 | var str string 127 | if err := value.Decode(&str); err != nil { 128 | return err 129 | } 130 | 131 | peerID, err := peer.Decode(str) 132 | if err == nil { 133 | provider.maybePeer = peerID 134 | return nil 135 | } 136 | 137 | addr, err := address.NewFromString(str) 138 | if err == nil { 139 | provider.maybeAddr = addr 140 | return nil 141 | } 142 | 143 | return &yaml.TypeError{ 144 | Errors: []string{fmt.Sprintf("line %d: '%s' is not a valid storage provider address or peer ID", value.Line, str)}, 145 | } 146 | } 147 | 148 | func (provider ConfigStorageProvider) MarshalYAML() (interface{}, error) { 149 | if provider.maybeAddr != address.Undef { 150 | return provider.maybeAddr.String(), nil 151 | } 152 | 153 | return provider.maybePeer.String(), nil 154 | } 155 | 156 | type MinerConfig struct { 157 | RetrievalTimeout time.Duration `yaml:"retrieval-timeout"` 158 | MaxConcurrentRetrievals uint `yaml:"max-concurrent-retrievals"` 159 | } 160 | 161 | // All config values should be safe to leave uninitialized 162 | type Config struct { 163 | EstuaryURL string `yaml:"estuary-url"` 164 | HeartbeatInterval time.Duration `yaml:"heartbeat-interval"` 165 | AdvertiseToken string `yaml:"advertise-token"` 166 | LookupEndpointType EndpointType `yaml:"lookup-endpoint-type"` 167 | LookupEndpointURL string `yaml:"lookup-endpoint-url"` 168 | MaxBitswapWorkers uint `yaml:"max-bitswap-workers"` 169 | RoutingTableType bitswap.RoutingTableType `yaml:"routing-table-type"` 170 | PruneThreshold ConfigByteCount `yaml:"prune-threshold"` 171 | PinDuration time.Duration `yaml:"pin-duration"` 172 | LogResourceManager bool `yaml:"log-resource-manager"` 173 | LogRetrievals bool `yaml:"log-retrieval-stats"` 174 | DisableRetrieval bool `yaml:"disable-retrieval"` 175 | PaidRetrievals bool `yaml:"paid-retrievals"` 176 | CidBlacklist []cid.Cid `yaml:"cid-blacklist"` 177 | MinerBlacklist []ConfigStorageProvider `yaml:"miner-blacklist"` 178 | MinerWhitelist []ConfigStorageProvider `yaml:"miner-whitelist"` 179 | EventRecorderEndpointURL string `yaml:"event-recorder-endpoint-url"` 180 | InstanceId string `yaml:"instance-name"` 181 | 182 | DefaultMinerConfig MinerConfig `yaml:"default-miner-config"` 183 | MinerConfigs map[ConfigStorageProvider]MinerConfig `yaml:"miner-configs"` 184 | } 185 | 186 | // Extract relevant config points into a filecoin retriever config 187 | func (cfg *Config) ExtractFilecoinRetrieverConfig(ctx context.Context, minerPeerGetter *mpg.MinerPeerGetter) (lassieretriever.RetrieverConfig, error) { 188 | convertedMinerCfgs := make(map[peer.ID]lassieretriever.MinerConfig) 189 | for provider, minerCfg := range cfg.MinerConfigs { 190 | peer, err := provider.GetPeerID(ctx, minerPeerGetter) 191 | if err != nil { 192 | return lassieretriever.RetrieverConfig{}, err 193 | } 194 | 195 | convertedMinerCfgs[peer] = lassieretriever.MinerConfig(minerCfg) 196 | } 197 | 198 | blacklist, err := configStorageProviderListToPeerMap(ctx, minerPeerGetter, cfg.MinerBlacklist) 199 | if err != nil { 200 | return lassieretriever.RetrieverConfig{}, err 201 | } 202 | 203 | whitelist, err := configStorageProviderListToPeerMap(ctx, minerPeerGetter, cfg.MinerWhitelist) 204 | if err != nil { 205 | return lassieretriever.RetrieverConfig{}, err 206 | } 207 | 208 | return lassieretriever.RetrieverConfig{ 209 | MinerBlacklist: blacklist, 210 | MinerWhitelist: whitelist, 211 | DefaultMinerConfig: lassieretriever.MinerConfig(cfg.DefaultMinerConfig), 212 | MinerConfigs: convertedMinerCfgs, 213 | PaidRetrievals: cfg.PaidRetrievals, 214 | }, nil 215 | } 216 | 217 | // Extract relevant config points into a bitswap provider config 218 | func (cfg *Config) ExtractBitswapProviderConfig(ctx context.Context) bitswap.ProviderConfig { 219 | return bitswap.ProviderConfig{ 220 | CidBlacklist: cidListToMap(ctx, cfg.CidBlacklist), 221 | MaxBitswapWorkers: cfg.MaxBitswapWorkers, 222 | RoutingTableType: cfg.RoutingTableType, 223 | } 224 | } 225 | 226 | func LoadConfig(path string) (Config, error) { 227 | config := DefaultConfig() 228 | 229 | bytes, err := os.ReadFile(path) 230 | if err != nil { 231 | return config, err 232 | } 233 | 234 | if err := yaml.Unmarshal(bytes, &config); err != nil { 235 | return config, fmt.Errorf("failed to parse config: %w", err) 236 | } 237 | 238 | return config, nil 239 | } 240 | 241 | func DefaultConfig() Config { 242 | return Config{ 243 | InstanceId: "autoretrieve-unnamed", 244 | LookupEndpointType: EndpointTypeIndexer, 245 | LookupEndpointURL: "https://cid.contact", 246 | MaxBitswapWorkers: 1, 247 | RoutingTableType: bitswap.RoutingTableTypeDHT, 248 | PruneThreshold: 0, 249 | PinDuration: 1 * time.Hour, 250 | LogResourceManager: false, 251 | LogRetrievals: false, 252 | DisableRetrieval: false, 253 | PaidRetrievals: false, 254 | CidBlacklist: nil, 255 | MinerBlacklist: nil, 256 | MinerWhitelist: nil, 257 | 258 | DefaultMinerConfig: MinerConfig{ 259 | RetrievalTimeout: 1 * time.Minute, 260 | MaxConcurrentRetrievals: 1, 261 | }, 262 | MinerConfigs: make(map[ConfigStorageProvider]MinerConfig), 263 | 264 | HeartbeatInterval: time.Second * 15, 265 | } 266 | } 267 | 268 | func WriteConfig(cfg Config, path string) error { 269 | bytes, err := yaml.Marshal(&cfg) 270 | if err != nil { 271 | return err 272 | } 273 | 274 | return os.WriteFile(path, bytes, 0644) 275 | } 276 | 277 | func cidListToMap(ctx context.Context, cidList []cid.Cid) map[cid.Cid]bool { 278 | cidMap := make(map[cid.Cid]bool, len(cidList)) 279 | for _, cid := range cidList { 280 | cidMap[cid] = true 281 | } 282 | return cidMap 283 | } 284 | 285 | func configStorageProviderListToPeerMap(ctx context.Context, minerPeerGetter *mpg.MinerPeerGetter, minerList []ConfigStorageProvider) (map[peer.ID]bool, error) { 286 | peerMap := make(map[peer.ID]bool, len(minerList)) 287 | for _, provider := range minerList { 288 | peer, err := provider.GetPeerID(ctx, minerPeerGetter) 289 | if err != nil { 290 | return nil, err 291 | } 292 | peerMap[peer] = true 293 | } 294 | return peerMap, nil 295 | } 296 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | autoretrieve: 3 | build: . 4 | environment: 5 | - FULLNODE_API_INFO 6 | - AUTORETRIEVE_DATA_DIR 7 | - AUTORETRIEVE_ENDPOINT_URL 8 | - AUTORETRIEVE_ENDPOINT_TYPE 9 | - AUTORETRIEVE_USE_FULLRT 10 | - AUTORETRIEVE_LOG_RESOURCE_MANAGER 11 | - AUTORETRIEVE_LOG_RETRIEVALS 12 | - GOLOG_LOG_LEVEL 13 | ports: 14 | - "6746:6746" 15 | volumes: 16 | - ${AUTORETRIEVE_DATA_DIR:-~/.autoretrieve}:/root/.autoretrieve 17 | # db: 18 | # image: postgres -------------------------------------------------------------------------------- /docker-env: -------------------------------------------------------------------------------- 1 | FULLNODE_API_INFO=wss://api.chain.love. 2 | AUTORETRIEVE_ENDPOINT=https://cid.contact 3 | AUTORETRIEVE_ENDPOINT_TYPE=indexer 4 | AUTORETRIEVE_PRUNE_THRESHOLD=1000000000 -------------------------------------------------------------------------------- /docs/updating-docker-image-on-ec2.md: -------------------------------------------------------------------------------- 1 | # Updating the Docker image on EC2 2 | 3 | Autoretrieve uses a private AWS Elastic Container Registry (ECR) to store autoretrieve images. The process involves uploading to the regristry, logging into the EC2 machine, pulling the latest image, and then running it. 4 | 5 | ## Prerequisites 6 | - AWS credentials (with permission to push to ECR) 7 | - [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-welcome.html) 8 | - [Docker](https://docs.docker.com/get-docker/) 9 | 10 | ## Uploading to ECR 11 | Get an Elastic ECR login token and pipe it to the `docker login` command. 12 | 13 | **NOTE:** This will use the currently configured AWS profile. For more details about running the AWS CLI with different profiles, see the [AWS CLI documentation on using name profiles](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html#using-profiles). 14 | ``` 15 | $ aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 407967248065.dkr.ecr.us-east-2.amazonaws.com 16 | 17 | Login Succeeded 18 | ``` 19 | 20 | Build the image locally using [`docker build`](https://docs.docker.com/engine/reference/commandline/build/). 21 | 22 | **NOTE:** The image takes advantage of [Docker BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/) to speed up builds. It is recommended to use the `DOCKER_BUILDKIT=1` 23 | ``` 24 | $ DOCKER_BUILDKIT=1 docker build -t autoretrieve . 25 | ``` 26 | 27 | Tag the image using [`docker tag`](https://docs.docker.com/engine/reference/commandline/tag/). 28 | 29 | **NOTE:** To push an image to a private registry, and not the central Docker registry, we must tag it with the registry hostname. In this case, the host name is the ECR repository URI, **407967248065.dkr.ecr.us-east-2.amazonaws.com**. 30 | ``` 31 | $ docker tag autoretrieve:latest 407967248065.dkr.ecr.us-east-2.amazonaws.com/autoretrieve:latest 32 | ``` 33 | 34 | Push the image to the ECR repository using [`docker push`](https://docs.docker.com/engine/reference/commandline/push/). 35 | ``` 36 | $ docker push 407967248065.dkr.ecr.us-east-2.amazonaws.com/autoretrieve:latest 37 | ``` 38 | 39 | ## Updating the image on an EC2 instance 40 | 41 | **Note:** The machine needs to have been setup with the [Docker Credential Helper for ECR](https://github.com/awslabs/amazon-ecr-credential-helper) prior to running the following command. 42 | 43 | The following command will pull the latest autoretrieve docker image from ECR using [`docker run`](https://docs.docker.com/engine/reference/commandline/run/). 44 | ``` 45 | $ docker pull 407967248065.dkr.ecr.us-east-2.amazonaws.com/autoretrieve:latest 46 | ``` 47 | 48 | The following command will run the docker image. The environment variables passed in are configuring the autoretrieve instance, and can be substituted for any valid configuration value for the given argument. We use the `DOCKER_BUILDKIT` environment variable for speedy runs. 49 | ``` 50 | $ DOCKER_BUILDKIT=1 docker run --rm -it -v /storage/autoretrieve/data:/root/.autoretrieve -p 6746:6746 -p 8080:8080 407967248065.dkr.ecr.us-east-2.amazonaws.com/autoretrieve:latest 51 | ``` -------------------------------------------------------------------------------- /endpoint/estuary.go: -------------------------------------------------------------------------------- 1 | package endpoint 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "net/http" 9 | "net/url" 10 | "path" 11 | 12 | "github.com/application-research/autoretrieve/minerpeergetter" 13 | "github.com/filecoin-project/go-address" 14 | "github.com/filecoin-project/lassie/pkg/types" 15 | "github.com/ipfs/go-cid" 16 | ) 17 | 18 | var ( 19 | ErrInvalidEndpointURL = errors.New("invalid endpoint URL") 20 | ErrEndpointRequestFailed = errors.New("endpoint request failed") 21 | ErrEndpointBodyInvalid = errors.New("endpoint body invalid") 22 | ) 23 | 24 | type EstuaryEndpoint struct { 25 | url string 26 | mpg *minerpeergetter.MinerPeerGetter 27 | } 28 | 29 | type retrievalCandidate struct { 30 | Miner address.Address 31 | RootCid cid.Cid 32 | DealID uint 33 | } 34 | 35 | func NewEstuaryEndpoint(url string, mpg *minerpeergetter.MinerPeerGetter) *EstuaryEndpoint { 36 | return &EstuaryEndpoint{ 37 | url: url, 38 | mpg: mpg, 39 | } 40 | } 41 | 42 | func (ee *EstuaryEndpoint) FindCandidates(ctx context.Context, cid cid.Cid) ([]types.RetrievalCandidate, error) { 43 | // Create URL with CID 44 | endpointURL, err := url.Parse(ee.url) 45 | if err != nil { 46 | return nil, fmt.Errorf("%w: '%s'", ErrInvalidEndpointURL, ee.url) 47 | } 48 | endpointURL.Path = path.Join(endpointURL.Path, cid.String()) 49 | 50 | // Request candidates from endpoint 51 | resp, err := http.Get(endpointURL.String()) 52 | if err != nil { 53 | return nil, fmt.Errorf("%w: HTTP request failed: %v", ErrEndpointRequestFailed, err) 54 | } 55 | defer resp.Body.Close() 56 | if resp.StatusCode != http.StatusOK { 57 | return nil, fmt.Errorf("%w: %s sent status %v", ErrEndpointRequestFailed, endpointURL, resp.StatusCode) 58 | } 59 | 60 | // Read candidate list from response body 61 | var unfiltered []retrievalCandidate 62 | if err := json.NewDecoder(resp.Body).Decode(&unfiltered); err != nil { 63 | return nil, ErrEndpointBodyInvalid 64 | } 65 | 66 | converted := make([]types.RetrievalCandidate, 0, len(unfiltered)) 67 | for _, original := range unfiltered { 68 | minerPeer, err := ee.mpg.MinerPeer(ctx, original.Miner) 69 | if err != nil { 70 | return nil, fmt.Errorf("%w: failed to get miner peer: %v", ErrEndpointRequestFailed, err) 71 | } 72 | converted = append(converted, types.RetrievalCandidate{ 73 | MinerPeer: minerPeer, 74 | RootCid: original.RootCid, 75 | }) 76 | } 77 | return converted, nil 78 | } 79 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/application-research/autoretrieve 2 | 3 | go 1.19 4 | 5 | require ( 6 | contrib.go.opencensus.io/exporter/prometheus v0.4.2 7 | github.com/dustin/go-humanize v1.0.0 8 | github.com/filecoin-project/go-address v1.1.0 9 | github.com/filecoin-project/go-state-types v0.9.9 10 | github.com/filecoin-project/lassie v0.2.1-0.20230203015847-2b8b452e446c 11 | github.com/filecoin-project/lotus v1.18.0 12 | github.com/ipfs/go-bitswap v0.10.2 13 | github.com/ipfs/go-block-format v0.0.3 14 | github.com/ipfs/go-blockservice v0.4.0 15 | github.com/ipfs/go-cid v0.3.2 16 | github.com/ipfs/go-datastore v0.6.0 17 | github.com/ipfs/go-ds-flatfs v0.5.1 18 | github.com/ipfs/go-ds-leveldb v0.5.0 19 | github.com/ipfs/go-graphsync v0.14.0 20 | github.com/ipfs/go-ipfs-blockstore v1.2.0 21 | github.com/ipfs/go-ipfs-blocksutil v0.0.1 22 | github.com/ipfs/go-ipfs-exchange-offline v0.3.0 23 | github.com/ipfs/go-ipld-format v0.4.0 24 | github.com/ipfs/go-log/v2 v2.5.1 25 | github.com/ipfs/go-merkledag v0.8.1 26 | github.com/ipfs/go-peertaskqueue v0.8.0 27 | github.com/ipld/go-ipld-prime v0.19.0 28 | github.com/libp2p/go-libp2p v0.23.2 29 | github.com/libp2p/go-libp2p-kad-dht v0.18.0 30 | github.com/multiformats/go-multiaddr v0.7.0 31 | github.com/prometheus/client_golang v1.14.0 32 | github.com/urfave/cli/v2 v2.23.7 33 | github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc 34 | go.opencensus.io v0.24.0 35 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 36 | gopkg.in/yaml.v2 v2.4.0 37 | gopkg.in/yaml.v3 v3.0.1 38 | ) 39 | 40 | require ( 41 | github.com/BurntSushi/toml v1.2.1 // indirect 42 | github.com/DataDog/zstd v1.4.5 // indirect 43 | github.com/GeertJohan/go.incremental v1.0.0 // indirect 44 | github.com/GeertJohan/go.rice v1.0.2 // indirect 45 | github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee // indirect 46 | github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa // indirect 47 | github.com/StackExchange/wmi v1.2.1 // indirect 48 | github.com/Stebalien/go-bitfield v0.0.1 // indirect 49 | github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect 50 | github.com/akavel/rsrc v0.8.0 // indirect 51 | github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect 52 | github.com/benbjohnson/clock v1.3.0 // indirect 53 | github.com/beorn7/perks v1.0.1 // indirect 54 | github.com/bep/debounce v1.2.1 // indirect 55 | github.com/buger/goterm v1.0.3 // indirect 56 | github.com/cespare/xxhash v1.1.0 // indirect 57 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 58 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect 59 | github.com/cilium/ebpf v0.6.2 // indirect 60 | github.com/containerd/cgroups v1.0.4 // indirect 61 | github.com/coreos/go-systemd/v22 v22.4.0 // indirect 62 | github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect 63 | github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect 64 | github.com/daaku/go.zipexe v1.0.0 // indirect 65 | github.com/davecgh/go-spew v1.1.1 // indirect 66 | github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect 67 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect 68 | github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e // indirect 69 | github.com/dgraph-io/badger/v2 v2.2007.3 // indirect 70 | github.com/dgraph-io/ristretto v0.1.0 // indirect 71 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect 72 | github.com/docker/go-units v0.5.0 // indirect 73 | github.com/elastic/go-sysinfo v1.7.0 // indirect 74 | github.com/elastic/go-windows v1.0.0 // indirect 75 | github.com/elastic/gosigar v0.14.2 // indirect 76 | github.com/fatih/color v1.13.0 // indirect 77 | github.com/filecoin-project/dagstore v0.5.5 // indirect 78 | github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f // indirect 79 | github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 // indirect 80 | github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 // indirect 81 | github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 // indirect 82 | github.com/filecoin-project/go-bitfield v0.2.4 // indirect 83 | github.com/filecoin-project/go-cbor-util v0.0.1 // indirect 84 | github.com/filecoin-project/go-commp-utils v0.1.3 // indirect 85 | github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20220905160352-62059082a837 // indirect 86 | github.com/filecoin-project/go-crypto v0.0.1 // indirect 87 | github.com/filecoin-project/go-data-transfer v1.15.2 // indirect 88 | github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc2.0.20230201023650-106231c0a5a1 // indirect 89 | github.com/filecoin-project/go-ds-versioning v0.1.2 // indirect 90 | github.com/filecoin-project/go-fil-commcid v0.1.0 // indirect 91 | github.com/filecoin-project/go-fil-markets v1.25.3-0.20230107010325-143abaddd0f3 // indirect 92 | github.com/filecoin-project/go-hamt-ipld v0.1.5 // indirect 93 | github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 // indirect 94 | github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 // indirect 95 | github.com/filecoin-project/go-jsonrpc v0.1.8 // indirect 96 | github.com/filecoin-project/go-legs v0.4.9 // indirect 97 | github.com/filecoin-project/go-padreader v0.0.1 // indirect 98 | github.com/filecoin-project/go-paramfetch v0.0.4 // indirect 99 | github.com/filecoin-project/go-statemachine v1.0.3 // indirect 100 | github.com/filecoin-project/go-statestore v0.2.0 // indirect 101 | github.com/filecoin-project/index-provider v0.9.2 // indirect 102 | github.com/filecoin-project/pubsub v1.0.0 // indirect 103 | github.com/filecoin-project/specs-actors v0.9.15 // indirect 104 | github.com/filecoin-project/specs-actors/v2 v2.3.6 // indirect 105 | github.com/filecoin-project/specs-actors/v3 v3.1.2 // indirect 106 | github.com/filecoin-project/specs-actors/v4 v4.0.2 // indirect 107 | github.com/filecoin-project/specs-actors/v5 v5.0.6 // indirect 108 | github.com/filecoin-project/specs-actors/v6 v6.0.2 // indirect 109 | github.com/filecoin-project/specs-actors/v7 v7.0.1 // indirect 110 | github.com/filecoin-project/specs-actors/v8 v8.0.1 // indirect 111 | github.com/flynn/noise v1.0.0 // indirect 112 | github.com/francoispqt/gojay v1.2.13 // indirect 113 | github.com/fsnotify/fsnotify v1.5.4 // indirect 114 | github.com/gbrlsnchs/jwt/v3 v3.0.1 // indirect 115 | github.com/gdamore/encoding v1.0.0 // indirect 116 | github.com/gdamore/tcell/v2 v2.2.0 // indirect 117 | github.com/go-kit/log v0.2.1 // indirect 118 | github.com/go-logfmt/logfmt v0.5.1 // indirect 119 | github.com/go-logr/logr v1.2.3 // indirect 120 | github.com/go-logr/stdr v1.2.2 // indirect 121 | github.com/go-ole/go-ole v1.2.5 // indirect 122 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect 123 | github.com/godbus/dbus/v5 v5.1.0 // indirect 124 | github.com/gogo/protobuf v1.3.2 // indirect 125 | github.com/golang/glog v1.0.0 // indirect 126 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 127 | github.com/golang/mock v1.6.0 // indirect 128 | github.com/golang/protobuf v1.5.2 // indirect 129 | github.com/golang/snappy v0.0.4 // indirect 130 | github.com/google/gopacket v1.1.19 // indirect 131 | github.com/google/uuid v1.3.0 // indirect 132 | github.com/gorilla/mux v1.8.0 // indirect 133 | github.com/gorilla/websocket v1.5.0 // indirect 134 | github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 // indirect 135 | github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1 // indirect 136 | github.com/hannahhoward/go-pubsub v1.0.0 // indirect 137 | github.com/hashicorp/errwrap v1.1.0 // indirect 138 | github.com/hashicorp/go-multierror v1.1.1 // indirect 139 | github.com/hashicorp/golang-lru v0.5.4 // indirect 140 | github.com/huin/goupnp v1.0.3 // indirect 141 | github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94 // indirect 142 | github.com/ipfs/bbloom v0.0.4 // indirect 143 | github.com/ipfs/go-cidutil v0.1.0 // indirect 144 | github.com/ipfs/go-ds-badger2 v0.1.2 // indirect 145 | github.com/ipfs/go-ds-measure v0.2.0 // indirect 146 | github.com/ipfs/go-fs-lock v0.0.7 // indirect 147 | github.com/ipfs/go-ipfs-cmds v0.7.0 // indirect 148 | github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect 149 | github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect 150 | github.com/ipfs/go-ipfs-files v0.1.1 // indirect 151 | github.com/ipfs/go-ipfs-http-client v0.4.0 // indirect 152 | github.com/ipfs/go-ipfs-pq v0.0.2 // indirect 153 | github.com/ipfs/go-ipfs-util v0.0.2 // indirect 154 | github.com/ipfs/go-ipld-cbor v0.0.6 // indirect 155 | github.com/ipfs/go-ipld-legacy v0.1.1 // indirect 156 | github.com/ipfs/go-ipns v0.3.0 // indirect 157 | github.com/ipfs/go-log v1.0.5 // indirect 158 | github.com/ipfs/go-metrics-interface v0.0.1 // indirect 159 | github.com/ipfs/go-path v0.3.0 // indirect 160 | github.com/ipfs/go-unixfs v0.4.0 // indirect 161 | github.com/ipfs/go-unixfsnode v1.4.0 // indirect 162 | github.com/ipfs/go-verifcid v0.0.2 // indirect 163 | github.com/ipfs/interface-go-ipfs-core v0.7.0 // indirect 164 | github.com/ipld/go-car v0.5.0 // indirect 165 | github.com/ipld/go-car/v2 v2.5.1 // indirect 166 | github.com/ipld/go-codec-dagpb v1.5.0 // indirect 167 | github.com/ipld/go-ipld-selector-text-lite v0.0.1 // indirect 168 | github.com/ipni/storetheindex v0.5.4 // indirect 169 | github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect 170 | github.com/jackpal/go-nat-pmp v1.0.2 // indirect 171 | github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect 172 | github.com/jbenet/goprocess v0.1.4 // indirect 173 | github.com/jessevdk/go-flags v1.4.0 // indirect 174 | github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect 175 | github.com/jpillora/backoff v1.0.0 // indirect 176 | github.com/kelseyhightower/envconfig v1.4.0 // indirect 177 | github.com/klauspost/compress v1.15.10 // indirect 178 | github.com/klauspost/cpuid/v2 v2.1.1 // indirect 179 | github.com/koron/go-ssdp v0.0.3 // indirect 180 | github.com/libp2p/go-buffer-pool v0.1.0 // indirect 181 | github.com/libp2p/go-cidranger v1.1.0 // indirect 182 | github.com/libp2p/go-flow-metrics v0.1.0 // indirect 183 | github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect 184 | github.com/libp2p/go-libp2p-core v0.20.1 // indirect 185 | github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect 186 | github.com/libp2p/go-libp2p-pubsub v0.8.1 // indirect 187 | github.com/libp2p/go-libp2p-record v0.2.0 // indirect 188 | github.com/libp2p/go-libp2p-xor v0.1.0 // indirect 189 | github.com/libp2p/go-msgio v0.2.0 // indirect 190 | github.com/libp2p/go-nat v0.1.0 // indirect 191 | github.com/libp2p/go-netroute v0.2.0 // indirect 192 | github.com/libp2p/go-openssl v0.1.0 // indirect 193 | github.com/libp2p/go-reuseport v0.2.0 // indirect 194 | github.com/libp2p/go-yamux/v4 v4.0.0 // indirect 195 | github.com/lucas-clemente/quic-go v0.29.1 // indirect 196 | github.com/lucasb-eyer/go-colorful v1.0.3 // indirect 197 | github.com/magefile/mage v1.9.0 // indirect 198 | github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect 199 | github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect 200 | github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect 201 | github.com/mattn/go-colorable v0.1.9 // indirect 202 | github.com/mattn/go-isatty v0.0.16 // indirect 203 | github.com/mattn/go-pointer v0.0.1 // indirect 204 | github.com/mattn/go-runewidth v0.0.10 // indirect 205 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 206 | github.com/miekg/dns v1.1.50 // indirect 207 | github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect 208 | github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect 209 | github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect 210 | github.com/minio/sha256-simd v1.0.0 // indirect 211 | github.com/mitchellh/go-homedir v1.1.0 // indirect 212 | github.com/mr-tron/base58 v1.2.0 // indirect 213 | github.com/multiformats/go-base32 v0.1.0 // indirect 214 | github.com/multiformats/go-base36 v0.1.0 // indirect 215 | github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect 216 | github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect 217 | github.com/multiformats/go-multibase v0.1.1 // indirect 218 | github.com/multiformats/go-multicodec v0.6.0 // indirect 219 | github.com/multiformats/go-multihash v0.2.1 // indirect 220 | github.com/multiformats/go-multistream v0.3.3 // indirect 221 | github.com/multiformats/go-varint v0.0.6 // indirect 222 | github.com/nkovacs/streamquote v1.0.0 // indirect 223 | github.com/nxadm/tail v1.4.8 // indirect 224 | github.com/onsi/ginkgo v1.16.5 // indirect 225 | github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect 226 | github.com/opentracing/opentracing-go v1.2.0 // indirect 227 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect 228 | github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect 229 | github.com/pkg/errors v0.9.1 // indirect 230 | github.com/pmezard/go-difflib v1.0.0 // indirect 231 | github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect 232 | github.com/prometheus/client_model v0.3.0 // indirect 233 | github.com/prometheus/common v0.37.0 // indirect 234 | github.com/prometheus/procfs v0.8.0 // indirect 235 | github.com/prometheus/statsd_exporter v0.22.7 // indirect 236 | github.com/raulk/clock v1.1.0 // indirect 237 | github.com/raulk/go-watchdog v1.3.0 // indirect 238 | github.com/rivo/uniseg v0.1.0 // indirect 239 | github.com/rs/cors v1.7.0 // indirect 240 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 241 | github.com/rvagg/go-prioritywaitqueue v1.0.3 // indirect 242 | github.com/shirou/gopsutil v2.18.12+incompatible // indirect 243 | github.com/sirupsen/logrus v1.8.1 // indirect 244 | github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect 245 | github.com/spaolacci/murmur3 v1.1.0 // indirect 246 | github.com/stretchr/testify v1.8.1 // indirect 247 | github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect 248 | github.com/valyala/bytebufferpool v1.0.0 // indirect 249 | github.com/valyala/fasttemplate v1.0.1 // indirect 250 | github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba // indirect 251 | github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect 252 | github.com/whyrusleeping/cbor-gen v0.0.0-20220514204315-f29c37e9c44c // indirect 253 | github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect 254 | github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4 // indirect 255 | github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect 256 | github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect 257 | github.com/zondax/hid v0.9.0 // indirect 258 | github.com/zondax/ledger-go v0.12.1 // indirect 259 | go.opentelemetry.io/otel v1.10.0 // indirect 260 | go.opentelemetry.io/otel/trace v1.10.0 // indirect 261 | go.uber.org/atomic v1.10.0 // indirect 262 | go.uber.org/dig v1.12.0 // indirect 263 | go.uber.org/fx v1.15.0 // indirect 264 | go.uber.org/multierr v1.8.0 // indirect 265 | go.uber.org/zap v1.23.0 // indirect 266 | go4.org v0.0.0-20200411211856-f5505b9728dd // indirect 267 | golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect 268 | golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect 269 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect 270 | golang.org/x/net v0.0.0-20220920183852-bf014ff85ad5 // indirect 271 | golang.org/x/sync v0.0.0-20220907140024-f12130a52804 // indirect 272 | golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect 273 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect 274 | golang.org/x/text v0.3.7 // indirect 275 | golang.org/x/tools v0.1.12 // indirect 276 | google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 // indirect 277 | google.golang.org/grpc v1.46.2 // indirect 278 | google.golang.org/protobuf v1.28.1 // indirect 279 | gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect 280 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 281 | howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect 282 | lukechampine.com/blake3 v1.1.7 // indirect 283 | ) 284 | 285 | replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi 286 | -------------------------------------------------------------------------------- /keystore/keystore.go: -------------------------------------------------------------------------------- 1 | package keystore 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/filecoin-project/lotus/chain/types" 10 | "github.com/whyrusleeping/base32" 11 | "golang.org/x/xerrors" 12 | ) 13 | 14 | type DiskKeyStore struct { 15 | path string 16 | } 17 | 18 | func OpenOrInitKeystore(p string) (*DiskKeyStore, error) { 19 | if _, err := os.Stat(p); err == nil { 20 | return &DiskKeyStore{p}, nil 21 | } else if !os.IsNotExist(err) { 22 | return nil, err 23 | } 24 | 25 | if err := os.Mkdir(p, 0700); err != nil { 26 | return nil, err 27 | } 28 | 29 | return &DiskKeyStore{p}, nil 30 | } 31 | 32 | var kstrPermissionMsg = "permissions of key: '%s' are too relaxed, " + 33 | "required: 0600, got: %#o" 34 | 35 | // List lists all the keys stored in the KeyStore 36 | func (fsr *DiskKeyStore) List() ([]string, error) { 37 | 38 | dir, err := os.Open(fsr.path) 39 | if err != nil { 40 | return nil, xerrors.Errorf("opening dir to list keystore: %w", err) 41 | } 42 | defer dir.Close() //nolint:errcheck 43 | files, err := dir.Readdir(-1) 44 | if err != nil { 45 | return nil, xerrors.Errorf("reading keystore dir: %w", err) 46 | } 47 | keys := make([]string, 0, len(files)) 48 | for _, f := range files { 49 | if f.Mode()&0077 != 0 { 50 | return nil, xerrors.Errorf(kstrPermissionMsg, f.Name(), f.Mode()) 51 | } 52 | name, err := base32.RawStdEncoding.DecodeString(f.Name()) 53 | if err != nil { 54 | return nil, xerrors.Errorf("decoding key: '%s': %w", f.Name(), err) 55 | } 56 | keys = append(keys, string(name)) 57 | } 58 | return keys, nil 59 | } 60 | 61 | // Get gets a key out of keystore and returns types.KeyInfo coresponding to named key 62 | func (fsr *DiskKeyStore) Get(name string) (types.KeyInfo, error) { 63 | 64 | encName := base32.RawStdEncoding.EncodeToString([]byte(name)) 65 | keyPath := filepath.Join(fsr.path, encName) 66 | 67 | fstat, err := os.Stat(keyPath) 68 | if os.IsNotExist(err) { 69 | return types.KeyInfo{}, xerrors.Errorf("opening key '%s': %w", name, types.ErrKeyInfoNotFound) 70 | } else if err != nil { 71 | return types.KeyInfo{}, xerrors.Errorf("opening key '%s': %w", name, err) 72 | } 73 | 74 | if fstat.Mode()&0077 != 0 { 75 | return types.KeyInfo{}, xerrors.Errorf(kstrPermissionMsg, name, fstat.Mode()) 76 | } 77 | 78 | file, err := os.Open(keyPath) 79 | if err != nil { 80 | return types.KeyInfo{}, xerrors.Errorf("opening key '%s': %w", name, err) 81 | } 82 | defer file.Close() //nolint: errcheck // read only op 83 | 84 | data, err := io.ReadAll(file) 85 | if err != nil { 86 | return types.KeyInfo{}, xerrors.Errorf("reading key '%s': %w", name, err) 87 | } 88 | 89 | var res types.KeyInfo 90 | err = json.Unmarshal(data, &res) 91 | if err != nil { 92 | return types.KeyInfo{}, xerrors.Errorf("decoding key '%s': %w", name, err) 93 | } 94 | 95 | return res, nil 96 | } 97 | 98 | // Put saves key info under given name 99 | func (fsr *DiskKeyStore) Put(name string, info types.KeyInfo) error { 100 | 101 | encName := base32.RawStdEncoding.EncodeToString([]byte(name)) 102 | keyPath := filepath.Join(fsr.path, encName) 103 | 104 | _, err := os.Stat(keyPath) 105 | if err == nil { 106 | return xerrors.Errorf("checking key before put '%s': %w", name, types.ErrKeyExists) 107 | } else if !os.IsNotExist(err) { 108 | return xerrors.Errorf("checking key before put '%s': %w", name, err) 109 | } 110 | 111 | keyData, err := json.Marshal(info) 112 | if err != nil { 113 | return xerrors.Errorf("encoding key '%s': %w", name, err) 114 | } 115 | 116 | err = os.WriteFile(keyPath, keyData, 0600) 117 | if err != nil { 118 | return xerrors.Errorf("writing key '%s': %w", name, err) 119 | } 120 | return nil 121 | } 122 | 123 | func (fsr *DiskKeyStore) Delete(name string) error { 124 | 125 | encName := base32.RawStdEncoding.EncodeToString([]byte(name)) 126 | keyPath := filepath.Join(fsr.path, encName) 127 | 128 | _, err := os.Stat(keyPath) 129 | if os.IsNotExist(err) { 130 | return xerrors.Errorf("checking key before delete '%s': %w", name, types.ErrKeyInfoNotFound) 131 | } else if err != nil { 132 | return xerrors.Errorf("checking key before delete '%s': %w", name, err) 133 | } 134 | 135 | err = os.Remove(keyPath) 136 | if err != nil { 137 | return xerrors.Errorf("deleting key '%s': %w", name, err) 138 | } 139 | return nil 140 | } 141 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "mime/multipart" 10 | "net/http" 11 | _ "net/http/pprof" 12 | "os" 13 | "os/signal" 14 | "path" 15 | "path/filepath" 16 | "runtime" 17 | "time" 18 | 19 | "github.com/application-research/autoretrieve/bitswap" 20 | "github.com/application-research/autoretrieve/blocks" 21 | "github.com/application-research/autoretrieve/metrics" 22 | "github.com/dustin/go-humanize" 23 | "github.com/ipfs/go-blockservice" 24 | "github.com/ipfs/go-cid" 25 | flatfs "github.com/ipfs/go-ds-flatfs" 26 | blockstore "github.com/ipfs/go-ipfs-blockstore" 27 | offline "github.com/ipfs/go-ipfs-exchange-offline" 28 | format "github.com/ipfs/go-ipld-format" 29 | "github.com/ipfs/go-log/v2" 30 | "github.com/ipfs/go-merkledag" 31 | crypto "github.com/libp2p/go-libp2p/core/crypto" 32 | "github.com/libp2p/go-libp2p/core/peer" 33 | "github.com/urfave/cli/v2" 34 | "gopkg.in/yaml.v2" 35 | ) 36 | 37 | var logger = log.Logger("autoretrieve") 38 | 39 | // Relative to data dir 40 | const ( 41 | datastoreSubdir = "datastore" 42 | walletSubdir = "wallet" 43 | blockstoreSubdir = "blockstore" 44 | configPath = "config.yaml" 45 | ) 46 | 47 | func init() { 48 | // Needed by Lotus' GetGatewayAPI() call 49 | if os.Getenv("FULLNODE_API_INFO") == "" { 50 | os.Setenv("FULLNODE_API_INFO", "wss://api.chain.love") 51 | } 52 | } 53 | 54 | func main() { 55 | log.SetLogLevel("autoretrieve", "DEBUG") 56 | 57 | app := cli.NewApp() 58 | 59 | app.Flags = []cli.Flag{ 60 | &cli.StringFlag{ 61 | Name: "data-dir", 62 | EnvVars: []string{"AUTORETRIEVE_DATA_DIR"}, 63 | }, 64 | &cli.StringFlag{ 65 | Name: "lookup-endpoint-url", 66 | Usage: "Indexer or Estuary endpoint to get retrieval candidates from", 67 | EnvVars: []string{"AUTORETRIEVE_LOOKUP_ENDPOINT_URL"}, 68 | }, 69 | &cli.StringFlag{ 70 | Name: "lookup-endpoint-type", 71 | Usage: "Type of endpoint for finding data (valid values are \"estuary\" and \"indexer\")", 72 | EnvVars: []string{"AUTORETRIEVE_LOOKUP_ENDPOINT_TYPE"}, 73 | }, 74 | &cli.BoolFlag{ 75 | Name: "disable-retrieval", 76 | Usage: "Whether to disable the retriever module, for testing provider only", 77 | EnvVars: []string{"AUTORETRIEVE_DISABLE_RETRIEVAL"}, 78 | }, 79 | &cli.StringFlag{ 80 | Name: "routing-table-type", 81 | Usage: "[dht|fullrt|disabled]", 82 | EnvVars: []string{"AUTORETRIEVE_ROUTING_TABLE_TYPE"}, 83 | }, 84 | &cli.BoolFlag{ 85 | Name: "log-resource-manager", 86 | Usage: "Whether to present output about the current state of the libp2p resource manager", 87 | EnvVars: []string{"AUTORETRIEVE_LOG_RESOURCE_MANAGER"}, 88 | }, 89 | } 90 | 91 | app.Action = cmd 92 | 93 | app.Commands = []*cli.Command{ 94 | { 95 | Name: "gen-config", 96 | Action: cmdGenConfig, 97 | Usage: "Generate a new config with default values", 98 | }, 99 | { 100 | Name: "print-config", 101 | Action: cmdPrintConfig, 102 | Usage: "Print detected config values as autoretrieve sees them", 103 | }, 104 | { 105 | Name: "check-cid", 106 | Action: cmdTestBlockstore, 107 | Usage: "Takes a CID argument and tries walking the DAG using the local blockstore", 108 | }, 109 | { 110 | Name: "register-estuary", 111 | Action: cmdRegisterEstuary, 112 | Usage: "Automatically registers this instance with Estuary for content advertisement and updates the config as necessary", 113 | ArgsUsage: " ", 114 | }, 115 | } 116 | 117 | ctx := contextWithInterruptCancel() 118 | if err := app.RunContext(ctx, os.Args); err != nil { 119 | fmt.Printf("%v\n", err) 120 | } 121 | } 122 | 123 | // Creates a context that will get cancelled when the user presses Ctrl+C or 124 | // otherwise triggers an interrupt signal. 125 | func contextWithInterruptCancel() context.Context { 126 | ctx, cancel := context.WithCancel(context.Background()) 127 | go func() { 128 | ch := make(chan os.Signal, 1) 129 | signal.Notify(ch, os.Interrupt) 130 | 131 | <-ch 132 | 133 | signal.Ignore(os.Interrupt) 134 | fmt.Printf("Interrupt detected, gracefully exiting... (interrupt again to force termination)\n") 135 | cancel() 136 | }() 137 | 138 | return ctx 139 | } 140 | 141 | // Main command entry point. 142 | func cmd(ctx *cli.Context) error { 143 | 144 | cfg, err := getFullConfig(ctx) 145 | if err != nil { 146 | return err 147 | } 148 | 149 | go func() { 150 | http.Handle("/metrics", metrics.PrometheusHandler()) 151 | http.HandleFunc("/debug/stacktrace", func(w http.ResponseWriter, r *http.Request) { 152 | buf := make([]byte, 64<<20) 153 | for i := 0; ; i++ { 154 | n := runtime.Stack(buf, true) 155 | if n < len(buf) { 156 | buf = buf[:n] 157 | break 158 | } 159 | if len(buf) >= 1<<30 { 160 | // Filled 1 GB - stop there. 161 | break 162 | } 163 | buf = make([]byte, 2*len(buf)) 164 | } 165 | _, err := w.Write(buf) 166 | if err != nil { 167 | logger.Error(err) 168 | } 169 | }) 170 | if err := http.ListenAndServe("0.0.0.0:8080", nil); err != nil { 171 | logger.Errorf("Could not start prometheus endpoint server: %s", err) 172 | } 173 | }() 174 | 175 | autoretrieve, err := New(ctx, dataDirPath(ctx), cfg) 176 | if err != nil { 177 | logger.Errorf("Error starting autoretrieve: %s", err.Error()) 178 | return err 179 | } 180 | 181 | <-ctx.Context.Done() 182 | 183 | autoretrieve.Close() 184 | 185 | return nil 186 | } 187 | 188 | func cmdTestBlockstore(ctx *cli.Context) error { 189 | // Initialize blockstore manager 190 | parseShardFunc, err := flatfs.ParseShardFunc("/repo/flatfs/shard/v1/next-to-last/3") 191 | if err != nil { 192 | return err 193 | } 194 | 195 | blockstoreDatastore, err := flatfs.CreateOrOpen(filepath.Join(dataDirPath(ctx), blockstoreSubdir), parseShardFunc, false) 196 | if err != nil { 197 | return err 198 | } 199 | 200 | blockstore := blockstore.NewBlockstoreNoPrefix(blockstoreDatastore) 201 | 202 | blockManager := blocks.NewManager(blockstore, 0) 203 | if err != nil { 204 | return err 205 | } 206 | 207 | bs := blockservice.New(blockManager, offline.Exchange(blockManager)) 208 | ds := merkledag.NewDAGService(bs) 209 | 210 | cset := cid.NewSet() 211 | c, err := cid.Parse(ctx.Args().First()) 212 | if err != nil { 213 | return err 214 | } 215 | 216 | var size int 217 | var count int 218 | complete := true 219 | 220 | if err := merkledag.Walk(ctx.Context, func(ctx context.Context, c cid.Cid) ([]*format.Link, error) { 221 | node, err := ds.Get(ctx, c) 222 | if err != nil { 223 | return nil, err 224 | } 225 | 226 | if c.Type() == cid.Raw { 227 | return nil, nil 228 | } 229 | 230 | return node.Links(), nil 231 | }, c, func(c cid.Cid) bool { 232 | blockSize, err := blockManager.GetSize(ctx.Context, c) 233 | if err != nil { 234 | fmt.Printf("Error getting block size: %v\n", err) 235 | complete = false 236 | return false 237 | } 238 | 239 | size += blockSize 240 | count++ 241 | 242 | return cset.Visit(c) 243 | }); err != nil { 244 | fmt.Printf("Failed: %v\n", err) 245 | } 246 | 247 | fmt.Printf("Got size %s from %d blocks\n", humanize.IBytes(uint64(size)), count) 248 | 249 | if complete { 250 | fmt.Printf("Tree is complete\n") 251 | } else { 252 | fmt.Printf("Tree is incomplete\n") 253 | } 254 | 255 | return nil 256 | } 257 | 258 | func cmdGenConfig(ctx *cli.Context) error { 259 | cfg := DefaultConfig() 260 | if err := applyConfigCLIOverrides(ctx, &cfg); err != nil { 261 | return err 262 | } 263 | 264 | cfgPath := fullConfigPath(ctx) 265 | fmt.Printf("Writing config to '%s'\n", cfgPath) 266 | return WriteConfig(cfg, cfgPath) 267 | } 268 | 269 | func cmdPrintConfig(ctx *cli.Context) error { 270 | cfg, err := getFullConfig(ctx) 271 | if err != nil { 272 | return err 273 | } 274 | 275 | bytes, err := yaml.Marshal(cfg) 276 | if err != nil { 277 | return err 278 | } 279 | 280 | fmt.Printf("%s\n", string(bytes)) 281 | 282 | return nil 283 | } 284 | 285 | func cmdRegisterEstuary(ctx *cli.Context) error { 286 | endpointURL := ctx.Args().Get(0) 287 | token := ctx.Args().Get(1) 288 | 289 | if endpointURL == "" { 290 | return fmt.Errorf("an Estuary endpoint URL is required (first argument)") 291 | } 292 | 293 | if token == "" { 294 | return fmt.Errorf("an Estuary admin token is required (second argument)") 295 | } 296 | 297 | // Get public IP address 298 | 299 | publicIPRes, err := http.Get("http://ip-api.com/json") 300 | if err != nil { 301 | return fmt.Errorf("could not get public ip: %v", err) 302 | } 303 | defer publicIPRes.Body.Close() 304 | 305 | var ipInfo struct { 306 | Query string 307 | } 308 | 309 | if err := json.NewDecoder(publicIPRes.Body).Decode(&ipInfo); err != nil { 310 | return fmt.Errorf("could not get public ip: %v", err) 311 | } 312 | 313 | fmt.Printf("Using IP: %s\n", ipInfo.Query) 314 | 315 | // Get peer key 316 | 317 | peerkey, err := loadPeerKey(dataDirPath(ctx)) 318 | if err != nil { 319 | return fmt.Errorf("couldn't get peer key: %v", err) 320 | } 321 | 322 | // peerkeyPubProto, err := crypto.PublicKeyToProto(peerkey.GetPublic()) 323 | // if err != nil { 324 | // return fmt.Errorf("couldn't get peer public key: %v", err) 325 | // } 326 | 327 | peerkeyPubBytes, err := crypto.MarshalPublicKey(peerkey.GetPublic()) 328 | if err != nil { 329 | return fmt.Errorf("couldn't marshal peer public key: %v", err) 330 | } 331 | peerkeyPub := crypto.ConfigEncodeKey(peerkeyPubBytes) 332 | 333 | // fmt.Printf("Using peer ID: %s\n", peer) 334 | fmt.Printf("Using public key: %s\n", peerkeyPub) 335 | 336 | // Do registration 337 | body := &bytes.Buffer{} 338 | writer := multipart.NewWriter(body) 339 | writer.WriteField("addresses", fmt.Sprintf("/ip4/%s/tcp/6746", ipInfo.Query)) 340 | writer.WriteField("pubKey", peerkeyPub) 341 | writer.Close() 342 | 343 | req, err := http.NewRequest( 344 | "POST", 345 | endpointURL+"/admin/autoretrieve/init", 346 | body, 347 | ) 348 | 349 | if err != nil { 350 | return fmt.Errorf("building registration request failed: %v", err) 351 | } 352 | 353 | req.Header.Set("Authorization", "Bearer "+token) 354 | req.Header.Set("Content-Type", writer.FormDataContentType()) 355 | 356 | res, err := http.DefaultClient.Do(req) 357 | if err != nil { 358 | return fmt.Errorf("registration request failed: %v", err) 359 | } 360 | if res.StatusCode != http.StatusOK { 361 | return fmt.Errorf("registration request failed: expected status code 200 but got %d", res.StatusCode) 362 | } 363 | defer res.Body.Close() 364 | 365 | var output struct { 366 | Handle string 367 | Token string 368 | LastConnection string 369 | AdvertiseInterval string 370 | AddrInfo peer.AddrInfo 371 | Error interface{} 372 | } 373 | // outputStr, err := ioutil.ReadAll(res.Body) 374 | if err := json.NewDecoder(res.Body).Decode(&output); err != nil { 375 | return fmt.Errorf("couldn't decode response: %v", err) 376 | } 377 | 378 | if output.Error != nil && output.Error != "" { 379 | return fmt.Errorf("registration failed: %v", output.Error) 380 | } 381 | 382 | cfg, err := LoadConfig(fullConfigPath(ctx)) 383 | if err != nil { 384 | return fmt.Errorf("could not load config: %s", err) 385 | } 386 | cfg.EstuaryURL = endpointURL 387 | cfg.AdvertiseToken = output.Token 388 | 389 | advertiseInterval, err := time.ParseDuration(output.AdvertiseInterval) 390 | if err != nil { 391 | return fmt.Errorf("could not parse advertisement interval: %s", err) 392 | } 393 | cfg.HeartbeatInterval = advertiseInterval 394 | 395 | if err := WriteConfig(cfg, fullConfigPath(ctx)); err != nil { 396 | return fmt.Errorf("failed to write config: %v", err) 397 | } 398 | 399 | logger.Infof("Successfully registered") 400 | 401 | return nil 402 | } 403 | 404 | func getFullConfig(ctx *cli.Context) (Config, error) { 405 | cfgPath := fullConfigPath(ctx) 406 | fmt.Printf("Reading config from '%s'\n", cfgPath) 407 | cfg, err := LoadConfig(cfgPath) 408 | if err != nil { 409 | if errors.Is(err, os.ErrNotExist) { 410 | fmt.Printf("NOTE: no config file found, using defaults; run autoretrieve or use the gen-config subcommand to generate one\n-----\n") 411 | cfg = DefaultConfig() 412 | } else { 413 | return Config{}, err 414 | } 415 | } 416 | 417 | if err := applyConfigCLIOverrides(ctx, &cfg); err != nil { 418 | return Config{}, err 419 | } 420 | 421 | return cfg, nil 422 | } 423 | 424 | func dataDirPath(ctx *cli.Context) string { 425 | dataDir := ctx.String("data-dir") 426 | 427 | if dataDir == "" { 428 | homeDir, err := os.UserHomeDir() 429 | if err != nil { 430 | homeDir = "./" 431 | } 432 | 433 | dataDir = path.Join(homeDir, "/.autoretrieve") 434 | } 435 | 436 | return dataDir 437 | } 438 | 439 | func fullConfigPath(ctx *cli.Context) string { 440 | return path.Join(dataDirPath(ctx), configPath) 441 | } 442 | 443 | // Modifies a config in-place using args passed in through CLI 444 | func applyConfigCLIOverrides(ctx *cli.Context, cfg *Config) error { 445 | if ctx.IsSet("lookup-endpoint-type") { 446 | lookupEndpointType, err := ParseEndpointType(ctx.String("lookup-endpoint-type")) 447 | if err != nil { 448 | return err 449 | } 450 | 451 | cfg.LookupEndpointType = lookupEndpointType 452 | } 453 | 454 | if ctx.IsSet("lookup-endpoint-url") { 455 | cfg.LookupEndpointURL = ctx.String("lookup-endpoint-url") 456 | } 457 | 458 | if ctx.IsSet("routing-table-type") { 459 | routingTableType, err := bitswap.ParseRoutingTableType(ctx.String("routing-table-type")) 460 | if err != nil { 461 | return err 462 | } 463 | 464 | cfg.RoutingTableType = routingTableType 465 | } 466 | 467 | if ctx.IsSet("disable-retrieval") { 468 | cfg.DisableRetrieval = ctx.Bool("disable-retrieval") 469 | } 470 | 471 | if ctx.IsSet("log-resource-manager") { 472 | cfg.LogResourceManager = ctx.Bool("log-resource-manager") 473 | } 474 | 475 | return nil 476 | } 477 | -------------------------------------------------------------------------------- /messagepusher/msgpusher.go: -------------------------------------------------------------------------------- 1 | package messagepusher 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/filecoin-project/go-address" 9 | "github.com/filecoin-project/go-state-types/abi" 10 | "github.com/filecoin-project/go-state-types/big" 11 | "github.com/filecoin-project/lotus/api" 12 | "github.com/filecoin-project/lotus/chain/types" 13 | "github.com/filecoin-project/lotus/chain/wallet" 14 | ) 15 | 16 | // a simple nonce tracking message pusher that assumes it is the only thing with access to the given key 17 | type MsgPusher struct { 18 | gapi api.Gateway 19 | w *wallet.LocalWallet 20 | 21 | nlk sync.Mutex 22 | nonces map[address.Address]uint64 23 | } 24 | 25 | func NewMsgPusher(gapi api.Gateway, w *wallet.LocalWallet) *MsgPusher { 26 | return &MsgPusher{ 27 | gapi: gapi, 28 | w: w, 29 | nonces: make(map[address.Address]uint64), 30 | } 31 | } 32 | 33 | func (mp *MsgPusher) MpoolPushMessage(ctx context.Context, msg *types.Message, maxFee *api.MessageSendSpec) (*types.SignedMessage, error) { 34 | mp.nlk.Lock() 35 | defer mp.nlk.Unlock() 36 | 37 | kaddr, err := mp.gapi.StateAccountKey(ctx, msg.From, types.EmptyTSK) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | n, ok := mp.nonces[kaddr] 43 | if !ok { 44 | act, err := mp.gapi.StateGetActor(ctx, kaddr, types.EmptyTSK) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | n = act.Nonce 50 | mp.nonces[kaddr] = n 51 | } 52 | 53 | msg.Nonce = n 54 | 55 | estim, err := mp.gapi.GasEstimateMessageGas(ctx, msg, &api.MessageSendSpec{}, types.EmptyTSK) 56 | if err != nil { 57 | return nil, fmt.Errorf("failed to estimate gas: %w", err) 58 | } 59 | 60 | estim.GasFeeCap = abi.NewTokenAmount(4000000000) 61 | estim.GasPremium = big.Mul(estim.GasPremium, big.NewInt(2)) 62 | 63 | sig, err := mp.w.WalletSign(ctx, kaddr, estim.Cid().Bytes(), api.MsgMeta{Type: api.MTChainMsg}) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | smsg := &types.SignedMessage{ 69 | Message: *estim, 70 | Signature: *sig, 71 | } 72 | 73 | _, err = mp.gapi.MpoolPush(ctx, smsg) 74 | if err != nil { 75 | return nil, err 76 | } 77 | 78 | mp.nonces[kaddr]++ 79 | 80 | return smsg, nil 81 | } 82 | -------------------------------------------------------------------------------- /metrics/opentelemetry.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "go.opencensus.io/stats" 5 | "go.opencensus.io/stats/view" 6 | "go.opencensus.io/tag" 7 | ) 8 | 9 | // Measures 10 | var ( 11 | BitswapRequestCount = stats.Int64("bitswap_request_total", "The number of bitswap requests received", stats.UnitDimensionless) 12 | BitswapResponseCount = stats.Int64("bitswap_response_total", "The number of bitswap responses", stats.UnitDimensionless) 13 | BitswapRetrieverRequestCount = stats.Int64("bitswap_retriever_request_total", "The number of bitswap messages that required a retriever lookup", stats.UnitDimensionless) 14 | BlockstoreCacheHitCount = stats.Int64("blockstore_cache_hit_total", "The number of blocks from the local blockstore served to peers", stats.UnitDimensionless) 15 | BytesTransferredTotal = stats.Int64("data_transferred_bytes_total", "The number of bytes transferred from storage providers to retrieval clients", stats.UnitBytes) 16 | RetrievalDealCost = stats.Int64("retrieval_deal_cost_fil", "The cost in FIL of a retrieval deal with a storage provider", stats.UnitDimensionless) 17 | RetrievalDealActiveCount = stats.Int64("retrieval_deal_active_total", "The number of active retrieval deals that have not yet succeeded or failed", stats.UnitDimensionless) 18 | RetrievalDealDuration = stats.Float64("retrieval_deal_duration_seconds", "The duration in seconds of a retrieval deal with a storage provider", stats.UnitSeconds) 19 | RetrievalDealFailCount = stats.Int64("retrieval_deal_fail_total", "The number of failed retrieval deals with storage providers", stats.UnitDimensionless) 20 | RetrievalDealSize = stats.Int64("retrieval_deal_size_bytes", "The size in bytes of a retrieval deal with a storage provider", stats.UnitDimensionless) 21 | RetrievalDealSuccessCount = stats.Int64("retrieval_deal_success_total", "The number of successful retrieval deals with storage providers", stats.UnitDimensionless) 22 | RetrievalRequestCount = stats.Int64("retrieval_request_total", "The number of retrieval deals initiated with storage providers", stats.UnitDimensionless) 23 | RetrievalErrorPaychCount = stats.Int64("retrieval_error_paych_total", "The number of retrieval errors for 'failed to get payment channel'", stats.UnitDimensionless) 24 | RetrievalErrorRejectedCount = stats.Int64("retrieval_error_rejected_total", "The number of retrieval errors for 'response rejected'", stats.UnitDimensionless) 25 | RetrievalErrorTooManyCount = stats.Int64("retrieval_error_toomany_total", "The number of retrieval errors for 'Too many retrieval deals received'", stats.UnitDimensionless) 26 | RetrievalErrorACLCount = stats.Int64("retrieval_error_acl_total", "The number of retrieval errors for 'Access Control'", stats.UnitDimensionless) 27 | RetrievalErrorMaintenanceCount = stats.Int64("retrieval_error_maintenance_total", "The number of retrieval errors for 'Under maintenance, retry later'", stats.UnitDimensionless) 28 | RetrievalErrorNoOnlineCount = stats.Int64("retrieval_error_noonline_total", "The number of retrieval errors for 'miner is not accepting online retrieval deals'", stats.UnitDimensionless) 29 | RetrievalErrorUnconfirmedCount = stats.Int64("retrieval_error_unconfirmed_total", "The number of retrieval errors for 'unconfirmed block transfer'", stats.UnitDimensionless) 30 | RetrievalErrorTimeoutCount = stats.Int64("retrieval_error_timeout_total", "The number of retrieval errors for 'timeout after X'", stats.UnitDimensionless) 31 | RetrievalErrorOtherCount = stats.Int64("retrieval_error_other_total", "The number of retrieval errors with uncategorized causes", stats.UnitDimensionless) 32 | QueryErrorFailedToDialCount = stats.Int64("query_error_failed_to_dial", "The number of query errors because we coult not connect to the provider", stats.UnitDimensionless) 33 | QueryErrorFailedToOpenStreamCount = stats.Int64("query_error_failed_to_open_stream", "The number of query errors where the miner did not respond on the retrieval protocol", stats.UnitDimensionless) 34 | QueryErrorResponseEOFCount = stats.Int64("query_error_response_eof", "The number of query errors because where the response terminated early", stats.UnitDimensionless) 35 | QueryErrorResponseStreamResetCount = stats.Int64("query_error_response_stream_reset", "The number of query errors because where the response stream was reset", stats.UnitDimensionless) 36 | QueryErrorDAGStoreCount = stats.Int64("query_error_dagstore", "The number of query failures cause the provider experienced a DAG Store error", stats.UnitDimensionless) 37 | QueryErrorDealNotFoundCount = stats.Int64("query_error_dealstate", "The number of query failures cause the provider couldn't find the relevant deal", stats.UnitDimensionless) 38 | QueryErrorOtherCount = stats.Int64("query_error_error_other_total", "The number of retrieval errors with uncategorized causes", stats.UnitDimensionless) 39 | 40 | // Indexer Candidates 41 | IndexerCandidatesPerRequestCount = stats.Int64("indexer_candidates_per_request_total", "The number of indexer candidates received per request", stats.UnitDimensionless) 42 | RequestWithIndexerCandidatesCount = stats.Int64("request_with_indexer_candidates_total", "The number of requests that result in non-zero candidates from the indexer", stats.UnitDimensionless) 43 | RequestWithIndexerCandidatesFilteredCount = stats.Int64("request_with_indexer_candidates_filtered_total", "The number of requests that result in non-zero candidates from the indexer after filtering", stats.UnitDimensionless) 44 | 45 | // Query 46 | RequestWithSuccessfulQueriesCount = stats.Int64("request_with_successful_queries_total", "The number of requests that result in a non-zero number of successful queries from SPs", stats.UnitDimensionless) 47 | RequestWithSuccessfulQueriesFilteredCount = stats.Int64("request_with_successful_queries_filtered_total", "The number of requests that result in a non-zero number of successful queries from SPs after filtering", stats.UnitDimensionless) 48 | SuccessfulQueriesPerRequestCount = stats.Int64("successful_queries_per_request_total", "The number of successful queries received per request", stats.UnitDimensionless) 49 | SuccessfulQueriesPerRequestFilteredCount = stats.Int64("successful_queries_per_request_filtered_total", "The number of successful queries received per request after filtering", stats.UnitDimensionless) 50 | 51 | // Retrieval 52 | FailedRetrievalsPerRequestCount = stats.Int64("failed_retrievals_per_request_total", "The number of failed retrieval attempts per request", stats.UnitDimensionless) 53 | ) 54 | 55 | // QueryErrorMetricMatches is a mapping of retrieval error message substrings 56 | // during the query phase (i.e. that can be matched against error messages) 57 | // and metrics to report for that error. 58 | var QueryErrorMetricMatches = map[string]*stats.Int64Measure{ 59 | "failed to dial": QueryErrorFailedToDialCount, 60 | "failed to open stream to peer": QueryErrorFailedToOpenStreamCount, 61 | "failed to read response: EOF": QueryErrorResponseEOFCount, 62 | "failed to read response: stream reset": QueryErrorResponseStreamResetCount, 63 | } 64 | 65 | // QueryResponseMetricMatches is a mapping of retrieval error message substrings 66 | // during the query phase when a response is sent but it is a failure 67 | // and metrics to report for that failure. 68 | var QueryResponseMetricMatches = map[string]*stats.Int64Measure{ 69 | "getting pieces for cid": QueryErrorDAGStoreCount, 70 | "failed to fetch storage deal state": QueryErrorDealNotFoundCount, 71 | } 72 | 73 | // ErrorMetricMatches is a mapping of retrieval error message substrings (i.e. 74 | // that can be matched against error messages) and metrics to report for that 75 | // error. 76 | var ErrorMetricMatches = map[string]*stats.Int64Measure{ 77 | "failed to get payment channel": RetrievalErrorPaychCount, 78 | "response rejected": RetrievalErrorRejectedCount, 79 | "Too many retrieval deals received": RetrievalErrorTooManyCount, 80 | "Access Control": RetrievalErrorACLCount, 81 | "Under maintenance, retry later": RetrievalErrorMaintenanceCount, 82 | "miner is not accepting online retrieval deals": RetrievalErrorNoOnlineCount, 83 | "unconfirmed block transfer": RetrievalErrorUnconfirmedCount, 84 | "timeout after ": RetrievalErrorTimeoutCount, 85 | } 86 | 87 | // Tags 88 | var ( 89 | BitswapDontHaveReason, _ = tag.NewKey("bitswap_dont_have_reason") 90 | BitswapTopic, _ = tag.NewKey("bitswap_topic") 91 | EndpointURL, _ = tag.NewKey("endpoint_url") 92 | 93 | Error, _ = tag.NewKey("error") 94 | Method, _ = tag.NewKey("method") 95 | Status, _ = tag.NewKey("status") 96 | ) 97 | 98 | // Views 99 | var ( 100 | bitswapRequestView = &view.View{ 101 | Measure: BitswapRequestCount, 102 | Aggregation: view.Count(), 103 | } 104 | bitswapResponseView = &view.View{ 105 | Measure: BitswapResponseCount, 106 | Aggregation: view.Count(), 107 | TagKeys: []tag.Key{BitswapTopic, BitswapDontHaveReason}, 108 | } 109 | bitswapRetreiverRequestView = &view.View{ 110 | Measure: BitswapRetrieverRequestCount, 111 | Aggregation: view.Count(), 112 | } 113 | blockstoreCacheHitView = &view.View{ 114 | Measure: BlockstoreCacheHitCount, 115 | Aggregation: view.Count(), 116 | } 117 | bytesTransferredView = &view.View{ 118 | Measure: BytesTransferredTotal, 119 | Aggregation: view.Sum(), 120 | } 121 | failedRetrievalsPerRequestView = &view.View{ 122 | Measure: FailedRetrievalsPerRequestCount, 123 | Aggregation: view.Distribution(0, 1, 2, 3, 4, 5, 10, 20, 40), 124 | } 125 | requestWithIndexerCandidatesFilteredView = &view.View{ 126 | Measure: RequestWithIndexerCandidatesFilteredCount, 127 | Aggregation: view.Count(), 128 | } 129 | requestWithIndexerCandidatesView = &view.View{ 130 | Measure: RequestWithIndexerCandidatesCount, 131 | Aggregation: view.Count(), 132 | } 133 | requestWithSuccessfulQueriesFilteredView = &view.View{ 134 | Measure: RequestWithSuccessfulQueriesFilteredCount, 135 | Aggregation: view.Count(), 136 | } 137 | requestWithSuccessfulQueriesView = &view.View{ 138 | Measure: RequestWithSuccessfulQueriesCount, 139 | Aggregation: view.Count(), 140 | } 141 | indexerCandidatesPerRequestView = &view.View{ 142 | Measure: IndexerCandidatesPerRequestCount, 143 | Aggregation: view.Distribution(0, 1, 2, 3, 4, 5, 10, 20, 40), 144 | } 145 | retrievalDealActiveView = &view.View{ 146 | Measure: RetrievalDealActiveCount, 147 | Aggregation: view.Count(), 148 | } 149 | retrievalDealCostView = &view.View{ 150 | Measure: RetrievalDealCost, 151 | Aggregation: view.Distribution(), 152 | } 153 | retrievalDealDurationView = &view.View{ 154 | Measure: RetrievalDealDuration, 155 | Aggregation: view.Distribution(0, 10, 20, 30, 40, 50, 60, 120, 240, 480, 540, 600), 156 | } 157 | retrievalDealFailView = &view.View{ 158 | Measure: RetrievalDealFailCount, 159 | Aggregation: view.Count(), 160 | } 161 | retrievalDealSuccessView = &view.View{ 162 | Measure: RetrievalDealSuccessCount, 163 | Aggregation: view.Count(), 164 | } 165 | retrievalDealSizeView = &view.View{ 166 | Measure: RetrievalDealSize, 167 | Aggregation: view.Distribution(), 168 | } 169 | retrievalRequestCountView = &view.View{ 170 | Measure: RetrievalRequestCount, 171 | Aggregation: view.Count(), 172 | } 173 | retrievalErrorPaychView = &view.View{ 174 | Measure: RetrievalErrorPaychCount, 175 | Aggregation: view.Count(), 176 | } 177 | retrievalErrorRejectedView = &view.View{ 178 | Measure: RetrievalErrorRejectedCount, 179 | Aggregation: view.Count(), 180 | } 181 | retrievalErrorTooManyView = &view.View{ 182 | Measure: RetrievalErrorTooManyCount, 183 | Aggregation: view.Count(), 184 | } 185 | retrievalErrorACLView = &view.View{ 186 | Measure: RetrievalErrorACLCount, 187 | Aggregation: view.Count(), 188 | } 189 | retrievalErrorMaintenanceView = &view.View{ 190 | Measure: RetrievalErrorMaintenanceCount, 191 | Aggregation: view.Count(), 192 | } 193 | retrievalErrorNoOnlineView = &view.View{ 194 | Measure: RetrievalErrorNoOnlineCount, 195 | Aggregation: view.Count(), 196 | } 197 | retrievalErrorUnconfirmedView = &view.View{ 198 | Measure: RetrievalErrorUnconfirmedCount, 199 | Aggregation: view.Count(), 200 | } 201 | retrievalErrorTimeoutView = &view.View{ 202 | Measure: RetrievalErrorTimeoutCount, 203 | Aggregation: view.Count(), 204 | } 205 | retrievalErrorOtherView = &view.View{ 206 | Measure: RetrievalErrorOtherCount, 207 | Aggregation: view.Count(), 208 | } 209 | successfulQueriesPerRequestFilteredView = &view.View{ 210 | Measure: SuccessfulQueriesPerRequestFilteredCount, 211 | Aggregation: view.Distribution(0, 1, 2, 3, 4, 5, 10, 20, 40), 212 | } 213 | successfulQueriesPerRequestView = &view.View{ 214 | Measure: SuccessfulQueriesPerRequestCount, 215 | Aggregation: view.Distribution(0, 1, 2, 3, 4, 5, 10, 20, 40), 216 | } 217 | queryErrorFailedToDialView = &view.View{ 218 | Measure: QueryErrorFailedToDialCount, 219 | Aggregation: view.Count(), 220 | } 221 | queryErrorFailedToOpenStreamView = &view.View{ 222 | Measure: QueryErrorFailedToOpenStreamCount, 223 | Aggregation: view.Count(), 224 | } 225 | queryErrorResponseEOFView = &view.View{ 226 | Measure: QueryErrorResponseEOFCount, 227 | Aggregation: view.Count(), 228 | } 229 | queryErrorResponseStreamResetView = &view.View{ 230 | Measure: QueryErrorResponseStreamResetCount, 231 | Aggregation: view.Count(), 232 | } 233 | queryErrorDAGStoreView = &view.View{ 234 | Measure: QueryErrorDAGStoreCount, 235 | Aggregation: view.Count(), 236 | } 237 | queryErrorDealNotFoundView = &view.View{ 238 | Measure: QueryErrorDealNotFoundCount, 239 | Aggregation: view.Count(), 240 | } 241 | queryErrorOtherView = &view.View{ 242 | Measure: QueryErrorOtherCount, 243 | Aggregation: view.Count(), 244 | } 245 | ) 246 | 247 | var DefaultViews = []*view.View{ 248 | bitswapRequestView, 249 | bitswapResponseView, 250 | bitswapRetreiverRequestView, 251 | blockstoreCacheHitView, 252 | bytesTransferredView, 253 | failedRetrievalsPerRequestView, 254 | requestWithIndexerCandidatesFilteredView, 255 | requestWithIndexerCandidatesView, 256 | requestWithSuccessfulQueriesFilteredView, 257 | requestWithSuccessfulQueriesView, 258 | indexerCandidatesPerRequestView, 259 | retrievalDealActiveView, 260 | retrievalDealCostView, 261 | retrievalDealDurationView, 262 | retrievalDealFailView, 263 | retrievalDealSuccessView, 264 | retrievalDealSizeView, 265 | retrievalRequestCountView, 266 | retrievalErrorPaychView, 267 | retrievalErrorRejectedView, 268 | retrievalErrorTooManyView, 269 | retrievalErrorACLView, 270 | retrievalErrorMaintenanceView, 271 | retrievalErrorNoOnlineView, 272 | retrievalErrorUnconfirmedView, 273 | retrievalErrorTimeoutView, 274 | retrievalErrorOtherView, 275 | successfulQueriesPerRequestFilteredView, 276 | successfulQueriesPerRequestView, 277 | queryErrorFailedToDialView, 278 | queryErrorFailedToOpenStreamView, 279 | queryErrorResponseEOFView, 280 | queryErrorResponseStreamResetView, 281 | queryErrorDAGStoreView, 282 | queryErrorDealNotFoundView, 283 | queryErrorOtherView, 284 | } 285 | -------------------------------------------------------------------------------- /metrics/server.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "net/http" 5 | 6 | "contrib.go.opencensus.io/exporter/prometheus" 7 | logging "github.com/ipfs/go-log/v2" 8 | promclient "github.com/prometheus/client_golang/prometheus" 9 | "go.opencensus.io/stats/view" 10 | ) 11 | 12 | var logger = logging.Logger("metrics") 13 | 14 | func PrometheusHandler(views ...*view.View) http.Handler { 15 | if err := view.Register(DefaultViews...); err != nil { 16 | logger.Errorf("cannot register default metric views: %s", err) 17 | } 18 | 19 | if err := view.Register(views...); err != nil { 20 | logger.Errorf("cannot register metric views: %s", err) 21 | } 22 | 23 | registry, ok := promclient.DefaultRegisterer.(*promclient.Registry) 24 | if !ok { 25 | logger.Warnf("failed to export default prometheus registry; some metrics will be unavailable; unexpected type: %T", promclient.DefaultRegisterer) 26 | } 27 | 28 | exp, err := prometheus.NewExporter(prometheus.Options{ 29 | Registry: registry, 30 | Namespace: "autoretrieve", 31 | }) 32 | if err != nil { 33 | logger.Errorf("cannot create the prometheus stats exporter: %v", err) 34 | } 35 | 36 | return exp 37 | } 38 | -------------------------------------------------------------------------------- /minerpeergetter/minerpeergetter.go: -------------------------------------------------------------------------------- 1 | package minerpeergetter 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/filecoin-project/go-address" 8 | lotusapi "github.com/filecoin-project/lotus/api" 9 | "github.com/filecoin-project/lotus/chain/types" 10 | peer "github.com/libp2p/go-libp2p/core/peer" 11 | "github.com/multiformats/go-multiaddr" 12 | ) 13 | 14 | type MinerPeerGetter struct { 15 | api lotusapi.Gateway 16 | } 17 | 18 | func NewMinerPeerGetter(api lotusapi.Gateway) *MinerPeerGetter { 19 | return &MinerPeerGetter{api: api} 20 | } 21 | 22 | func (mpg *MinerPeerGetter) MinerPeer(ctx context.Context, miner address.Address) (peer.AddrInfo, error) { 23 | minfo, err := mpg.api.StateMinerInfo(ctx, miner, types.EmptyTSK) 24 | if err != nil { 25 | return peer.AddrInfo{}, err 26 | } 27 | 28 | if minfo.PeerId == nil { 29 | return peer.AddrInfo{}, fmt.Errorf("miner %s has no peer ID set", miner) 30 | } 31 | 32 | var maddrs []multiaddr.Multiaddr 33 | for _, mma := range minfo.Multiaddrs { 34 | ma, err := multiaddr.NewMultiaddrBytes(mma) 35 | if err != nil { 36 | return peer.AddrInfo{}, fmt.Errorf("miner %s had invalid multiaddrs in their info: %w", miner, err) 37 | } 38 | maddrs = append(maddrs, ma) 39 | } 40 | 41 | return peer.AddrInfo{ 42 | ID: *minfo.PeerId, 43 | Addrs: maddrs, 44 | }, nil 45 | } 46 | -------------------------------------------------------------------------------- /paychannelapi.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/application-research/autoretrieve/messagepusher" 7 | "github.com/filecoin-project/go-address" 8 | "github.com/filecoin-project/go-state-types/crypto" 9 | "github.com/filecoin-project/lotus/api" 10 | "github.com/filecoin-project/lotus/chain/types" 11 | "github.com/filecoin-project/lotus/chain/wallet" 12 | ) 13 | 14 | type payChannelApiProvider struct { 15 | api.Gateway 16 | wallet *wallet.LocalWallet 17 | mp *messagepusher.MsgPusher 18 | } 19 | 20 | func (a *payChannelApiProvider) MpoolPushMessage(ctx context.Context, msg *types.Message, maxFee *api.MessageSendSpec) (*types.SignedMessage, error) { 21 | return a.mp.MpoolPushMessage(ctx, msg, maxFee) 22 | } 23 | 24 | func (a *payChannelApiProvider) WalletHas(ctx context.Context, addr address.Address) (bool, error) { 25 | return a.wallet.WalletHas(ctx, addr) 26 | } 27 | 28 | func (a *payChannelApiProvider) WalletSign(ctx context.Context, addr address.Address, data []byte) (*crypto.Signature, error) { 29 | return a.wallet.WalletSign(ctx, addr, data, api.MsgMeta{Type: api.MTUnknown}) 30 | } 31 | -------------------------------------------------------------------------------- /paychannelmanager/lassiepaychannelmanager.go: -------------------------------------------------------------------------------- 1 | package paychannelmanager 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/filecoin-project/go-address" 8 | "github.com/filecoin-project/go-state-types/big" 9 | "github.com/filecoin-project/go-state-types/builtin/v8/paych" 10 | "github.com/filecoin-project/lotus/chain/stmgr" 11 | "github.com/filecoin-project/lotus/chain/types" 12 | "github.com/filecoin-project/lotus/paychmgr" 13 | ) 14 | 15 | type LassiePayChannelManager struct { 16 | *paychmgr.Manager 17 | clientAddress address.Address 18 | } 19 | 20 | func NewLassiePayChannelManager(ctx context.Context, shutdown func(), stateManagerAPI stmgr.StateManagerAPI, payChanStore *paychmgr.Store, payChanApi paychmgr.PaychAPI) *LassiePayChannelManager { 21 | payChanMgr := paychmgr.NewManager(ctx, shutdown, stateManagerAPI, payChanStore, payChanApi) 22 | return &LassiePayChannelManager{ 23 | Manager: payChanMgr, 24 | } 25 | } 26 | 27 | func (lpcm *LassiePayChannelManager) CreateVoucher(ctx context.Context, ch address.Address, voucher paych.SignedVoucher) (*paych.SignedVoucher, big.Int, error) { 28 | result, err := lpcm.Manager.CreateVoucher(ctx, ch, voucher) 29 | return result.Voucher, result.Shortfall, err 30 | } 31 | 32 | func (lpcm *LassiePayChannelManager) GetPayChannelWithMinFunds(ctx context.Context, dest address.Address) (address.Address, error) { 33 | avail, err := lpcm.Manager.AvailableFundsByFromTo(ctx, lpcm.clientAddress, dest) 34 | if err != nil { 35 | return address.Undef, err 36 | } 37 | 38 | reqBalance, err := types.ParseFIL("0.01") 39 | if err != nil { 40 | return address.Undef, err 41 | } 42 | fmt.Println("available", avail.ConfirmedAmt) 43 | 44 | if types.BigCmp(avail.ConfirmedAmt, types.BigInt(reqBalance)) >= 0 { 45 | return *avail.Channel, nil 46 | } 47 | 48 | amount := types.BigMul(types.BigInt(reqBalance), types.NewInt(2)) 49 | 50 | fmt.Println("getting payment channel: ", lpcm.clientAddress, dest, amount) 51 | pchaddr, mcid, err := lpcm.Manager.GetPaych(ctx, lpcm.clientAddress, dest, amount, paychmgr.GetOpts{ 52 | Reserve: false, 53 | OffChain: false, 54 | }) 55 | if err != nil { 56 | return address.Undef, fmt.Errorf("failed to get payment channel: %w", err) 57 | } 58 | 59 | fmt.Println("got payment channel: ", pchaddr, mcid) 60 | if !mcid.Defined() { 61 | if pchaddr == address.Undef { 62 | return address.Undef, fmt.Errorf("GetPaych returned nothing") 63 | } 64 | 65 | return pchaddr, nil 66 | } 67 | 68 | return lpcm.Manager.GetPaychWaitReady(ctx, mcid) 69 | } 70 | -------------------------------------------------------------------------------- /scripts/autoretrieve-service/autoretrieve-register.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Register Autoretrieve on Estuary 3 | 4 | [Service] 5 | Type=oneshot 6 | EnvironmentFile=/etc/estuary/config.env 7 | ExecStart=/usr/local/bin/autoretrieve register-estuary ${ESTUARY_API} ${ESTUARY_ADMIN_TOKEN} 8 | 9 | [Install] 10 | WantedBy=multi-user.target 11 | -------------------------------------------------------------------------------- /scripts/autoretrieve-service/autoretrieve.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Autoretrieve 3 | After=network-online.target 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/autoretrieve 7 | Restart=always 8 | RestartSec=10 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /scripts/autoretrieve-service/config.env: -------------------------------------------------------------------------------- 1 | ESTUARY_URL=http://localhost:3004 2 | ESTUARY_ADMIN_TOKEN= 3 | GOLOG_FILE="/var/log/autoretrieve.log" 4 | GOLOG_OUTPUT="stderr+file" 5 | --------------------------------------------------------------------------------